# 2012/5/25
# for Python v3.x

import os
import sys
import re
import time
import urllib.request

# DEFAULT PARAMETERS:
# ++++++++++++++++++++++
MAX_DEPTH_NUM = 5   # The "depth" means that how many times you follow the link tree in web pages.
                    # You can override the default MAX_DEPTH_NUM by the second argument of the script.
MIN_ACCESS_INTERVAL_SEC = 0 # n seconds. 0 is recommended if MAX_ROBOTS_CACHE_NUM > 0.
URL_PROXY_DICT = {}    # A dictionary of protocols and proxies.
                       # Example: URL_PROXY_DICT = {"http":"192.168.3.150:8080"}
EXC_EXTENSIONS_RESTR = "gif|png|bmp|jpg|jpeg|img|mov|mpg|mpeg|avi|exe|gz|zip|css|pdf|rdf"
                       # A string of file extensions to be excluded from the robot exploration.
                       # List file extensions that you want to exlude by the regular expression OR ('|').
                       # The robot will not follow web links to files of these extensions.
                       # Case incensitive.
HTTP_REQ_TIMEOUT_SEC = 3 # n seconds.
MAX_COLLECTED_URLS_NUM = 0 # Slice the collected_urls to the MIN.  no slice if it is 0.
MIN_COLLECTED_URLS_NUM = 0 # Valid only when MAX_COLLECTED_URLS_NUM > 0.
MAX_ROBOTS_CACHE_NUM = 10000    # How many robots.txts you cache.
                                # No robots.txt check if it is 0.
MY_ROBOT_NAME_STR = 'Python3-urllib'
# ++++++++++++++++++++++

# FUNCTIONS:
def make_robots_check_func(my_robot_name, url_opener, robots_timeout=5, max_cache=10000):
    '''
        Called in get_urls().
        A higher-order function that returns sub-function that checks robots.txt and returns check results.  The higher-order function is used to set default values and keep parameters for the sub-fucntions, as a kind of closure.
        The same thing could be implemented by a class.  But I don't like using class to work out just a simple thing.
    '''

    import fnmatch
    baseurl_re = re.compile('(https?://[^/]+)', re.I)
    robots_dict = {}
    robots_re = re.compile('(User-agent: *)([^\s#]+)(.*\n)((?:.+\n?)+)', re.I)
    robots_rule_line_re = re.compile('^(Disallow|Crawl-delay): *([^\s#]+).*$', re.I | re.M)
    
    tagCanFetch = 'can-fetch'
    tagDisallow = 'disallow'
    tagCrawlDelay = 'crawl-delay'
    tagBaseUrl = 'base-url'
    tagRobotsDict = 'robots-dict'
    
    def check_robots(url):
        can_fetch = True
        baseurl = baseurl_re.match(url).group(1)
        
        if len(robots_dict) > max_cache:
            robots_dict.clear()
        
        if max_cache > 0:
            if not baseurl in robots_dict:
                robots_dict[baseurl] = {tagCrawlDelay:0, tagDisallow:[]}
                try:
                    got_robots_txt = url_opener.open(baseurl + '/robots.txt', timeout=robots_timeout)
                    got_robots_cont = got_robots_txt.read().decode('utf-8')
                    
                    applied_rules = ''
                    for m in robots_re.finditer(got_robots_cont):
                        if fnmatch.fnmatch(my_robot_name, m.group(2)):
                            applied_rules = m.groups()[3]
                            break
                        elif m.group(2) == '*':
                            applied_rules = m.groups()[3]
                    
                    if applied_rules != '':
                        for rdm in robots_rule_line_re.finditer(applied_rules):
                            if rdm.group(1).lower() == tagCrawlDelay and rdm.group(2).isdigit():
                                robots_dict[baseurl][tagCrawlDelay] = int(rdm.group(2))
                            if rdm.group(1).lower() == tagDisallow:
                                robots_dict[baseurl][tagDisallow].append(rdm.group(2).lower())
                except:
                    pass
        else:
            robots_dict[baseurl] = {tagCrawlDelay:0, tagDisallow:[]}
        
        if robots_dict[baseurl][tagDisallow]:
            for rls in robots_dict[baseurl][tagDisallow]:
                if rls == '/' or fnmatch.fnmatch(url.lower(), baseurl.lower() + rls + '*'):
                    can_fetch = False
        
        return {tagCanFetch:can_fetch, 
                tagBaseUrl:baseurl, 
                tagCrawlDelay:robots_dict[baseurl][tagCrawlDelay], 
                tagRobotsDict:robots_dict
                }
    
    return check_robots


def get_urls(start_url, max_depth, default_interval_sec, url_proxy, exc_extensions, http_request_timeout, min_collected_urls_num, max_collected_urls_num, my_robot_name, max_robots_num):
    urls = [(start_url, 0)]    # the array of tuple(url, link_depth).  urls alread got.
    collected_urls = [start_url.rstrip("/")]    # a list of already got urls that include all urls that were still not retrieved.
    href_re = re.compile("href=\"(https?://[^\'\">]+)\"", re.I)
    exsuffixre = re.compile("\\.(" + exc_extensions + ")$", re.I)
    
    # getting OpenerDirector object instance.
    op = urllib.request.build_opener(urllib.request.ProxyHandler(url_proxy))
    rb_func = make_robots_check_func(my_robot_name, op, max_cache=max_robots_num)
    
    while len(urls) > 0:
        target = urls.pop()
        collected_urls_length = len(collected_urls)
        
        try:
            # checking robots.txt and access permission from this robot.
            rb_result = rb_func(target[0])
            can_fetch = rb_result['can-fetch']
            base_url = rb_result['base-url']
            robots_dict = rb_result['robots-dict']
            crawl_delay = robots_dict[base_url]['crawl-delay']
            if not can_fetch:
                raise ValueError('<Rejected by the robots.txt of {0}>'.format(base_url))
            elif crawl_delay > http_request_timeout:
                raise ValueError('<Too long crawl-delay time ({0} sec) in the robots.txt of {1}>'.format(crawl_delay, base_url))
            elif crawl_delay > 0:
                interval_sec = crawl_delay
                print('<Delay waiting... {0} sec for {1}>'.format(crawl_delay, target[0]), file=sys.stderr, end=os.linesep)
            else:
                interval_sec = default_interval_sec
            
            time.sleep(interval_sec)
            with op.open(target[0], timeout=http_request_timeout) as got_req:    # OpenerDirector.open(url, data, timeout)
                print('[depth-now={0}:unexplored-url-links={1}:checked-urls={2}:robots.txt-cache={3}] {4}'.format(target[1], len(urls), collected_urls_length, len(robots_dict), target[0]), end=os.linesep)
                
                reqi = got_req.info()
                depth_now = target[1]
                
                if depth_now < int(max_depth) and reqi.get_content_type().find("text") >= 0:
                    chset = reqi.get_charsets()[0]
                    
                    if chset:
                        got_page = got_req.read().decode(chset)
                    else:
                        got_page = str(got_req.read())

                    for matched_url in href_re.finditer(got_page):    # as read() returns bytes.  3.x is stricter than v2.x in the point.
                        if matched_url:
                            got_url = matched_url.group(1)
                        else:
                            continue
                        
                        if not exsuffixre.search(got_url):
                            if not got_url.rstrip("/") in collected_urls:
                                urls.append((got_url, depth_now + 1))
                                if 0 < max_collected_urls_num < collected_urls_length:
                                    cidx = collected_urls_length - min_collected_urls_num
                                    collected_urls[:] = collected_urls[cidx:]
                                collected_urls.append(got_url.rstrip("/"))
                            # else:
                            #     print('<The URL has been already checked: {0}>'.format(got_url), file=sys.stderr, end=os.linesep)
                        # else:
                        #     print('<The file extension was excluded: {0}>'.format(got_url), file=sys.stderr, end=os.linesep)
        except KeyboardInterrupt:
            return
        except (Exception, BaseException) as e:
            print(str(e) + ": " + target[0], file=sys.stderr)
            continue
            #raise



get_urls(sys.argv[1], 
            MAX_DEPTH_NUM if len(sys.argv) <= 2 else sys.argv[2], 
            MIN_ACCESS_INTERVAL_SEC, 
            URL_PROXY_DICT,
            EXC_EXTENSIONS_RESTR,
            HTTP_REQ_TIMEOUT_SEC,
            MIN_COLLECTED_URLS_NUM,
            MAX_COLLECTED_URLS_NUM,
            MY_ROBOT_NAME_STR,
            MAX_ROBOTS_CACHE_NUM
            )