<commit_before>#!/usr/bin/env python

import os
import re
import json
import logging
import threading

from urlparse import urlparse
from optparse import OptionParser
from collections import OrderedDict

from . import thread
from .exc import CheckerException
from .checker import (
    phantomjs_checker,
    requests_checker,
    standardize_url,
)

path = os.path.abspath(__file__)
dir_path = os.path.dirname(path)

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(thread)d - %(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
log.addHandler(handler)


class Linkchecker(object):

    def __init__(self, pool, ignore_ssl_errors=False,
                 ignore_url_patterns=None, domains=None, maxdepth=None,
                 timeout=None):
        self.pool = pool
        # Option passed to phantomjs
        self.ignore_ssl_errors = ignore_ssl_errors

        self.ignored_urls_regex = set(re.compile(p)
                                      for p in ignore_url_patterns or [])
        self.valid_domains = domains or []
        self.maxdepth = maxdepth
        self.timeout = timeout

        self.__checkLock = threading.Condition(threading.Lock())
        self.results = OrderedDict()

        # Security to not crawl all the web
        self.max_nb_urls = 50000

    def check(self, url, parent_url=None, depth=0):
        try:
            result = phantomjs_checker(
                url, parent_url=parent_url,
                ignore_ssl_errors=self.ignore_ssl_errors)
            for page in result:
                if self.results.get(page['url']) is None:
                    self.results[page['url']] = page

            if len(self.results) > self.max_nb_urls:
                return

            depth += 1
            if self.maxdepth is not None and depth > self.maxdepth:
                # We are too far
                return
            self.feed_result(url, result[-1], depth)
        except CheckerException as e:
            if self.results.get(url) is None:
                self.results[url] = {
                    'checker': 'phantomjs_checker',
                    'url': url,
                    'redirect_url': None,
                    'status_code': 500,
                    'status': 'Internal Error - %s' % unicode(e),
                    'parent_url': parent_url,
                }

    def quick_check(self, url, parent_url=None):
        try:
            result = requests_checker(url, self.ignore_ssl_errors)
            for page in result:
                if self.results.get(page['url']) is None:
                    self.results[page['url']] = page
        except CheckerException as e:
            if self.results.get(url) is None:
                self.results[url] = {
                    'checker': 'requests',
                    'url': url,
                    'redirect_url': None,
                    'status_code': 500,
                    'status': 'Internal Error - %s' % unicode(e),
                    'parent_url': parent_url,
                }

    def schedule(self, url, parent_url=None, new_depth=0):
        self.results[standardize_url(url)] = None
        if urlparse(url).hostname not in self.valid_domains:
            self.pool.add_task(self.quick_check, parent_url=parent_url,
                               url=url)
        else:
            self.pool.add_task(self.check, url=url, parent_url=parent_url,
                               depth=new_depth)

    def filter_ignore_urls_patterns(self, urls):
        return (u for u in urls
                if not any(r.search(u) for r in self.ignored_urls_regex))

    def feed_result(self, url, result, new_depth):
        urls_to_check = self.filter_ignore_urls_patterns(result['urls'])

        # Feed discovered url to the checker
        self.__checkLock.acquire()
        try:
            for next_url in urls_to_check:<commit_msg>Check the link in the last (not too far) page<commit_after>#!/usr/bin/env python

import os
import re
import json
import logging
import threading

from urlparse import urlparse
from optparse import OptionParser
from collections import OrderedDict

from . import thread
from .exc import CheckerException
from .checker import (
    phantomjs_checker,
    requests_checker,
    standardize_url,
)

path = os.path.abspath(__file__)
dir_path = os.path.dirname(path)

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(thread)d - %(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
log.addHandler(handler)


class Linkchecker(object):

    def __init__(self, pool, ignore_ssl_errors=False,
                 ignore_url_patterns=None, domains=None, maxdepth=None,
                 timeout=None):
        self.pool = pool
        # Option passed to phantomjs
        self.ignore_ssl_errors = ignore_ssl_errors

        self.ignored_urls_regex = set(re.compile(p)
                                      for p in ignore_url_patterns or [])
        self.valid_domains = domains or []
        self.maxdepth = maxdepth
        self.timeout = timeout

        self.__checkLock = threading.Condition(threading.Lock())
        self.results = OrderedDict()

        # Security to not crawl all the web
        self.max_nb_urls = 50000

    def check(self, url, parent_url=None, depth=0):
        try:
            result = phantomjs_checker(
                url, parent_url=parent_url,
                ignore_ssl_errors=self.ignore_ssl_errors)
        except CheckerException as e:
            if self.results.get(url) is None:
                self.results[url] = {
                    'checker': 'phantomjs_checker',
                    'url': standardize_url(url),
                    'redirect_url': None,
                    'status_code': 500,
                    'status': 'Internal Error - %s' % unicode(e),
                    'parent_url': parent_url,
                }

        for page in result:
            if self.results.get(page['url']) is None:
                self.results[page['url']] = page

        if len(self.results) > self.max_nb_urls:
            return
        self.feed_result(url, result[-1], depth+1)

    def quick_check(self, url, parent_url=None):
        try:
            result = requests_checker(url, parent_url=parent_url,
                                      ignore_ssl_errors=self.ignore_ssl_errors)
            for page in result:
                if self.results.get(page['url']) is None:
                    self.results[page['url']] = page
        except CheckerException as e:
            if self.results.get(url) is None:
                self.results[url] = {
                    'checker': 'requests',
                    'url': standardize_url(url),
                    'redirect_url': None,
                    'status_code': 500,
                    'status': 'Internal Error - %s' % unicode(e),
                    'parent_url': parent_url,
                }

    def schedule(self, url, parent_url=None, new_depth=0):
        too_far = False
        if self.maxdepth is not None:
            if new_depth > (self.maxdepth + 1):
                # We are too far
                return
            if new_depth > self.maxdepth:
                # Check the link with requestspy
                too_far = True
        self.results[standardize_url(url)] = None

        if too_far or urlparse(url).hostname not in self.valid_domains:
            self.pool.add_task(self.quick_check, parent_url=parent_url,
                               url=url)
        else:
            self.pool.add_task(self.check, url=url, parent_url=parent_url,
                               depth=new_depth)

    def filter_ignore_urls_patterns(self, urls):
        return (u for u in urls
                if not any(r.search(u) for r in self.ignored_urls_regex))

    def feed_result(self, url, result, new_depth):
        urls_to_check = self.filter_ignore_urls_patterns(result['urls'])

        # Feed discovered url to the checker
        self.__checkLock.acquire()
        try:
            for next_url in urls_to_check:
{'fraction_non_alphanumeric': 0.09306260575296109, 'fraction_numerical': 0.003746676335508823, 'mean_word_length': 1.8893156424581006, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4970400', 'n_tokens_mistral': 2446, 'n_tokens_neox': 2333, 'n_words': 552}