# Path: pipeline/archivebot/wpull/ignoracle.py
# class Ignoracle(object):
#     '''
#     An Ignoracle tests a URL against a list of patterns and returns whether or
#     not that URL should be grabbed.
# 
#     An Ignoracle's pattern list starts as the empty list.
#     '''
# 
#     # Note that set_patterns() is called from a different thread than ignores().
#     # The lock prevents race conditions on iteration over self._compiled.
# 
#     patterns = []
# 
#     def __init__(self):
#         self._primary = None
#         self._compiled = []
#         self._lock = threading.Lock()
# 
#     def set_patterns(self, strings):
#         '''
#         Given a list of strings, replaces this Ignoracle's pattern state with
#         that list.
#         '''
# 
#         patterns = []
# 
#         for string in strings:
#             if isinstance(string, bytes):
#                 string = string.decode('utf-8')
# 
#             patterns.append(string)
# 
#         with self._lock:
#             self.patterns = patterns
#             self._primary = None
#             # Don't replace _compiled here; _primary acts as a trigger for the recompilation.
# 
#     def ignores(self, url_record: wpull.pipeline.item.URLRecord):
#         '''
#         If an ignore pattern matches the given URL, returns that pattern as a string.
#         Otherwise, returns False.
#         '''
# 
#         params = parameterize_record_info(url_record)
# 
#         primaryUrl = params.get('primary_url') or ''
#         primaryNetloc = params.get('primary_netloc') or ''
#         if self._primary != (primaryUrl, primaryNetloc):
#             with self._lock:
#                 self._compiled = []
#                 escapedPrimaryUrl = re.escape(primaryUrl)
#                 escapedPrimaryNetloc = re.escape(primaryNetloc)
#                 for pattern in self.patterns:
#                     try:
#                         expanded = pattern.replace('{primary_url}', escapedPrimaryUrl)
#                         expanded = expanded.replace('{primary_netloc}', escapedPrimaryNetloc)
#                         compiledPattern = re.compile(expanded)
#                     except re.error as error:
#                         print('Pattern %s is invalid (error: %s).  Ignored.'
#                               % (pattern, str(error)), file=sys.stderr)
#                     self._compiled.append((pattern, compiledPattern))
#                 self._primary = (primaryUrl, primaryNetloc)
# 
#         for pattern, compiled in self._compiled:
#             match = compiled.search(url_record.url)
# 
#             if match:
#                 return pattern
# 
#         return False
# 
# def parameterize_record_info(record_info: wpull.pipeline.item.URLRecord):
#     '''
#     Given a wpull record_info dict, generates a dict with primary_url and
#     primary_netloc keys.  This is meant to be used in Ignoracle.ignores.
# 
#     The primary_url key is generally the URL the job was started with, or a
#     URL from a URL list.
# 
#     If primary_url is a valid URL, the primary_netloc key is the network
#     location component of primary_url (i.e. for HTTP,
#     [user:password@]host[:port]).  Otherwise, primary_netloc is None.
#     '''
# 
#     primary_url = None
#     primary_netloc = None
# 
#     if record_info.level == 0:
#         primary_url = record_info.url
#     else:
#         primary_url = record_info.parent_url
# 
#     if primary_url:
#         parsed = urlparse(primary_url)
#         primary_netloc = parsed.netloc
# 
#     return dict(
#         primary_url=primary_url,
#         primary_netloc=primary_netloc
#     )


import unittest
import re
from .ignoracle import Ignoracle, parameterize_record_info

    def test_set_patterns_converts_bytes_to_utf8(self):
        self.oracle.set_patterns([b'foobar'])

        self.assertEqual(self.oracle.patterns[0], 'foobar')

@unittest.skip("Pending a fix for wpull 2.x interface")
class TestRecordInfoParameterization(unittest.TestCase):
    def test_uses_top_url_if_present(self):
        record_info = dict(
            top_url='http://www.example.com/'
        )

        result = parameterize_record_info(record_info)

        self.assertEqual('http://www.example.com/', result['primary_url'])
        self.assertEqual('www.example.com', result['primary_netloc'])

    def test_uses_url_for_level_zero_url(self):
        record_info = dict(
            url='http://www.example.com/',
            level=0
        )

        result = parameterize_record_info(record_info)

        self.assertEqual('http://www.example.com/', result['primary_url'])
        self.assertEqual('www.example.com', result['primary_netloc'])

    def test_missing_primary_url_results_in_no_netloc(self):

# Path: pipeline/archivebot/wpull/ignoracle.py
# class Ignoracle(object):
#     '''
#     An Ignoracle tests a URL against a list of patterns and returns whether or
#     not that URL should be grabbed.
# 
#     An Ignoracle's pattern list starts as the empty list.
#     '''
# 
#     # Note that set_patterns() is called from a different thread than ignores().
#     # The lock prevents race conditions on iteration over self._compiled.
# 
#     patterns = []
# 
#     def __init__(self):
#         self._primary = None
#         self._compiled = []
#         self._lock = threading.Lock()
# 
#     def set_patterns(self, strings):
#         '''
#         Given a list of strings, replaces this Ignoracle's pattern state with
#         that list.
#         '''
# 
#         patterns = []
# 
#         for string in strings:
#             if isinstance(string, bytes):
#                 string = string.decode('utf-8')
# 
#             patterns.append(string)
# 
#         with self._lock:
#             self.patterns = patterns
#             self._primary = None
#             # Don't replace _compiled here; _primary acts as a trigger for the recompilation.
# 
#     def ignores(self, url_record: wpull.pipeline.item.URLRecord):
#         '''
#         If an ignore pattern matches the given URL, returns that pattern as a string.
#         Otherwise, returns False.
#         '''
# 
#         params = parameterize_record_info(url_record)
# 
#         primaryUrl = params.get('primary_url') or ''
#         primaryNetloc = params.get('primary_netloc') or ''
#         if self._primary != (primaryUrl, primaryNetloc):
#             with self._lock:
#                 self._compiled = []
#                 escapedPrimaryUrl = re.escape(primaryUrl)
#                 escapedPrimaryNetloc = re.escape(primaryNetloc)
#                 for pattern in self.patterns:
#                     try:
#                         expanded = pattern.replace('{primary_url}', escapedPrimaryUrl)
#                         expanded = expanded.replace('{primary_netloc}', escapedPrimaryNetloc)
#                         compiledPattern = re.compile(expanded)
#                     except re.error as error:
#                         print('Pattern %s is invalid (error: %s).  Ignored.'
#                               % (pattern, str(error)), file=sys.stderr)
#                     self._compiled.append((pattern, compiledPattern))
#                 self._primary = (primaryUrl, primaryNetloc)
# 
#         for pattern, compiled in self._compiled:
#             match = compiled.search(url_record.url)
# 
#             if match:
#                 return pattern
# 
#         return False
# 
# def parameterize_record_info(record_info: wpull.pipeline.item.URLRecord):
#     '''
#     Given a wpull record_info dict, generates a dict with primary_url and
#     primary_netloc keys.  This is meant to be used in Ignoracle.ignores.
# 
#     The primary_url key is generally the URL the job was started with, or a
#     URL from a URL list.
# 
#     If primary_url is a valid URL, the primary_netloc key is the network
#     location component of primary_url (i.e. for HTTP,
#     [user:password@]host[:port]).  Otherwise, primary_netloc is None.
#     '''
# 
#     primary_url = None
#     primary_netloc = None
# 
#     if record_info.level == 0:
#         primary_url = record_info.url
#     else:
#         primary_url = record_info.parent_url
# 
#     if primary_url:
#         parsed = urlparse(primary_url)
#         primary_netloc = parsed.netloc
# 
#     return dict(
#         primary_url=primary_url,
#         primary_netloc=primary_netloc
#     )


# Path: pipeline/archivebot/wpull/ignoracle_test.py
import unittest
import re
from .ignoracle import Ignoracle, parameterize_record_info


    def test_set_patterns_converts_bytes_to_utf8(self):
        self.oracle.set_patterns([b'foobar'])

        self.assertEqual(self.oracle.patterns[0], 'foobar')

@unittest.skip("Pending a fix for wpull 2.x interface")
class TestRecordInfoParameterization(unittest.TestCase):
    def test_uses_top_url_if_present(self):
        record_info = dict(
            top_url='http://www.example.com/'
        )

        result = parameterize_record_info(record_info)

        self.assertEqual('http://www.example.com/', result['primary_url'])
        self.assertEqual('www.example.com', result['primary_netloc'])

    def test_uses_url_for_level_zero_url(self):
        record_info = dict(
            url='http://www.example.com/',
            level=0
        )

        result = parameterize_record_info(record_info)

        self.assertEqual('http://www.example.com/', result['primary_url'])
        self.assertEqual('www.example.com', result['primary_netloc'])

    def test_missing_primary_url_results_in_no_netloc(self):

        result = parameterize_record_info(dict())