<commit_before>#
#  The OpenDiamond Platform for Interactive Search
#
#  Copyright (c) 2011 Carnegie Mellon University
#  All rights reserved.
#
#  This software is distributed under the terms of the Eclipse Public
#  License, Version 1.0 which can be found in the file named LICENSE.
#  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
#  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
#

'''Scope list retrieval, parsing, and iteration.'''

from __future__ import with_statement
import logging
import urllib2
from urlparse import urljoin
import threading
from xml.sax import make_parser, SAXParseException
from xml.sax.handler import ContentHandler

from opendiamond.server.object_ import Object

BASE_URL = 'http://localhost:5873/'

_log = logging.getLogger(__name__)

class _ScopeListHandler(ContentHandler):
    '''Gatherer for results produced by incremental scope list parsing.'''

    def __init__(self):
        ContentHandler.__init__(self)
        self.count = 0
        self.pending_objects = []

    # We're overriding a method; we can't control its name
    # pylint: disable=C0103
    def startElement(self, name, attrs):
        if name == 'objectlist':
            # count is optional
            count = attrs.get('count')
            if count is not None:
                self.count += int(count)
        elif name == 'count':
            self.count += int(attrs['adjust'])
        elif name == 'object':
            self.pending_objects.append(attrs['src'])
    # pylint: enable=C0103


class ScopeListLoader(object):
    '''Iterator over the objects in the scope lists referenced by the scope
    cookies.'''

    def __init__(self, config, server_id, cookies):
        self.server_id = server_id
        self.cookies = cookies
        self._config = config
        self._lock = threading.Lock()
        self._handler = _ScopeListHandler()
        self._generator = self._generator_func()

    def __iter__(self):
        return self

    # Generators confuse pylint, pylint #20062
    # pylint: disable=E1101
    def next(self):
        '''Return the next Object.'''
        with self._lock:
            return self._generator.next()
    # pylint: enable=E1101

    def _generator_func(self):
        # Build URL opener
        handlers = []
        if self._config.http_proxy is not None:
            handlers.append(urllib2.ProxyHandler({
                'http': self._config.http_proxy,
                'https': self._config.http_proxy,
            }))
        opener = urllib2.build_opener(*handlers)
        opener.addheaders = [('User-Agent', self._config.user_agent)]
        # Build XML parser
        parser = make_parser()
        parser.setContentHandler(self._handler)
        # Walk scope URLs
        for cookie in self.cookies:
            for scope_url in cookie:
                scope_url = urljoin(BASE_URL, scope_url)
                try:
                    # We use urllib2 here because different parts of a single
                    # HTTP response will be handled from different threads.
                    # pycurl does not support this.
                    fh = opener.open(scope_url)
                    # Read the scope list in 4 KB chunks
                    while True:
                        buf = fh.read(4096)
                        if len(buf) == 0:
                            break
                        parser.feed(buf)
                        while len(self._handler.pending_objects) > 0:<commit_msg>opendiamond.server.scopelist: Drop obsolete pylint override<commit_after>#
#  The OpenDiamond Platform for Interactive Search
#
#  Copyright (c) 2011 Carnegie Mellon University
#  All rights reserved.
#
#  This software is distributed under the terms of the Eclipse Public
#  License, Version 1.0 which can be found in the file named LICENSE.
#  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
#  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
#

'''Scope list retrieval, parsing, and iteration.'''

from __future__ import with_statement
import logging
import urllib2
from urlparse import urljoin
import threading
from xml.sax import make_parser, SAXParseException
from xml.sax.handler import ContentHandler

from opendiamond.server.object_ import Object

BASE_URL = 'http://localhost:5873/'

_log = logging.getLogger(__name__)

class _ScopeListHandler(ContentHandler):
    '''Gatherer for results produced by incremental scope list parsing.'''

    def __init__(self):
        ContentHandler.__init__(self)
        self.count = 0
        self.pending_objects = []

    # We're overriding a method; we can't control its name
    # pylint: disable=C0103
    def startElement(self, name, attrs):
        if name == 'objectlist':
            # count is optional
            count = attrs.get('count')
            if count is not None:
                self.count += int(count)
        elif name == 'count':
            self.count += int(attrs['adjust'])
        elif name == 'object':
            self.pending_objects.append(attrs['src'])
    # pylint: enable=C0103


class ScopeListLoader(object):
    '''Iterator over the objects in the scope lists referenced by the scope
    cookies.'''

    def __init__(self, config, server_id, cookies):
        self.server_id = server_id
        self.cookies = cookies
        self._config = config
        self._lock = threading.Lock()
        self._handler = _ScopeListHandler()
        self._generator = self._generator_func()

    def __iter__(self):
        return self

    def next(self):
        '''Return the next Object.'''
        with self._lock:
            return self._generator.next()

    def _generator_func(self):
        # Build URL opener
        handlers = []
        if self._config.http_proxy is not None:
            handlers.append(urllib2.ProxyHandler({
                'http': self._config.http_proxy,
                'https': self._config.http_proxy,
            }))
        opener = urllib2.build_opener(*handlers)
        opener.addheaders = [('User-Agent', self._config.user_agent)]
        # Build XML parser
        parser = make_parser()
        parser.setContentHandler(self._handler)
        # Walk scope URLs
        for cookie in self.cookies:
            for scope_url in cookie:
                scope_url = urljoin(BASE_URL, scope_url)
                try:
                    # We use urllib2 here because different parts of a single
                    # HTTP response will be handled from different threads.
                    # pycurl does not support this.
                    fh = opener.open(scope_url)
                    # Read the scope list in 4 KB chunks
                    while True:
                        buf = fh.read(4096)
                        if len(buf) == 0:
                            break
                        parser.feed(buf)
                        while len(self._handler.pending_objects) > 0:
{'fraction_non_alphanumeric': 0.08263264714409974, 'fraction_numerical': 0.010582777616700493, 'mean_word_length': 2.1259628454916175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3690091', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1808, 'n_words': 574}