<commit_before># -*- encoding: utf-8 -*-
import csv
import urllib
import urlparse
import sys
import time
from lxml.html import parse
import eventlet
from eventlet.green import urllib2

from searchtester.scoring import calculate_score

class SearchSystemTest(object):

    def __init__(self, endpoint, selector='ol li a', param='q', extra={}):
        self.endpoint = endpoint
        self.selector = selector
        self.param = param
        self.params = extra

    def make_url(self, query):
        params = {
            self.param: query.encode('utf-8'),
        }
        params.update(self.params)
        endpoint = list(urlparse.urlparse(self.endpoint))
        existing = urlparse.parse_qs(endpoint[4])
        params.update(existing)
        endpoint[4] = urllib.urlencode(params, True)
        url = urlparse.urlunparse(endpoint)
        return url

    def test_search(self, query, expected):
        """
        Takes a query string and a list of expected result URLs.
        Returns the number of each URL if on the first result
        page, or None in a position if not present.
        """
        #print expected, query
        url = self.make_url(query)
        # apply base of the search URL endpoint
        expected = map(lambda x: urlparse.urljoin(url, x), expected)
        f = urllib2.urlopen(url)
        root = parse(f).getroot()
        f.close()
        #print root
        
        positions = {} # index in expected list
        for i, link in enumerate(root.cssselect(self.selector)):
            actual = link.get("href")
            resolved = urlparse.urljoin(url, actual)
            for idx, possible in enumerate(expected):
                # print i, idx, actual, resolved, possible
                
                # note that we could just do:
                # if resolved == possible and positions.get(idx, None) is None:
                # but we don't so that duplicate search results get spotted.
                if resolved == possible:
                    if positions.get(idx, None) is not None:
                        # print "found", possible, "already at", positions[idx]
                        del positions[idx]
                    else:
                        # print "found", possible, "at", i
                        positions[idx] = i
        result = []
        for i in range(0, len(expected)):
            result.append(positions.get(i, None))
        return result

    def test_searches(self, in_filename, out_filename, poolsize=10):
        """
        Takes a file where each line gives a search and a list of
        expected URLs.
        """<commit_msg>Refactor the core matching code.<commit_after># -*- encoding: utf-8 -*-
import csv
import urllib
import urlparse
import sys
import time
from lxml.html import parse
import eventlet
from eventlet.green import urllib2

from searchtester.scoring import calculate_score

def find_expected_result_positions(results, expected, searchurl):
    positions = {} # index in expected list
    for i, link in enumerate(results):
        resolved = urlparse.urljoin(searchurl, link)
        #print link, resolved
        for idx, possible in enumerate(expected):
            #print i, idx, link, resolved, possible
                
            # note that we could just do:
            # if resolved == possible and positions.get(idx, None) is None:
            # but we don't so that duplicate search results get spotted.
            if resolved == possible:
                if positions.get(idx, None) is not None:
                    #print "found", possible, "already at", positions[idx]
                    del positions[idx]
                else:
                    #print "found", possible, "at", i
                    positions[idx] = i
    result = []
    for i in range(0, len(expected)):
        result.append(positions.get(i, None))
    return result


class SearchSystemTest(object):

    def __init__(self, endpoint, selector='ol li a', param='q', extra={}):
        self.endpoint = endpoint
        self.selector = selector
        self.param = param
        self.params = extra

    def make_url(self, query):
        params = {
            self.param: query.encode('utf-8'),
        }
        params.update(self.params)
        endpoint = list(urlparse.urlparse(self.endpoint))
        existing = urlparse.parse_qs(endpoint[4])
        params.update(existing)
        endpoint[4] = urllib.urlencode(params, True)
        url = urlparse.urlunparse(endpoint)
        return url

    def test_search(self, query, expected):
        """
        Takes a query string and a list of expected result URLs.
        Returns the number of each URL if on the first result
        page, or None in a position if not present.
        """
        #print expected, query
        url = self.make_url(query)
        # apply base of the search URL endpoint
        expected = map(lambda x: urlparse.urljoin(url, x), expected)
        f = urllib2.urlopen(url)
        root = parse(f).getroot()
        f.close()
        #print root
        
        return find_expected_result_positions(
            map(lambda x: x.get("href"), root.cssselect(self.selector)),
            expected,
            url
        )

    def test_searches(self, in_filename, out_filename, poolsize=10):
        """
        Takes a file where each line gives a search and a list of
        expected URLs.
        """
{'fraction_non_alphanumeric': 0.09166666666666666, 'fraction_numerical': 0.0033333333333333335, 'mean_word_length': 1.9839779005524862, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2841167', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1409, 'n_words': 475}