<commit_before>#!/usr/bin/python

import sys, pprint, cgi, urllib, os

def flatten(d):
    l = d[1]
    for entry in d[0].values():
        l += flatten(entry)
    return l

class Query:
    """ Represents a Flamenco-style query.

        .query : (string, int)
         Maps top-level classifier names (e.g. 'Environment', or 'Topic')
         to an integer representing the value to be searched for.  Top-level
         classifiers can be specified multiple times.
    """
    def __init__(self, cursor, trove, query=[]):
        self.cursor = cursor
        self.trove = trove
        self.query = query
        self.field_list = []
        for group, tid in query:
            try:
                int(tid)
            except ValueError:
                # skip invalid input
                continue
            self.field_list.append(self.trove[int(tid)].path_split)

        # get the packages that match classifiers
        self.cursor.execute('''
select rc.trove_id, r.name, r.version, r.summary
from releases r, release_classifiers rc
where r.name=rc.name and r.version=rc.version
  and r._pypi_hidden=FALSE
''')

        # Now sort into useful structures
        self.by_classifier = {}
        self.by_package = {}
        self.pkg_summary = {}
        for tid, name, version, summary in self.cursor.fetchall():
            self.pkg_summary[(name, version)] = summary
            l = self.trove[int(tid)].path_split
            self.by_package.setdefault((name, version), []).append(l)
            d = self.by_classifier
            for arc in l[:-1]:
                if not d.has_key(arc):
                    d[arc] = ({}, [])
                d = d[arc][0]
            arc = l[-1]
            if not d.has_key(arc):
                d[arc] = ({}, [])
            d[arc][1].append((name, version))

    def get_matches(self, addl_fields=[]):
        matches = {}
        query_fields = self.field_list + addl_fields
        for package, classifiers in self.by_package.items():
            for required in query_fields:
                # make sure the field appears in this package's classifiers
                for classifier in classifiers:
                    if classifier[:len(required)] == required:
                        # match, yay
                        break<commit_msg>more unicode fun<commit_after>#!/usr/bin/python

import sys, pprint, cgi, urllib, os

def flatten(d):
    l = d[1]
    for entry in d[0].values():
        l += flatten(entry)
    return l

class Query:
    """ Represents a Flamenco-style query.

        .query : (string, int)
         Maps top-level classifier names (e.g. 'Environment', or 'Topic')
         to an integer representing the value to be searched for.  Top-level
         classifiers can be specified multiple times.
    """
    def __init__(self, cursor, trove, query=[]):
        self.cursor = cursor
        self.trove = trove
        self.query = query
        self.field_list = []
        for group, tid in query:
            try:
                int(tid)
            except ValueError:
                # skip invalid input
                continue
            self.field_list.append(self.trove[int(tid)].path_split)

        # get the packages that match classifiers
        self.cursor.execute('''
select rc.trove_id, r.name, r.version, r.summary
from releases r, release_classifiers rc
where r.name=rc.name and r.version=rc.version
  and r._pypi_hidden=FALSE
''')

        # Now sort into useful structures
        self.by_classifier = {}
        self.by_package = {}
        self.pkg_summary = {}
        for tid, name, version, summary in self.cursor.fetchall():
            if not isinstance(name, unicode):
                name = name.decode('utf-8')
            if not isinstance(version, unicode):
                version = version.decode('utf-8')
            if summary and not isinstance(summary, unicode):
                summary = summary.decode('utf-8')
            self.pkg_summary[(name, version)] = summary
            l = self.trove[int(tid)].path_split
            self.by_package.setdefault((name, version), []).append(l)
            d = self.by_classifier
            for arc in l[:-1]:
                if not d.has_key(arc):
                    d[arc] = ({}, [])
                d = d[arc][0]
            arc = l[-1]
            if not d.has_key(arc):
                d[arc] = ({}, [])
            d[arc][1].append((name, version))

    def get_matches(self, addl_fields=[]):
        matches = {}
        query_fields = self.field_list + addl_fields
        for package, classifiers in self.by_package.items():
            for required in query_fields:
                # make sure the field appears in this package's classifiers
                for classifier in classifiers:
                    if classifier[:len(required)] == required:
                        # match, yay
                        break
{'fraction_non_alphanumeric': 0.11020241259456144, 'fraction_numerical': 0.003066857493355142, 'mean_word_length': 1.7545045045045045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2583175', 'n_tokens_mistral': 1424, 'n_tokens_neox': 1352, 'n_words': 381}