<commit_before>import sys

import envoy

def run_stdout(cmd, env=None, cwd=None):
    child = envoy.run(cmd, env=env)
    if child.status_code != 0:
        raise SubprocessError(cmd)
    return child.std_out

class SubprocessError(Exception):
    def __init__(self, cmd):
        super(SubprocessError, self).__init__('Failed to run: "%s"' % (cmd,))

class WeightedDistanceScore(object):
    def __init__(self):
        self._needle = ''

    def update(self, needle):
        self._needle = needle

    def score(self, path):
        needle_len = len(self._needle)
        if needle_len <= 0:
            return (0, path)

        mult = path.find(self._needle[-1])
        if mult < 0:
            return (-1, '')

        return (mult * needle_len, path[mult + 1:])

class FuzzyMatch(object):
    def __init__(self, files=None, scorer=None):
        self._library = {}
        if files is not None:
            for f in files:
                self._library[f] = {
                        'score': 0,
                        'remainder': f}
        self._matches = self._library.copy()
        self._scorer = scorer
        self._search = ''

    def update_scores(self, search):
        if len(search) < len(self._search):
            self._matches = self._library.copy()

        self._scorer.update(search)
        scorer = self._scorer.score

        for path, vals in self._matches.items():
            score, remainder = scorer(vals['remainder'])
            if score < 0:
                del self._matches[path]
            else:
                self._matches[path]['score'] = score
                self._matches[path]['remainder'] = remainder

    def top_matches(self, depth=10):
        ret = sorted(self._matches, key=lambda x: self._matches[x]['score'])[:depth]
        ret.reverse()
        return ret

class Scanner(object):
    def __init__(self, scanner_conf):
        self._dir_type = None
        self._conf = scanner_conf

        c = envoy.run('git rev-parse')
        if c.status_code == 0:
            self._dir_type = 'git'

    def scan(self):
        try:
            if self._dir_type == 'git':
                files = run_stdout(self._conf['git'])
            else:
                files = run_stdout(self._conf['default'])
        except KeyError:
            files = run_stdout(self._conf['default'])

        return [file.strip() for file in files.split()]

<commit_msg>pyfs:  correct top_matches order<commit_after>import sys

import envoy

def run_stdout(cmd, env=None, cwd=None):
    child = envoy.run(cmd, env=env)
    if child.status_code != 0:
        raise SubprocessError(cmd)
    return child.std_out

class SubprocessError(Exception):
    def __init__(self, cmd):
        super(SubprocessError, self).__init__('Failed to run: "%s"' % (cmd,))

class WeightedDistanceScore(object):
    def __init__(self):
        self._needle = ''

    def update(self, needle):
        self._needle = needle

    def score(self, path):
        needle_len = len(self._needle)
        if needle_len <= 0:
            return (0, path)

        mult = path.find(self._needle[-1])
        if mult < 0:
            return (-1, '')

        return (mult * needle_len, path[mult + 1:])

class FuzzyMatch(object):
    def __init__(self, files=None, scorer=None):
        self._library = {}
        if files is not None:
            for f in files:
                self._library[f] = {
                        'score': 0,
                        'remainder': f}
        self._matches = self._library.copy()
        self._scorer = scorer
        self._search = ''

    def update_scores(self, search):
        if len(search) < len(self._search):
            self._matches = self._library.copy()

        self._scorer.update(search)
        scorer = self._scorer.score

        for path, vals in self._matches.items():
            score, remainder = scorer(vals['remainder'])
            if score < 0:
                del self._matches[path]
            else:
                self._matches[path]['score'] = score
                self._matches[path]['remainder'] = remainder

    def top_matches(self, depth=10):
        ret = sorted(self._matches, key=lambda x: self._matches[x]['score'])[:depth]
        return ret

class Scanner(object):
    def __init__(self, scanner_conf):
        self._dir_type = None
        self._conf = scanner_conf

        c = envoy.run('git rev-parse')
        if c.status_code == 0:
            self._dir_type = 'git'

    def scan(self):
        try:
            if self._dir_type == 'git':
                files = run_stdout(self._conf['git'])
            else:
                files = run_stdout(self._conf['default'])
        except KeyError:
            files = run_stdout(self._conf['default'])

        return [file.strip() for file in files.split()]


{'fraction_non_alphanumeric': 0.12283989173433271, 'fraction_numerical': 0.004996876951905059, 'mean_word_length': 1.9526736324523664, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2167311', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1403, 'n_words': 334}