<commit_before>import glob
import os
import re
from collections import defaultdict
from subprocess import check_output, CalledProcessError

from jinja2 import Environment, PackageLoader

from s3pypi import __prog__
from s3pypi.exceptions import S3PyPiError

__author__ = '<NAME>'
__copyright__ = 'Copyright 2016, November Five'
__license__ = 'MIT'


class Package(object):
    """Python package."""

    def __init__(self, name, files):
        self.name, self.version = name.rsplit('-', 1)
        self.files = set(files)

    def __str__(self):
        return '%s-%s' % (self.name, self.version)

    def _attrs(self):
        return self.name, self.version

    def __lt__(self, other):
        return self.version < other.version

    def __eq__(self, other):
        return isinstance(other, Package) and self._attrs() == other._attrs()

    def __hash__(self):
        return hash(self._attrs())

    @property
    def directory(self):
        return re.sub('[-_.]+', '-', self.name)

    @staticmethod
    def _find_package_name(text):
        match = re.search('^(copying files to|making hard links in) (.+)\.\.\.', text, flags=re.MULTILINE)

        if not match:
            raise RuntimeError('Package name not found in:\n' + text)

        return match.group(2)

    @staticmethod
    def create(wheel=True):
        cmd = ['python', 'setup.py', 'sdist', '--formats', 'gztar']

        if wheel:
            cmd.append('bdist_wheel')

        try:
            stdout = check_output(cmd).decode().strip()
        except CalledProcessError as e:
            raise RuntimeError(e.output.rstrip())

        name = Package._find_package_name(stdout)
        files = [name + '.tar.gz']

        if wheel:
            files.extend(os.path.basename(path) for path in
                         glob.glob(os.path.join('dist', name + '-*.whl')))

        return Package(name, files)


class Index(object):
    """Index of package versions, to be rendered to HTML."""

    template = Environment(loader=PackageLoader(__prog__, 'templates')).get_template('index.html.j2')

    def __init__(self, packages):
        self.packages = set(packages)

    @staticmethod
    def parse(html):
        filenames = defaultdict(set)

        for match in re.findall('<a href="((.+?-\d+\.\d+\.\d+).+)">', html):
            filenames[match[1]].add(match[0])

        return Index(Package(name, files) for name, files in filenames.items())
<commit_msg>Fix wheel file matching (#31)<commit_after>import glob
import os
import re
from collections import defaultdict
from subprocess import check_output, CalledProcessError

from jinja2 import Environment, PackageLoader

from s3pypi import __prog__
from s3pypi.exceptions import S3PyPiError

__author__ = '<NAME>'
__copyright__ = 'Copyright 2016, November Five'
__license__ = 'MIT'


class Package(object):
    """Python package."""

    def __init__(self, name, files):
        self.name, self.version = name.rsplit('-', 1)
        self.files = set(files)

    def __str__(self):
        return '%s-%s' % (self.name, self.version)

    def _attrs(self):
        return self.name, self.version

    def __lt__(self, other):
        return self.version < other.version

    def __eq__(self, other):
        return isinstance(other, Package) and self._attrs() == other._attrs()

    def __hash__(self):
        return hash(self._attrs())

    @property
    def directory(self):
        return re.sub('[-_.]+', '-', self.name)

    @staticmethod
    def _find_package_name(text):
        match = re.search('^(copying files to|making hard links in) (.+)\.\.\.', text, flags=re.MULTILINE)

        if not match:
            raise RuntimeError('Package name not found in:\n' + text)

        return match.group(2)

    @staticmethod
    def _find_wheel_name(text):
        match = re.search('creating \'.*(dist.*\.whl)\' and adding', text, flags=re.MULTILINE)

        if not match:
            raise RuntimeError('Wheel name not found in:\n' + text)

        return match.group(1)

    @staticmethod
    def create(wheel=True):
        cmd = ['python', 'setup.py', 'sdist', '--formats', 'gztar']

        if wheel:
            cmd.append('bdist_wheel')

        try:
            stdout = check_output(cmd).decode().strip()
        except CalledProcessError as e:
            raise RuntimeError(e.output.rstrip())

        name = Package._find_package_name(stdout)
        files = [name + '.tar.gz']

        if wheel:
            files += [os.path.basename(Package._find_wheel_name(stdout))]

        return Package(name, files)


class Index(object):
    """Index of package versions, to be rendered to HTML."""

    template = Environment(loader=PackageLoader(__prog__, 'templates')).get_template('index.html.j2')

    def __init__(self, packages):
        self.packages = set(packages)

    @staticmethod
    def parse(html):
        filenames = defaultdict(set)

        for match in re.findall('<a href="((.+?-\d+\.\d+\.\d+).+)">', html):
            filenames[match[1]].add(match[0])

        return Index(Package(name, files) for name, files in filenames.items())

{'fraction_non_alphanumeric': 0.13733568765940748, 'fraction_numerical': 0.005689621345889739, 'mean_word_length': 2.7184536834427426, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2834496', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1501, 'n_words': 397}