<commit_before>from __future__ import print_function, unicode_literals

from email.parser import Parser
import json
from os import path
import re
import tarfile
import zipfile

import pkg_resources

from binstar_client import errors
from binstar_client.inspect_package.uitls import extract_first, pop_key


def parse_requirement(line, deps, extras, extra):
    req = pkg_resources.Requirement.parse(line)
    if extra:
        extras[extra].append({'name':req.key, 'specs': req.specs or []})
    else:
        deps.append({'name':req.key, 'specs': req.specs or []})

def parse_requires_txt(requires_txt):
    deps = []
    error = False
    extras = {}
    extra = None

    for line in requires_txt.split('\n'):
        line = line.strip()

        if not line: continue

        if line.startswith('[') and line.endswith(']'):
            extra = line[1:-1]
            extras.setdefault(extra, [])
            # Dont parse this requirement
            continue
        try:
            parse_requirement(line, deps, extras, extra)
        except ValueError as err:
            error = True

    extras = [{'name': k, 'depends': v} for (k, v) in extras.items()]
    return {'has_dep_errors': error, 'depends': deps,
            'extras': extras}

def format_rqeuirements(requires):
    obj = []
    for req in requires:
        req = req.strip()
        req_spec = req.split(' ', 1)
        if len(req_spec) == 1:
            obj.append({'name': req, 'specs': []})
        else:
            req, spec = req_spec
            spec = spec.strip()
            if spec[0] == '(': spec = spec[1:]
            if spec[-1] == ')': spec = spec[:-1]

            req = pkg_resources.Requirement.parse('%s %s' % (req, spec))
            obj.append({'name': req.key, 'specs': req.specs or []})

    return obj

def format_run_requires_metadata(run_requires):
    deps = []
    extras = []
    environments = []
    for run_require in run_requires:
        extra = run_require.get('extra')
        env = run_require.get('environment')
        requires = run_require['requires']

        if env:
            obj = []
            environments.append({'name': env, 'depends': obj})
        elif extra is None:
            obj = deps
        else:
            obj = []
            extras.append({'name': extra, 'depends': obj})

        obj.extend(format_rqeuirements(requires))

    attrs = {'has_dep_errors': False, 'depends': deps,
             'extras':extras,
             'environments': environments}

    return attrs

def format_requires_metadata(run_requires):
    deps = []
    extras = []

    extras_re = re.compile('extra == [\'\"](.*)[\'\"]')
    has_dep_errors = False

    if not isinstance(run_requires, dict):
        if isinstance(run_requires, (basestring)):
            run_requires = {'': [run_requires]}<commit_msg>fix formatting<commit_after>from __future__ import print_function, unicode_literals

from email.parser import Parser
import json
from os import path
import re
import tarfile
import zipfile

import pkg_resources

from binstar_client import errors
from binstar_client.inspect_package.uitls import extract_first, pop_key


def parse_requirement(line, deps, extras, extra):
    req = pkg_resources.Requirement.parse(line)
    if extra:
        extras[extra].append({'name':req.key, 'specs': req.specs or []})
    else:
        deps.append({'name':req.key, 'specs': req.specs or []})

def parse_requires_txt(requires_txt):
    deps = []
    error = False
    extras = {}
    extra = None

    for line in requires_txt.split('\n'):
        line = line.strip()

        if not line: continue

        if line.startswith('[') and line.endswith(']'):
            extra = line[1:-1]
            extras.setdefault(extra, [])
            # Dont parse this requirement
            continue
        try:
            parse_requirement(line, deps, extras, extra)
        except ValueError as err:
            error = True

    extras = [{'name': k, 'depends': v} for (k, v) in extras.items()]
    return {'has_dep_errors': error, 'depends': deps,
            'extras': extras}

def format_rqeuirements(requires):
    obj = []
    for req in requires:
        req = req.strip()
        req_spec = req.split(' ', 1)
        if len(req_spec) == 1:
            obj.append({'name': req, 'specs': []})
        else:
            req, spec = req_spec
            spec = spec.strip()
            if spec[0] == '(': spec = spec[1:]
            if spec[-1] == ')': spec = spec[:-1]

            req = pkg_resources.Requirement.parse('%s %s' % (req, spec))
            obj.append({'name': req.key, 'specs': req.specs or []})

    return obj

def format_run_requires_metadata(run_requires):
    deps = []
    extras = []
    environments = []
    for run_require in run_requires:
        extra = run_require.get('extra')
        env = run_require.get('environment')
        requires = run_require['requires']

        if env:
            obj = []
            environments.append({'name': env, 'depends': obj})
        elif extra is None:
            obj = deps
        else:
            obj = []
            extras.append({'name': extra, 'depends': obj})

        obj.extend(format_rqeuirements(requires))

    attrs = {'has_dep_errors': False, 'depends': deps,
             'extras':extras,
             'environments': environments}

    return attrs

def format_requires_metadata(run_requires):
    deps = []
    extras = []

    extras_re = re.compile('extra == [\'\"](.*)[\'\"]')
    has_dep_errors = False

    if not isinstance(run_requires, dict):
        if isinstance(run_requires, basestring):
            run_requires = {'': [run_requires]}
{'fraction_non_alphanumeric': 0.13423056490171772, 'fraction_numerical': 0.0028333628475296618, 'mean_word_length': 2.330188679245283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1544017', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1734, 'n_words': 428}