<commit_before>from collections import defaultdict

from .models import Module, Release
from .semver import ForgeSpec


def release_specs(release, forgespec=True):
    """
    For the given release, return a list of all the dependencies as 2-tuples,
    comprising the module name and dependency specification (as ForgeSpec).
    """
    specs = []
    for depend in release.metadata.get('dependencies', []):
        spec = depend.get('version_requirement', '>= 0.0.0')
        if forgespec: spec = ForgeSpec(spec)
        specs.append((depend['name'], spec))
    return specs


def calculate_dependencies(release):
    queue = set([release])
    dependencies = defaultdict(set)
    dependencies[unicode(release.module)].add(
        ForgeSpec(str(release.version))
    )

    while queue:
        rel = queue.pop()
        for dep_item in release_specs(rel):
            dep_mod, dep_spec = dep_item
            if not dep_spec in dependencies[dep_mod]:
                dependencies[dep_mod].add(dep_spec)
                try:
                    for dep_rel in Module.get_for_full_name(dep_mod).releases.all():
                        if dep_rel.version in dep_spec:
                            queue.add(dep_rel)
                except Module.DoesNotExist:
                    raise Exception('Dependency module %s not found' % dep_mod)

    return dependencies


def release_dependencies(release):
    """
    This determines the dependencies for the given module release.
    """
    dependency_specs = calculate_dependencies(release)
    dependencies = defaultdict(list)

    for depend, specs in dependency_specs.iteritems():
        try:
            mod = Module.get_for_full_name(depend)
        except Module.DoesNotExist:
            raise Exception('Dependency module %s not found' % depend)
        releases = []
        for rel in mod.releases.all():
            if all([rel.version in spec for spec in specs]):
                dependencies[depend].append({
                        'version': str(rel.version),
                        'file': rel.tarball.url,
                        'dependencies': release_specs(rel, forgespec=False),
                })

    return dependencies<commit_msg>Added comments/docstrings, caching, and some simplification.<commit_after>from collections import defaultdict

from .models import Module, Release
from .semver import ForgeSpec


def release_specs(release):
    """
    For the given release, return a list of all the dependencies as 2-tuples,
    comprising the module name and dependency specification as given in the
    release's metadata.
    """
    return [
        (depend['name'],
         depend.get('version_requirement', '>= 0.0.0'))
        for depend in release.metadata.get('dependencies', [])
    ]


def calculate_dependencies(release):
    """
    This function does the heavy lifting of calculating dependencies for the
    given Release instance.  It returns two data structures:

     * A dictionary of dependencies, mapping the legacy name of a module
       the release depends on (e.g., 'puppetlabs/apache') to a set of
       version specifications (e.g., '>= 1.0.0') as ForgeSpec objects.
     * A cache dictionary mapping a release's primary key it's string
       version requirements as expressed in its metadata (in other words,
       what's returned by `release_specs()` for each release.
    """
    # Prime the queue with the given release.
    queue = set([release])

    # Add the given release to the dependency structure, and lock it's
    # version specification at the version of the release.  Use the
    # legacy module name, as that what the Forge uses in its metadata
    # to express dependencies.
    dependencies = defaultdict(set)
    dependencies[release.module.legacy_name].add(
        ForgeSpec(str(release.version))
    )

    # Create dictionary for mapping of releases to their list of dependency
    # specifications.
    spec_cache = {}

    while queue:
        # Pop the next release from the queue, and get it's dependency
        # specifications (caching them for later use).
        rel = queue.pop()
        if not rel.pk in spec_cache:
            spec_cache[rel.pk] = release_specs(rel)

        for dep_item in spec_cache[rel.pk]:
            # Pull out the legacy module name, and create a `ForgeSpec`
            # object from the metadata string.
            dep_name, dep_spec = dep_item
            dep_spec = ForgeSpec(dep_spec)

            # When the specification is not in our datastructure yet,
            # get the corresponding Module, and all Releases that conform
            # to the specification and add them to the queue.
            if not dep_spec in dependencies[dep_name]:
                try:
                    dep_mod = Module.objects.get_for_full_name(dep_name)
                except Module.DoesNotExist:
                    raise Exception('Dependency module %s not found' %
                                    dep_name)

                dependencies[dep_name].add(dep_spec)
                for dep_rel in dep_mod.releases.all():
                    if dep_rel.version in dep_spec:
                        queue.add(dep_rel)

    return dependencies, spec_cache


def release_dependencies(release):
    """
    This determines the dependencies for the given module release.
    """
    dependency_specs, spec_cache = calculate_dependencies(release)
    dependencies = defaultdict(list)

    for dep_name, specs in dependency_specs.iteritems():
        try:
            dep_mod = Module.objects.get_for_full_name(dep_name)
        except Module.DoesNotExist:
            raise Exception('Dependency module %s not found' % dep_name)

        releases = []
        for rel in dep_mod.releases.all():
            if all([rel.version in spec for spec in specs]):
                dependencies[dep_name].append({
                        'version': str(rel.version),
                        'file': rel.tarball.url,
                        'dependencies': spec_cache[rel.pk],
                })

    return dependencies
{'fraction_non_alphanumeric': 0.08063984168865436, 'fraction_numerical': 0.0018139841688654353, 'mean_word_length': 2.25201072386059, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7418452', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1522, 'n_words': 510}