<commit_before>#!/usr/bin/env python

#
# Build shell commands from package specifications in a yml database.
#

import getopt
import yaml
import sys
import shlex


def usage():
    print("""
{0} [--pkg=<pkg>] [--pkgs=<pkg1,pkg2>,...] [--script] \
[--docker] [--vagrant] [--split=...] [--distrib=...] \
    /path/to/<example>.yml""".format(sys.argv[0]))


class OutputMode:
    Script, Docker, Vagrant = range(3)

output_mode_spec = dict()
output_mode_spec['script'] = OutputMode.Script
output_mode_spec['docker'] = OutputMode.Docker
output_mode_spec['vagrant'] = OutputMode.Vagrant

def wildcard(spec):
    if 'wildcard' in spec:
        return spec['wildcard']
    else:
        return 'any'


def is_list(a):
    return isinstance(a, list)


def is_dict(a):
    return isinstance(a, dict)


def is_atom(a):
    return not (hasattr(a, '__iter__'))


def get_entry(spec=None, distrib=None, distrib_version=None, pkg=None,
              section=None):
    """
    Get one entry with precedence distrib with version > distrib >
    match distrib > wildcard.
    """

    distrib_full = '{0}-{1}'.format(distrib, distrib_version)

    if pkg in spec[section]:

        if distrib_full in spec[section][pkg]:
            return spec[section][pkg][distrib_full]

        elif distrib in spec[section][pkg]:
            return spec[section][pkg][distrib]

        elif distrib in spec['match']:
            match_distrib = spec['match'][distrib]
            if match_distrib in spec[section][pkg]:
                return spec[section][pkg][match_distrib]

        if wildcard(spec) in spec[section][pkg]:
            return spec[section][pkg][wildcard(spec)]

        else:
            return None

def pkg_entries(spec=None, distrib=None, distrib_version=None, pkg=None):
    """
    Find recursively entries for pkg and distribution distrib in a
    specification spec.
    """

    result = None

    if pkg in spec['pkgs']:
        result = get_entry(spec, distrib, distrib_version, pkg, 'pkgs')

    if result is None:
        return [result]

    elif is_dict(result):
        return [result]
    else:

        if is_atom(result):
            result = [result]
        r = list()

        for e in result:

            if e != pkg:
                ne = pkg_entries(spec=spec, distrib=distrib,
                                 distrib_version=distrib_version, pkg=e)

                if ne == [None]:
                    r.append(e)

                else:
                    r += ne

        return r


def begin(distrib=None, distrib_version=None, output_mode=None):
    """
    Distribution preamble.
    """
    if output_mode == OutputMode.Docker:
        sys.stdout.write('FROM {0}:{1}\n'.format(distrib, distrib_version))


def env(definitions=None, output_mode=None):
    """
    Environment specification.
    """

    if len(definitions) > 0:

        items = list()

        if output_mode == OutputMode.Docker:
            items.append('ENV')

        items += definitions

        sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))


def install(installer=None, command=None, pkg=None, pkgs=None,
            output_mode=OutputMode.Script):
    """
    Format an install command according to output mode.
    """
<commit_msg>[CI] mksenv.py: add distribution + version in a comment for script output<commit_after>#!/usr/bin/env python

#
# Build shell commands from package specifications in a yml database.
#

import getopt
import yaml
import sys
import shlex


def usage():
    print("""
{0} [--pkg=<pkg>] [--pkgs=<pkg1,pkg2>,...] [--script] \
[--docker] [--vagrant] [--split=...] [--distrib=...] \
    /path/to/<example>.yml""".format(sys.argv[0]))


class OutputMode:
    Script, Docker, Vagrant = range(3)

output_mode_spec = dict()
output_mode_spec['script'] = OutputMode.Script
output_mode_spec['docker'] = OutputMode.Docker
output_mode_spec['vagrant'] = OutputMode.Vagrant


def wildcard(spec):
    if 'wildcard' in spec:
        return spec['wildcard']
    else:
        return 'any'


def is_list(a):
    return isinstance(a, list)


def is_dict(a):
    return isinstance(a, dict)


def is_atom(a):
    return not (hasattr(a, '__iter__'))


def get_entry(spec=None, distrib=None, distrib_version=None, pkg=None,
              section=None):
    """
    Get one entry with precedence distrib with version > distrib >
    match distrib > wildcard.
    """

    distrib_full = '{0}-{1}'.format(distrib, distrib_version)

    if pkg in spec[section]:

        if distrib_full in spec[section][pkg]:
            return spec[section][pkg][distrib_full]

        elif distrib in spec[section][pkg]:
            return spec[section][pkg][distrib]

        elif distrib in spec['match']:
            match_distrib = spec['match'][distrib]
            if match_distrib in spec[section][pkg]:
                return spec[section][pkg][match_distrib]

        if wildcard(spec) in spec[section][pkg]:
            return spec[section][pkg][wildcard(spec)]

        else:
            return None


def pkg_entries(spec=None, distrib=None, distrib_version=None, pkg=None):
    """
    Find recursively entries for pkg and distribution distrib in a
    specification spec.
    """

    result = None

    if pkg in spec['pkgs']:
        result = get_entry(spec, distrib, distrib_version, pkg, 'pkgs')

    if result is None:
        return [result]

    elif is_dict(result):
        return [result]
    else:

        if is_atom(result):
            result = [result]
        r = list()

        for e in result:

            if e != pkg:
                ne = pkg_entries(spec=spec, distrib=distrib,
                                 distrib_version=distrib_version, pkg=e)

                if ne == [None]:
                    r.append(e)

                else:
                    r += ne

        return r


def begin(distrib=None, distrib_version=None, output_mode=None):
    """
    Distribution preamble.
    """
    if output_mode == OutputMode.Docker:
        sys.stdout.write('FROM {0}:{1}\n'.format(distrib, distrib_version))
    elif output_mode == OutputMode.Script:
        sys.stdout.write('# {0} {1}\n'.format(distrib,
                                              distrib_version))


def env(definitions=None, output_mode=None):
    """
    Environment specification.
    """

    if len(definitions) > 0:

        items = list()

        if output_mode == OutputMode.Docker:
            items.append('ENV')

        items += definitions

        sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))


def install(installer=None, command=None, pkg=None, pkgs=None,
            output_mode=OutputMode.Script):
    """
    Format an install command according to output mode.
    """

{'fraction_non_alphanumeric': 0.13092046470062557, 'fraction_numerical': 0.0035746201966041107, 'mean_word_length': 2.384576612903226, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '77863', 'n_tokens_mistral': 2174, 'n_tokens_neox': 2038, 'n_words': 503}