<commit_before>#!/usr/bin/env python

import yaml
import sys
from string import Template
import os
from os.path import isfile, join, splitext
import copy
import re
import chkcrontab_lib as check
import subprocess
import tempfile



def load(yaml_file):
    f = open(yaml_file)
    data = yaml.safe_load(f)
    f.close()
    return data


def validate_cron(file_path):
    whitelisted_users = None
    log = check.LogCounter()
    return check.check_crontab(file_path, log, whitelisted_users)


def convert_memlimit(value):
    if not isinstance(value, str):
        return value
    match = re.match(r"(\d+)[Mm][Bb]", value)
    if match:
        return int(match.group(1)) * pow(10, 6)

    return value


def convert_cpushares(value):
    return value


def substitute_all(values, string):
    if isinstance(string, str):
        template = Template(string)
        return template.safe_substitute(values)
    else:
        return string


def merge(source, destination):
    """
    Recursive map merge
    """
    for key, value in source.items():
        if isinstance(value, dict):
            # get node or create one
            node = destination.setdefault(key, {})
            merge(value, node)
        else:
            destination[key] = value

    return destination


def convert(file):
    command = ['kompose', '--suppress-warnings',
               '-f', file, 'convert', '--rc', '--stdout', '--yaml']
    try:
        res = subprocess.check_output(command, stderr=subprocess.STDOUT)
        return yaml.safe_load(res)
    except subprocess.CalledProcessError as err:
        print(file)
        with open(file, 'r') as fin:
            print(fin.read())
    except yaml.scanner.ScannerError as err:
        print(' '.join(command))
        print(res)
        raise err


def compose(file_name, yaml_doc):
    user = 'root'

    if not os.path.exists('.jobs'):
        os.makedirs('.jobs')

    if not os.path.exists('.jobs/cron'):
        os.makedirs('.jobs/cron')

    if not os.path.exists('.jobs/job'):
        os.makedirs('.jobs/job')

    config = copy.deepcopy(global_config)
    merge(yaml_doc.pop("Configuration", {}), config)

    cron_file = ".jobs/cron/" + file_name.lower()
    cron = file(cron_file, 'w')
    cron.write('SHELL=/bin/sh\n')
    cron.write('PATH=/usr/local/sbin:/usr/local/bin'
               ':/sbin:/bin:/usr/sbin:/usr/bin\n')

    for grouping, data in yaml_doc.iteritems():  # use safe_load instead load
        defaults = copy.copy(global_defaults)
        jobs = data.pop('Jobs')
        defaults.update(data)

        for job in jobs:
            jobName, jobData = job.popitem()
            jobName = jobName.lower()
            if os.path.exists('./jobs/job/' + jobName):
                print('A job of this name already exists {0}'.format(jobName))
                exit(2)

            dump = {'environment': {'job': jobName}}

            merge(defaults, dump)
            time = dump.pop('time', None)
            if isinstance(jobData, str):
                time = jobData
            else:
                merge(jobData, dump)
                time = dump.pop('time', time)

            for key, value in dump.iteritems():
                dump[key] = substitute_all(dump['environment'], value)

            if "mem_limit" in dump:
                dump["mem_limit"] = convert_memlimit(dump["mem_limit"])
                dump['environment']['mem_limit'] = str(dump['mem_limit'])

            stream = tempfile.NamedTemporaryFile()
            # Write a YAML representation of data to 'document.yaml'.
            yaml.dump({jobName: dump}, stream, default_flow_style=False)
            pods = convert(stream.name)
            stream.close()

            # Set image pull policy to always if image is latest
            for pod in pods['items']:
                pod['kind'] = 'Job'
                pod['apiVersion'] = 'extensions/v1beta1'
                pod.pop('status')
                pod['spec'].pop('replicas')
                if 'spec' in config:
                    merge(config['spec'], pod['spec']['template']['spec'])
                if 'annotations' in config:
                    pod['metadata']['annotations'] = {}
                    merge(config['annotations'],
                          pod['metadata']['annotations'])

                for container in pod['spec']['template']['spec']['containers']:
                    if container['image'].endswith(":latest"):
                        container['imagePullPolicy'] = 'Always'

            stream = file(".jobs/job/" + jobName + ".yaml", 'w')
            yaml.dump(pod, stream, default_flow_style=False)
            stream.close()

            if time:
                text = "{0} {1} /app/runner {2}" \<commit_msg>changing to v1 api<commit_after>#!/usr/bin/env python

import yaml
import sys
from string import Template
import os
from os.path import isfile, join, splitext
import copy
import re
import chkcrontab_lib as check
import subprocess
import tempfile



def load(yaml_file):
    f = open(yaml_file)
    data = yaml.safe_load(f)
    f.close()
    return data


def validate_cron(file_path):
    whitelisted_users = None
    log = check.LogCounter()
    return check.check_crontab(file_path, log, whitelisted_users)


def convert_memlimit(value):
    if not isinstance(value, str):
        return value
    match = re.match(r"(\d+)[Mm][Bb]", value)
    if match:
        return int(match.group(1)) * pow(10, 6)

    return value


def convert_cpushares(value):
    return value


def substitute_all(values, string):
    if isinstance(string, str):
        template = Template(string)
        return template.safe_substitute(values)
    else:
        return string


def merge(source, destination):
    """
    Recursive map merge
    """
    for key, value in source.items():
        if isinstance(value, dict):
            # get node or create one
            node = destination.setdefault(key, {})
            merge(value, node)
        else:
            destination[key] = value

    return destination


def convert(file):
    command = ['kompose', '--suppress-warnings',
               '-f', file, 'convert', '--rc', '--stdout', '--yaml']
    try:
        res = subprocess.check_output(command, stderr=subprocess.STDOUT)
        return yaml.safe_load(res)
    except subprocess.CalledProcessError as err:
        print(file)
        with open(file, 'r') as fin:
            print(fin.read())
    except yaml.scanner.ScannerError as err:
        print(' '.join(command))
        print(res)
        raise err


def compose(file_name, yaml_doc):
    user = 'root'

    if not os.path.exists('.jobs'):
        os.makedirs('.jobs')

    if not os.path.exists('.jobs/cron'):
        os.makedirs('.jobs/cron')

    if not os.path.exists('.jobs/job'):
        os.makedirs('.jobs/job')

    config = copy.deepcopy(global_config)
    merge(yaml_doc.pop("Configuration", {}), config)

    cron_file = ".jobs/cron/" + file_name.lower()
    cron = file(cron_file, 'w')
    cron.write('SHELL=/bin/sh\n')
    cron.write('PATH=/usr/local/sbin:/usr/local/bin'
               ':/sbin:/bin:/usr/sbin:/usr/bin\n')

    for grouping, data in yaml_doc.iteritems():  # use safe_load instead load
        defaults = copy.copy(global_defaults)
        jobs = data.pop('Jobs')
        defaults.update(data)

        for job in jobs:
            jobName, jobData = job.popitem()
            jobName = jobName.lower()
            if os.path.exists('./jobs/job/' + jobName):
                print('A job of this name already exists {0}'.format(jobName))
                exit(2)

            dump = {'environment': {'job': jobName}}

            merge(defaults, dump)
            time = dump.pop('time', None)
            if isinstance(jobData, str):
                time = jobData
            else:
                merge(jobData, dump)
                time = dump.pop('time', time)

            for key, value in dump.iteritems():
                dump[key] = substitute_all(dump['environment'], value)

            if "mem_limit" in dump:
                dump["mem_limit"] = convert_memlimit(dump["mem_limit"])
                dump['environment']['mem_limit'] = str(dump['mem_limit'])

            stream = tempfile.NamedTemporaryFile()
            # Write a YAML representation of data to 'document.yaml'.
            yaml.dump({jobName: dump}, stream, default_flow_style=False)
            pods = convert(stream.name)
            stream.close()

            # Set image pull policy to always if image is latest
            for pod in pods['items']:
                pod['kind'] = 'Job'
                pod['apiVersion'] = 'batch/v1'
                pod.pop('status')
                pod['spec'].pop('replicas')
                if 'spec' in config:
                    merge(config['spec'], pod['spec']['template']['spec'])
                if 'annotations' in config:
                    pod['metadata']['annotations'] = {}
                    merge(config['annotations'],
                          pod['metadata']['annotations'])

                for container in pod['spec']['template']['spec']['containers']:
                    if container['image'].endswith(":latest"):
                        container['imagePullPolicy'] = 'Always'

            stream = file(".jobs/job/" + jobName + ".yaml", 'w')
            yaml.dump(pod, stream, default_flow_style=False)
            stream.close()

            if time:
                text = "{0} {1} /app/runner {2}" \
{'fraction_non_alphanumeric': 0.1334318589675921, 'fraction_numerical': 0.0023223899503853054, 'mean_word_length': 2.050225370251127, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470980', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2691, 'n_words': 653}