<commit_before>#!/usr/bin/env python

import os
from fnmatch import fnmatch
import shutil

import click
import yaml
import networkx as nx


def ensure_dir(dir):
    try:
        os.makedirs(dir)
    except OSError:
        pass

CURDIR = os.path.dirname(os.path.realpath(__file__))

LIBRARY_PATH = os.path.join(CURDIR, 'fuel-library')
RESOURCE_TMP_WORKDIR = os.path.join(CURDIR, 'tmp/resources')
ensure_dir(RESOURCE_TMP_WORKDIR)
RESOURCE_DIR = os.path.join(CURDIR, 'resources')
VR_TMP_DIR = os.path.join(CURDIR, 'tmp/vrs')
ensure_dir(VR_TMP_DIR)
INPUTS_LOCATION = "/tmp/fuel_specs/"
DEPLOYMENT_GROUP_PATH = os.path.join(LIBRARY_PATH,
    'deployment', 'puppet', 'deployment_groups', 'tasks.yaml')

def clean_resources():
    shutil.rmtree(RESOURCE_TMP_WORKDIR)
    ensure_dir(RESOURCE_TMP_WORKDIR)

def clean_vr():
    shutil.rmtree(VR_TMP_DIR)
    ensure_dir(VR_TMP_DIR)

class Task(object):

    def __init__(self, task_data, task_path):
        self.data = task_data
        self.src_path = task_path
        self.name = self.data['id']
        self.type = self.data['type']

    def edges(self):
        data = self.data
        if 'required_for' in data:
            for req in data['required_for']:
                yield self.name, req
        if 'requires' in data:
            for req in data['requires']:
                yield req, self.name

        if 'groups' in data:
            for req in data['groups']:
                yield self.name, req
        if 'tasks' in data:
            for req in data['tasks']:
                yield req, self.name

    @property
    def manifest(self):
        after_naily = self.data['parameters']['puppet_manifest'].split('osnailyfacter/')[-1]
        return os.path.join(
            LIBRARY_PATH, 'deployment', 'puppet', 'osnailyfacter',
            after_naily)

    @property
    def dst_path(self):
        return os.path.join(RESOURCE_TMP_WORKDIR, self.name)

    @property
    def actions_path(self):
        return os.path.join(self.dst_path, 'actions')

    @property
    def meta_path(self):
        return os.path.join(self.dst_path, 'meta.yaml')

    def meta(self):
        data = {'id': self.name,
                'handler': 'puppetv2',
                'version': '8.0',
                'inputs': self.inputs()}
        return yaml.safe_dump(data, default_flow_style=False)

    @property
    def actions(self):
        """yield an iterable of src/dst
        """
        yield self.manifest, os.path.join(self.actions_path, 'run.pp')

    def inputs(self):
        """
        Inputs prepared by

        fuel_noop_tests.rb
        identity = spec.split('/')[-1]
        ENV["SPEC"] = identity

        hiera.rb
        File.open("/tmp/fuel_specs/#{ENV['SPEC']}", 'a') { |f| f << "- #{key}\n" }
        """
        lookup_stack_path = os.path.join(
            INPUTS_LOCATION, self.name+"_spec.rb'")
        if not os.path.exists(lookup_stack_path):
            return {}

        with open(lookup_stack_path) as f:
            data = yaml.safe_load(f) or []
        return {key: {'value': None} for key
                in set(data) if '::' not in key}


class RoleData(Task):

    name = 'globals'

    def meta(self):
        data = {'id': self.name,
                'handler': 'puppetv2',
                'version': '8.0',
                'inputs': self.inputs(),
                'manager': 'globals.py'}
        return yaml.safe_dump(data, default_flow_style=False)

    @property
    def actions(self):
        pass


class DGroup(object):

    def __init__(self, name, tasks):
        self.name = name
        self.tasks = tasks

    def resources(self):
        for t, _, _ in self.tasks:
            yield {'id': t.name+"{{index}}",
                   'from': 'f2s/resources/'+t.name,
                   'location': "{{node}}"}


    def events(self):
        for t, inner, outer in self.tasks:
            for dep in set(inner):
                yield {
                    'type': 'depends_on',
                    'state': 'success',
                    'parent_action': dep + '{{index}}.run',
                    'child_action': t.name + '{{index}}.run'}<commit_msg>Add values_from to vr generator<commit_after>#!/usr/bin/env python

import os
from fnmatch import fnmatch
import shutil

import click
import yaml
import networkx as nx


def ensure_dir(dir):
    try:
        os.makedirs(dir)
    except OSError:
        pass

CURDIR = os.path.dirname(os.path.realpath(__file__))

LIBRARY_PATH = os.path.join(CURDIR, 'fuel-library')
RESOURCE_TMP_WORKDIR = os.path.join(CURDIR, 'tmp/resources')
ensure_dir(RESOURCE_TMP_WORKDIR)
RESOURCE_DIR = os.path.join(CURDIR, 'resources')
VR_TMP_DIR = os.path.join(CURDIR, 'tmp/vrs')
ensure_dir(VR_TMP_DIR)
INPUTS_LOCATION = "/tmp/fuel_specs/"
DEPLOYMENT_GROUP_PATH = os.path.join(LIBRARY_PATH,
    'deployment', 'puppet', 'deployment_groups', 'tasks.yaml')

def clean_resources():
    shutil.rmtree(RESOURCE_TMP_WORKDIR)
    ensure_dir(RESOURCE_TMP_WORKDIR)

def clean_vr():
    shutil.rmtree(VR_TMP_DIR)
    ensure_dir(VR_TMP_DIR)

class Task(object):

    def __init__(self, task_data, task_path):
        self.data = task_data
        self.src_path = task_path
        self.name = self.data['id']
        self.type = self.data['type']

    def edges(self):
        data = self.data
        if 'required_for' in data:
            for req in data['required_for']:
                yield self.name, req
        if 'requires' in data:
            for req in data['requires']:
                yield req, self.name

        if 'groups' in data:
            for req in data['groups']:
                yield self.name, req
        if 'tasks' in data:
            for req in data['tasks']:
                yield req, self.name

    @property
    def manifest(self):
        after_naily = self.data['parameters']['puppet_manifest'].split('osnailyfacter/')[-1]
        return os.path.join(
            LIBRARY_PATH, 'deployment', 'puppet', 'osnailyfacter',
            after_naily)

    @property
    def dst_path(self):
        return os.path.join(RESOURCE_TMP_WORKDIR, self.name)

    @property
    def actions_path(self):
        return os.path.join(self.dst_path, 'actions')

    @property
    def meta_path(self):
        return os.path.join(self.dst_path, 'meta.yaml')

    def meta(self):
        data = {'id': self.name,
                'handler': 'puppetv2',
                'version': '8.0',
                'inputs': self.inputs()}
        return yaml.safe_dump(data, default_flow_style=False)

    @property
    def actions(self):
        """yield an iterable of src/dst
        """
        yield self.manifest, os.path.join(self.actions_path, 'run.pp')

    def inputs(self):
        """
        Inputs prepared by

        fuel_noop_tests.rb
        identity = spec.split('/')[-1]
        ENV["SPEC"] = identity

        hiera.rb
        File.open("/tmp/fuel_specs/#{ENV['SPEC']}", 'a') { |f| f << "- #{key}\n" }
        """
        lookup_stack_path = os.path.join(
            INPUTS_LOCATION, self.name+"_spec.rb'")
        if not os.path.exists(lookup_stack_path):
            return {}

        with open(lookup_stack_path) as f:
            data = yaml.safe_load(f) or []
        return {key: {'value': None} for key
                in set(data) if '::' not in key}


class RoleData(Task):

    name = 'globals'

    def meta(self):
        data = {'id': self.name,
                'handler': 'puppetv2',
                'version': '8.0',
                'inputs': self.inputs(),
                'manager': 'globals.py'}
        return yaml.safe_dump(data, default_flow_style=False)

    @property
    def actions(self):
        pass


class DGroup(object):

    def __init__(self, name, tasks):
        self.name = name
        self.tasks = tasks

    def resources(self):
        for t, _, _ in self.tasks:
            yield {'id': t.name+"{{index}}",
                   'from': 'f2s/resources/'+t.name,
                   'location': "{{node}}",
                   'values_from': RoleData.name+"{{index}}"}


    def events(self):
        for t, inner, outer in self.tasks:
            for dep in set(inner):
                yield {
                    'type': 'depends_on',
                    'state': 'success',
                    'parent_action': dep + '{{index}}.run',
                    'child_action': t.name + '{{index}}.run'}
{'fraction_non_alphanumeric': 0.134194474345174, 'fraction_numerical': 0.002152852529601722, 'mean_word_length': 2.077659182922341, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1613551', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2667, 'n_words': 581}