<commit_before>import yaml
import json
import _jsonnet
import re
import os
import logging
import os.path
import jinja2
from kpm.utils import convert_utf8
import kpm.template_filters as filters

logger = logging.getLogger(__name__)


with open(os.path.join(os.path.dirname(__file__), "jsonnet/manifest.jsonnet.j2")) as f:
    JSONNET_TEMPLATE = f.read()


def yaml_to_jsonnet(manifestyaml, tla_codes=None):
    print manifestyaml
    jinja_env = jinja2.Environment()
    jinja_env.filters.update(filters.jinja_filters())

    # 1. Resolve old manifest variables
    # Load 'old' manifest.yaml
    v = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(manifestyaml))))}
    # Get variable from the 'old' manfiest and update  them
    variables = v['manifest']['variables']
    if tla_codes is not None and 'params' in tla_codes:
        tla = json.loads(tla_codes['params']).get("variables", {})
        variables.update(tla)
    # Resolve the templated variables inside the 'old' manifest
    manifest_tpl = jinja_env.from_string(manifestyaml)

    # 2. Convert 'old' manifest.yaml to manifest.jsonnet
    rendered_manifestyaml = manifest_tpl.render(variables)
    v = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(rendered_manifestyaml))))}
    # Load the yaml -> jsonnet template
    template = jinja_env.from_string(JSONNET_TEMPLATE)
    templatedjsonnet = template.render(v)
    # @TODO keep yaml format and escape 'jsonnet' commands:  key: "<% $.variables.key %>"
    # jsonnet_str = re.sub(r'[\'"]<%(.*)%>["\']', r"\1", templatedjsonnet)<commit_msg>Remove print<commit_after>import yaml
import json
import _jsonnet
import re
import os
import logging
import os.path
import jinja2
from kpm.utils import convert_utf8
import kpm.template_filters as filters

logger = logging.getLogger(__name__)


with open(os.path.join(os.path.dirname(__file__), "jsonnet/manifest.jsonnet.j2")) as f:
    JSONNET_TEMPLATE = f.read()


def yaml_to_jsonnet(manifestyaml, tla_codes=None):
    jinja_env = jinja2.Environment()
    jinja_env.filters.update(filters.jinja_filters())

    # 1. Resolve old manifest variables
    # Load 'old' manifest.yaml
    v = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(manifestyaml))))}
    # Get variable from the 'old' manfiest and update  them
    variables = v['manifest']['variables']
    if tla_codes is not None and 'params' in tla_codes:
        tla = json.loads(tla_codes['params']).get("variables", {})
        variables.update(tla)
    # Resolve the templated variables inside the 'old' manifest
    manifest_tpl = jinja_env.from_string(manifestyaml)

    # 2. Convert 'old' manifest.yaml to manifest.jsonnet
    rendered_manifestyaml = manifest_tpl.render(variables)
    v = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(rendered_manifestyaml))))}
    # Load the yaml -> jsonnet template
    template = jinja_env.from_string(JSONNET_TEMPLATE)
    templatedjsonnet = template.render(v)
    # @TODO keep yaml format and escape 'jsonnet' commands:  key: "<% $.variables.key %>"
    # jsonnet_str = re.sub(r'[\'"]<%(.*)%>["\']', r"\1", templatedjsonnet)
{'fraction_non_alphanumeric': 0.12419974391805377, 'fraction_numerical': 0.005761843790012804, 'mean_word_length': 4.444250871080139, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '880853', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1027, 'n_words': 236}