<commit_before>#!/usr/bin/env python
# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-

"""
Convert a schema from the old JSON-based format to the new
YAML-based one.
"""

from __future__ import absolute_import, division, unicode_literals, print_function

import collections
import json
import sys

import yaml

from pyasdf import treeutil

import six


def convert(in_path, out_path):
    with open(in_path, "rb") as fd:
        tree = json.load(fd, object_pairs_hook=collections.OrderedDict)

    def yaml_representer(dumper, obj):
        if isinstance(obj, collections.OrderedDict):
            # PyYAML explicitly sorts by keys on mappings, so we have
            # to work really hard not have them sorted
            value = []
            node = yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value, flow_style=False)
            for key, val in obj.items():
                node_key = dumper.represent_data(key)
                node_val = dumper.represent_data(val)
                value.append((node_key, node_val))

            return node
        else:
            raise TypeError("Can't serialize {0}".format(obj))

    yaml.SafeDumper.add_representer(None, yaml_representer)

    def convert_references(node):
        if isinstance(node, six.string_types):
            if node.endswith('schema.json'):
                return node[:-4] + 'yaml'
        return node
    tree = treeutil.walk_and_modify(tree, convert_references)

    def remove_type_of_data(node):
        if isinstance(node, dict):
            if node.get('type') == 'data':
                del node['type']
        return node
    tree = treeutil.walk_and_modify(tree, remove_type_of_data)

    def convert_datatypes(node):
        if isinstance(node, dict) and 'dtype' in node:<commit_msg>Update for new walk_and_modify API<commit_after>#!/usr/bin/env python
# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-

"""
Convert a schema from the old JSON-based format to the new
YAML-based one.
"""

from __future__ import absolute_import, division, unicode_literals, print_function

import collections
import json
import sys

import yaml

from pyasdf import treeutil

import six


def convert(in_path, out_path):
    with open(in_path, "rb") as fd:
        tree = json.load(fd, object_pairs_hook=collections.OrderedDict)

    def yaml_representer(dumper, obj):
        if isinstance(obj, collections.OrderedDict):
            # PyYAML explicitly sorts by keys on mappings, so we have
            # to work really hard not have them sorted
            value = []
            node = yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value, flow_style=False)
            for key, val in obj.items():
                node_key = dumper.represent_data(key)
                node_val = dumper.represent_data(val)
                value.append((node_key, node_val))

            return node
        else:
            raise TypeError("Can't serialize {0}".format(obj))

    yaml.SafeDumper.add_representer(None, yaml_representer)

    def convert_references(node, json_id):
        if isinstance(node, six.string_types):
            if node.endswith('schema.json'):
                return node[:-4] + 'yaml'
        return node
    tree = treeutil.walk_and_modify(tree, convert_references)

    def remove_type_of_data(node, json_id):
        if isinstance(node, dict):
            if node.get('type') == 'data':
                del node['type']
        return node
    tree = treeutil.walk_and_modify(tree, remove_type_of_data)

    def convert_datatypes(node, json_id):
        if isinstance(node, dict) and 'dtype' in node:
{'fraction_non_alphanumeric': 0.09678298800436205, 'fraction_numerical': 0.004362050163576881, 'mean_word_length': 2.669, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '718555', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1079, 'n_words': 298}