<commit_before>#!/usr/bin/env python

"""Utilities for working with Pyon independent scripts"""

__author__ = '<NAME>, <NAME>'


def parse_args(tokens):
    """
    Parses extra args fron Python's argparse.
    Exploit yaml's spectacular type inference (and ensure consistency with config files)
    """
    args, kwargs = [], {}
    for token in tokens:
        token = token.lstrip('-')
        if '=' in token:
            key,val = token.split('=', 1)
            ipython_cfg = unflatten({key: yaml.load(val)})
            kwargs.update(ipython_cfg)
        else:
            args.append(yaml.load(token))

    return args, kwargs

def unflatten(dictionary):
    # From http://stackoverflow.com/questions/6037503/python-unflatten-dict/6037657#6037657
    resultDict = dict()
    for key, value in dictionary.iteritems():
        parts = key.split(".")
        d = resultDict
        for part in parts[:-1]:
            if part not in d:
                d[part] = dict()
            d = d[part]
        d[parts[-1]] = value
    return resultDict<commit_msg>Fixed script_util yaml import<commit_after>#!/usr/bin/env python

"""Utilities for working with Pyon independent scripts"""

__author__ = '<NAME>, <NAME>'

import yaml


def parse_args(tokens):
    """
    Parses extra args fron Python's argparse.
    Exploit yaml's spectacular type inference (and ensure consistency with config files)
    """
    args, kwargs = [], {}
    for token in tokens:
        token = token.lstrip('-')
        if '=' in token:
            key,val = token.split('=', 1)
            ipython_cfg = unflatten({key: yaml.load(val)})
            kwargs.update(ipython_cfg)
        else:
            args.append(yaml.load(token))

    return args, kwargs

def unflatten(dictionary):
    # From http://stackoverflow.com/questions/6037503/python-unflatten-dict/6037657#6037657
    resultDict = dict()
    for key, value in dictionary.iteritems():
        parts = key.split(".")
        d = resultDict
        for part in parts[:-1]:
            if part not in d:
                d[part] = dict()
            d = d[part]
        d[parts[-1]] = value
    return resultDict
{'fraction_non_alphanumeric': 0.12563983248022337, 'fraction_numerical': 0.022335970218706376, 'mean_word_length': 2.312788906009245, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4403080', 'n_tokens_mistral': 688, 'n_tokens_neox': 639, 'n_words': 176}