<commit_before>import itertools, os, requests, yaml
from requests.models import RequestEncodingMixin
from flask import abort, url_for
from config import API, basedir, CROSSWALKS, PROFILES
from datausa import cache, app
from datausa.utils.format import dictionary, num_format


def datafold(data):
    """List[dict]: combines the headers and data from an API call """
    return [dict(zip(data["headers"], d)) for d in data["data"]]

def datasplit(data):
    data = [v for (k,v) in data.items()]
    headers = [k for (k,v) in data[0].items()]
    data = [[v for (k, v) in d.items()] for d in data]
    return {"data": data, "headers": headers}

def fetch(attr_id, attr_type):
    """dict: Returns an attribute dict container information like 'name' and 'color' """
    if isinstance(attr_type, basestring) and "_iocode" in attr_type:
        attr_type = "iocode"
    if attr_type in attr_cache and attr_id in attr_cache[attr_type]:
        return attr_cache[attr_type][attr_id]
    else:
        name = dictionary[attr_id] if attr_id in dictionary else "N/A"
        return {
            "id": attr_id,
            "name": name
        }

def default_params(params):

    params["sumlevel"] = params.get("sumlevel", "all")
    params["year"] = params.get("year", "latest")
    params["sort"] = params.get("sort", "desc")
    params["order"] = params.get("order", "")
    params["exclude"] = params.get("exclude", "")
    if "force" not in params:
        params["required"] = params.get("required", params["order"])

    if "show" in params and params["show"] == "skill" or params["year"] == "none":
        del params["year"]<commit_msg>removed optional args from viz data calls<commit_after>import itertools, os, requests, yaml
from requests.models import RequestEncodingMixin
from flask import abort, url_for
from config import API, basedir, CROSSWALKS, PROFILES
from datausa import cache, app
from datausa.utils.format import dictionary, num_format


def datafold(data):
    """List[dict]: combines the headers and data from an API call """
    return [dict(zip(data["headers"], d)) for d in data["data"]]

def datasplit(data):
    data = [v for (k,v) in data.items()]
    headers = [k for (k,v) in data[0].items()]
    data = [[v for (k, v) in d.items()] for d in data]
    return {"data": data, "headers": headers}

def fetch(attr_id, attr_type):
    """dict: Returns an attribute dict container information like 'name' and 'color' """
    if isinstance(attr_type, basestring) and "_iocode" in attr_type:
        attr_type = "iocode"
    if attr_type in attr_cache and attr_id in attr_cache[attr_type]:
        return attr_cache[attr_type][attr_id]
    else:
        name = dictionary[attr_id] if attr_id in dictionary else "N/A"
        return {
            "id": attr_id,
            "name": name
        }

def default_params(params):

    params["sumlevel"] = params.get("sumlevel", "all")
    params["year"] = params.get("year", "latest")
    params["sort"] = params.get("sort", "desc")
    params["order"] = params.get("order", "")
    params["exclude"] = params.get("exclude", "")
    if "force" not in params:
        params["required"] = params.get("required", params["order"])

    if "show" in params and params["show"] == "skill" or params["year"] == "none":
        del params["year"]

    for optional in ["order", "exclude"]:
        if params[optional] == "":
            del params[optional]
{'fraction_non_alphanumeric': 0.14620568414884266, 'fraction_numerical': 0.0005859947260474656, 'mean_word_length': 3.6197564276048713, 'pattern_counts': {'":': 11, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1164101', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1056, 'n_words': 338}