<commit_before># import time
import json
from normality import stringify
from banal import ensure_list, is_mapping
from memorious.helpers.key import make_id

# https://www.fara.gov/search.html
# https://efile.fara.gov/ords/f?p=107:3:0::NO:::
STATES = ("Active", "Terminated")
REGISTRANTS_URL = "https://efile.fara.gov/api/v1/Registrants/json/%s"
DOCS_URL = "https://efile.fara.gov/api/v1/RegDocs/json/%s"
SHORTFORM_URL = "https://efile.fara.gov/api/v1/ShortFormRegistrants/json/%s/%s"
PRINCIPALS_URL = "https://efile.fara.gov/api/v1/ForeignPrincipals/json/%s/%s"


def _get_row_data(row):
    data = {}
    for key, value in row.items():
        key = stringify(key)
        if key is None:
            continue
        value = stringify(value)
        data[key] = value
    return data


def _get_rows(context, res):
    try:
        data = res.json
    except json.JSONDecodeError as exc:
        context.emit_warning("Request error: %s" % res.text, exc)
        return
    for key, rows in data.items():
        if not is_mapping(rows):
            context.log.info("Response [%s]: %s", res.url, rows)
            return
        for row in ensure_list(rows.get("ROW")):
            row = _get_row_data(row)
            row["DataTable"] = key<commit_msg>more fixes to fara<commit_after># import time
import json
from normality import stringify
from banal import ensure_list, is_mapping
from memorious.helpers.key import make_id

# https://www.fara.gov/search.html
# https://efile.fara.gov/ords/f?p=107:3:0::NO:::
STATES = ("Active", "Terminated")
REGISTRANTS_URL = "https://efile.fara.gov/api/v1/Registrants/json/%s"
DOCS_URL = "https://efile.fara.gov/api/v1/RegDocs/json/%s"
SHORTFORM_URL = "https://efile.fara.gov/api/v1/ShortFormRegistrants/json/%s/%s"
PRINCIPALS_URL = "https://efile.fara.gov/api/v1/ForeignPrincipals/json/%s/%s"


def _get_row_data(row):
    data = {}
    for key, value in row.items():
        key = stringify(key)
        if key is None:
            continue
        data[key] = stringify(value)
    return data


def _get_rows(context, res):
    try:
        data = res.json
    except json.JSONDecodeError:
        context.emit_warning("Request error: %s" % res.text)
        return
    for key, rows in data.items():
        if not is_mapping(rows):
            context.log.info("Response [%s]: %s", res.url, rows)
            return
        for row in ensure_list(rows.get("ROW")):
            row = _get_row_data(row)
            row["DataTable"] = stringify(key)
{'fraction_non_alphanumeric': 0.14274258219727345, 'fraction_numerical': 0.007217321571772253, 'mean_word_length': 2.862229102167183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3198560', 'n_tokens_mistral': 895, 'n_tokens_neox': 881, 'n_words': 177}