<jupyter_start><jupyter_text>Files<jupyter_code># The Search API can give us information about the files in processed datasets:

search_api = "$search_url"
# The Search API supports Elasticsearch queries:

import json

hits = json.loads(
    requests.post(
        search_api,
        json={
            "size": 10000,  # To make sure the list is not truncted, set this high.
            "query": {"ids": {"values": uuids}},
            "_source": ["files"],
        },  # Documents are large, so only request the fields we need.
    ).text
)["hits"]["hits"]
# Only processed datasets have file information.

files = {
    hit["_id"]: {
        file["rel_path"]: file["description"]
        for file in hit["_source"].get("files", [])
        if file
    }
    for hit in hits
}

pd.DataFrame(files).head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13157894736842105, 'fraction_numerical': 0.005980861244019139, 'mean_word_length': 2.911214953271028, 'pattern_counts': {'":': 5, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '303756', 'license': 'MIT', 'n_tokens_mistral': 263, 'n_tokens_neox': 260, 'n_words': 81, 'path': 'context/app/notebook/files.ipynb', 'repo_name': 'mccalluc/flask-data-portal', 'size': '836'}