<jupyter_start><jupyter_text>Initialization<jupyter_code>%load_ext autoreload
%autoreload 2

import os
import json
import numpy as np
import pprint
from typing import Any

import pandas as pd

if False:
    import sys
    sys.path.append("/commodity_research/p1_data_client_python_private")
    print(sys.path)
    os.environ["P1_API_TOKEN"]='e44e7c6b04ef3ea1cfb7a8a67db74751c177259e'
    os.environ["P1_EDGAR_API_TOKEN"]='8c9c9458b145202c7a6b6cceaabd82023e957a46d6cf7061ed8e1c94a168f2fd'

import p1_data_client_python as p1cli

# Enter your token here.
# You can get your token by signing up at `www.particle.one`.
# P1_API_TOKEN = "YOUR_TOKEN_HERE"
# An example token is like:

P1_API_TOKEN = os.environ["P1_EDGAR_API_TOKEN"]
print("P1_API_TOKEN=", P1_API_TOKEN)
# Initalize the client.
client = p1cli.edgar_client.EdgarClient(token=P1_API_TOKEN)
from_dttm = '2019-11-01'
to_dttm = '2019-11-30'<jupyter_output><empty_output><jupyter_text>Init CIK list from sp1500 universe<jupyter_code>import p1_data_client_python.edgar.utils as peutil

sp1500_cik_list = peutil.get_sp1500_cik_list()<jupyter_output><empty_output><jupyter_text>Big list of CIK demo<jupyter_code>payload = client.get_form_headers(
    form_type=['4'],
    cik=sp1500_cik_list,
    start_date=from_dttm,
    end_date=to_dttm
)
len(payload)<jupyter_output><empty_output><jupyter_text>Headers: all CIK vs no CIK queries<jupyter_code>headers_p1 = client.get_form_headers(
    form_type=['4'],
    start_date=from_dttm,
    end_date=to_dttm
)
headers_p2 = client.get_form_headers(
    form_type=['4'],
    cik=sp1500_cik_list,
    start_date=from_dttm,
    end_date=to_dttm
)
len(headers_p1)
len(headers_p2)<jupyter_output><empty_output><jupyter_text>Headers: 400 CIK vs filtered 400 CIK<jupyter_code>headers_p2 = client.get_form_headers(
    form_type=['4'],
    cik=sp1500_cik_list[:400],
    start_date=from_dttm,
    end_date=to_dttm
)
len(headers_p1[headers_p1['cik'].apply(lambda x: x in np.array(sp1500_cik_list[:400]).astype(int))])
len(headers_p2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10444119082479258, 'fraction_numerical': 0.06979014153245486, 'mean_word_length': 6.945736434108527, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '79444', 'license': 'MIT', 'n_tokens_mistral': 950, 'n_tokens_neox': 880, 'n_words': 123, 'path': 'notebooks/PartTask7039.ipynb', 'repo_name': 'ParticleDev/p1_data_client_python', 'size': '2049'}