<jupyter_start><jupyter_text>Symbols<jupyter_code>if False:
    dir_name = ".."
    files = glob.glob(os.path.join(dir_name, "symbols*.csv"))
    dbg.dassert(len(files), 1)
    file_name = files[0]
    
file_name = "s3://external-p1/ib/metadata/symbols-2021-04-01-143112738505.csv"
print("file_name=%s" % file_name)
symbols = pd.read_csv(file_name, sep="\t")

print(len(symbols))

symbols.head(3)
symbols.groupby("product").count()<jupyter_output><empty_output><jupyter_text>Exchanges<jupyter_code>if False:
    files = glob.glob(os.path.join(dir_name, "exchanges*.csv"))
    dbg.dassert(len(files), 1)
    file_name = files[0]
file_name = "s3://external-p1/ib/metadata/exchanges-2021-04-01-143112738505.csv"

print("file_name=%s" % file_name)
exchanges = pd.read_csv(file_name, sep="\t")

print(len(exchanges))

exchanges.head(3)<jupyter_output>file_name=../exchanges-2021-03-23-141008520313.csv
137<jupyter_text>Products<jupyter_code>print(symbols["product"].unique())<jupyter_output>['ETFs' 'Indices' 'Options' 'Futures' 'Stocks' 'Bills']<jupyter_text>Markets<jupyter_code>markets = sorted(symbols["market"].unique())
print("\n".join(markets))
grouped = symbols.groupby("market")

count = grouped[["product"]].count()
count = count.sort_values(by="product", ascending=False)

count.plot()

print(count.head(10))
idx = 0
market = count.index[idx]
print("market=", market)
mask = symbols["market"] == market
symbols_tmp = symbols[mask]

grouped = symbols_tmp.groupby("product")

grouped[["product"]].count()
pd.set_option("display.max_colwidth", 100)
symbols_tmp[["product", "url"]]
symbols_tmp["url"].values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18608907870652838, 'fraction_numerical': 0.04820012202562538, 'mean_word_length': 7.078817733990148, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '639485', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 700, 'n_tokens_neox': 646, 'n_words': 73, 'path': 'im/ib/metadata/extract/notebooks/Master_analyze_ib_metadata_crawler.ipynb', 'repo_name': 'ajmal017/amp', 'size': '1639'}