<jupyter_start><jupyter_text>To Address AnalysisFind out top 2000 `to` addresses.<jupyter_code>import gzip
import json
from json.decoder import JSONDecodeError
import lzma
import pandas as pd
from typing import Any, Dict, List, Set, Tuple
pd.io.json._json.loads = lambda s, *a, **kw: json.loads(s)<jupyter_output><empty_output><jupyter_text>Filter out useless dataTransactions sent to tokens, pairs, routers and verified contracts are useless, we don't care about them.<jupyter_code>def read_whitelist() -> Set[str]:
    result = set()
    with lzma.open('./data/bep20_tokens.json.xz', "rt") as f_in:
        txt = f_in.read()
        arr = json.loads(txt)
        result.update(arr)
    with lzma.open('./data/contracts_verified.json.xz', "rt") as f_in:
        txt = f_in.read()
        arr = json.loads(txt)
        result.update(arr)

    result.add('0x55d398326f99059ff775485246999027b3197955') # BNB
    result.add('0xe9e7cea3dedca5984780bafc599bd69add087d56') # BUSD
    result.add('0x10ed43c718714eb63d5aa57b78b54704e256024e') # pancakeswap
    result.add('0x18b2a687610328590bc8f2e5fedde3b582a49cda') # PancakeSwap: Prediction V2
    result.add('0x1b6c9c20693afde803b27f8782156c0f892abc2d') # FstswapRouter02
    result.add('0x093c639e4291fbdcc339557accdaff52aaac37d9') # Fstswap Farm, see https://mobile.twitter.com/FstSwapbright/status/1519946172911079424
    result.add('0x260e69ab6665b9ef67b60674e265b5d21c88cb45') # SecondLiveFactoryProxy
    result.add('0x3a6d8ca21d1cf76f653a67577fa0d27453350dd8') # BiswapRouter02
    result.add('0xffe811714ab35360b67ee195ace7c10d93f89d8c') # DRIP Faucet Proxy Contract, see https://cryptozoa.com/drip-how-to-have-fun-with-the-faucet-contract-89040160a3b6
    result.add('0xe2d26507981a4daaaa8040bae1846c14e0fb56bf') # Baked Beans
    return result
def filtered_ratio(file: str) -> Tuple[int, int]:
    whitelist = set(read_whitelist())
    
    total_count = 0
    remaining = 0

    # to -> count
    counter: Dict[str, int] = {}
    f_in = None
    if file.endswith('.json'):
        f_in = open(file, "rt")
    elif file.endswith('.json.gz'):
        f_in = gzip.open(file, "rt")
    elif file.endswith('.json.xz'):
        f_in = lzma.open(file, "rt")
    else:
        raise ValueError(file)
    with f_in:
        for line in f_in:
            try:
                obj = json.loads(line)
                to = obj["txContents"]['to']
                total_count += 1
                if to not in whitelist:
                    remaining += 1
            except JSONDecodeError as ex:
                pass # ignore malformed lines
    
    return (total_count, remaining)
%%time
filtered_ratio("./data/bloxroute-tx-gateway-virginia.json.xz")<jupyter_output>CPU times: user 36.2 s, sys: 278 ms, total: 36.5 s
Wall time: 36.5 s<jupyter_text>There are 2010561 lines, and 1372622 lines are not in the white list.  Histogram of `to`<jupyter_code>def read_to_count(file: str) -> Dict[str, int]:
    whitelist = set(read_whitelist())

    # to -> count
    counter: Dict[str, int] = {}
    f_in = None
    if file.endswith('.json'):
        f_in = open(file, "rt")
    elif file.endswith('.json.gz'):
        f_in = gzip.open(file, "rt")
    elif file.endswith('.json.xz'):
        f_in = lzma.open(file, "rt")
    else:
        raise ValueError(file)
    with f_in:
        for line in f_in:
            try:
                obj = json.loads(line)
                to = obj["txContents"]['to']
                if to not in whitelist:
                    if to in counter:
                        counter[to] += 1
                    else:
                        counter[to] = 1
            except JSONDecodeError as ex:
                pass # ignore malformed lines
    
    return counter
%%time
to_counter = read_to_count("./data/bloxroute-tx-gateway-virginia.json.xz")
to_count_df = pd.DataFrame(to_counter.items(), columns=['to', 'count']).sort_values(by='count', ascending=False, ignore_index=True)
to_count_df
to_count_df['count'].plot()
to_count_df['count'].cumsum().plot()
to_count_df['count'].describe([0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
filtered_df = to_count_df[to_count_df['count']>8]
filtered_df
filtered_df['count'].plot()
filtered_df['count'].cumsum().plot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12353217472992015, 'fraction_numerical': 0.08313762329732269, 'mean_word_length': 3.0141376060320453, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '142665', 'license': 'Apache-2.0', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1557, 'n_words': 298, 'path': 'notebooks/to-address.ipynb', 'repo_name': 'crypto-crawler/fullnode-benchmarks', 'size': '4258'}