<jupyter_start><jupyter_text>Substrings<jupyter_code>total_size = os.path.getsize(os.path.join(dir, "text.csv"))
total_size
substr = pd.read_csv(os.path.join(dir, "substring_bytes.tsv"), sep='\t')
substr.head()
duplicated = sum(substr.y - substr.x)

duplicated, duplicated / total_size
matches = pd.read_csv(os.path.join(dir, "matches.tsv"), sep='\t')
clusters = pd.read_csv(os.path.join(dir, "clusters.tsv"), sep='\t')
matches.head()
clusters.head()
ids = pd.read_csv(os.path.join(dir, "ids.csv"), names=["id"])
text = pd.read_csv(os.path.join(dir, "text.csv"), names=["text"])

print(len(ids), len(text))
text.head()
clusters = clusters[clusters.cluster != -1]<jupyter_output><empty_output><jupyter_text>Cluster size distribution<jupyter_code>sizes = clusters.groupby("cluster").size().value_counts().sort_index()
sizes = sizes.to_frame()
sizes.reset_index(inplace=True)
sizes = sizes.rename({"index": "size", 0: "count"}, axis=1)
sizes.plot.bar(x="size", y="count", figsize=(20, 8))<jupyter_output><empty_output><jupyter_text>Document length distribution<jupyter_code>ids = ids.assign(
    length = text.apply(lambda x: len(x["text"]), axis=1)
)
id2length = dict(ids[["id", "length"]].values.tolist())

matches["length"] = matches.apply(lambda x: max(
    id2length[x["id1"]],
    id2length[x["id2"]],
), axis=1)

import seaborn as sns
from matplotlib import pyplot as plt

fig, ax = plt.subplots(1, 1, figsize=(20, 8))

g = sns.histplot(data=matches, stat="count", multiple="stack",
             x="length", kde=False,
             palette="pastel", hue="diff",
             element="bars", legend=True)

g.set(xlim=(0, 200000))
clusters.groupby("cluster").apply(lambda x: np.max([id2length[r["id"]] for r in x.to_dict("records")])).plot.hist(bins=2000, figsize=(20, 8), xlim=(0, 200000))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20142700329308452, 'fraction_numerical': 0.021953896816684963, 'mean_word_length': 6.205533596837944, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '357625', 'license': 'Apache-2.0', 'n_tokens_mistral': 724, 'n_tokens_neox': 699, 'n_words': 112, 'path': 'ac_dc/deduplicate/visualize.ipynb', 'repo_name': 'Luvata/data_tooling', 'size': '1822'}