<jupyter_start><jupyter_text>Keywords --- Text nesting<jupyter_code>#
# Data loading
#
import pandas as pd

df = pd.read_json(
    "https://raw.githubusercontent.com/jdvelasq/techminer/master/data/tutorial/"
    + "keywords-text-nesting.json",
    orient="records",
    lines=True,
)
#
# Number of unique keywords
#
len(
    set(
        [w.strip() for x in df.keywords_cleaned if x is not None for w in x.split(";")]
    )
)
#
# Substrings in keywords.
# This step requires several minutes to run
#
from techminer import text_nesting

tn = text_nesting(df.keywords_cleaned, sep=';', max_distance=1, transformer=lambda x: x.lower())

#
# Creates a thesaurus with candidate substrings as a thesaurus
#
with open('thesaurus-text-nesting-raw.json', 'w') as f:
    f.write(tn.__repr__())

!head -n 30 thesaurus-text-nesting-raw.json<jupyter_output>{
  "adaboost algorithm": [
    "AdaBoost algorithm",
    "validating AdaBoost algorithm"
  ],
  "adaptive noise": [
    "Adaptive noise",
    "adaptive noise reducer"
  ],
  "algorithmic trading": [
    "Algorithmic trading",
    "Algorithmic trading models"
  ],
  "algorithms": [
    "Algorithmic approach",
    "Analysis algorithms",
    "Boruta algorithm",
    "Classification algorithm",
    "Clustering algorithms",
    "Immune algorithms",
    "Learning algorithms",
    "Learning-based algorithms",
    "Levenberg-Marquardt algorithm",
    "NARX algorithm",
    "State-of-the-art algorithms",
    "algorithms",
    "genetic algorithms",
    "hybrid algorithms"
  ],
  "arima": [<jupyter_text>**Note**. Previous file MUST be reviewed and edited by the analyst. The edited versio is called `thesaurus-text-nesting-edited.json.json`and we load our version from GitHub.<jupyter_code>#
# Reads the cleaned file from GitHub Repo
#
import requests

text = requests.get(url + "thesaurus-text-nesting-edited.json").text

#
# Loads the file
#
dictionary = json.loads(text)

#
# Create a new thesaurus
#
from techminer import Thesaurus

th = Thesaurus(dictionary, ignore_case=False, full_match=True, use_re=False)

#
# Apply the thesaurus to the keywords
#
df["keywords_cleaned"] = df.keywords_cleaned.map(
    lambda x: th.apply(x, sep=";")
)

#
# Remove extra blanks between keywords  
#
df["keywords_cleaned"] = df.keywords_cleaned.map(
    lambda x: ";".join(set([w.strip() for w in x.split(";")]))
)

#
# Replace empty strings by None
#
df["keywords_cleaned"] = df.keywords_cleaned.map(
    lambda x: x if x != "" else None
)
#
# Number of unique keywords
#
len(
    set(
        [w.strip() for x in df.keywords_cleaned if x is not None for w in x.split(";")]
    )
)
### df.to_json("keywords-record-deletion.json", orient="records", lines=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1337252020573108, 'fraction_numerical': 0.0011021307861866275, 'mean_word_length': 3.959927140255009, 'pattern_counts': {'":': 5, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '283681', 'license': 'MIT', 'n_tokens_mistral': 975, 'n_tokens_neox': 929, 'n_words': 241, 'path': 'sphinx/tutorial/08-keywords-text-nexting.ipynb', 'repo_name': 'jdvelasq/techminer-new', 'size': '2722'}