<jupyter_start><jupyter_text>Definition of data input:The input to the tag extractor is an excel or csv file. The first column of the data must be the intended tag keyword. To remove ambiguity, a header with name "tag_keyword" must be present. Additionally, all non-empty values in the columns to the right of the tag keyword are considered as prototypes. Occurences of these prototypes will be mapped to the tag keyword.<jupyter_code>def get_keywords_mapping(tags_sheet):
    tags_mapping = tags_sheet.set_index("tag_keyword").T.apply(
        # If prototypes have "underscores" create a copy with the underscore replaced with a space.
        lambda x: [[i] if "_" not in i else [i, i.replace("_", " ")] for i in x.dropna().tolist()] +

        # Add the tag keyword as well
        [[x.name, x.name.replace("_", " ")]])

    # Clean up the keywords to remove duplicates.
    tags_mapping = tags_mapping.map(lambda x: sorted(set([j for i in x for j in i])))

    tags_mapping = tags_mapping.map(
        lambda x: x + [inflect_engine.plural(i) for i in x if "_" not in i])

    # Clean up the keywords to remove duplicates.
    tags_mapping = tags_mapping.map(
        lambda x: sorted(set(x)))

    return tags_mapping
tags_mapping = get_keywords_mapping(tags_sheet)
tags_mapping.pop("Kakuma (Kenya)")
tags_mapping
tags_mapping["unhcr"]
jdc_tags_processor.add_keywords_from_dict(tags_mapping)
with open(dir_manager.get_data_dir("corpus", "WB", "TXT_ORIG", "wb_731917.txt"), "rb") as open_file:
    txt = open_file.read().decode("utf-8", errors="ignore")
txt += " the refugee, asylum seeker, internally displaced population  and stateless people are suffering in exile with other refugee, asylum seeker, and stateless people and climate refugees under the high Commission for refugees."
jdc_tags_processor.extract_keywords(txt)
tags_mapping.pop("Kakuma (Kenya)")
tags_mapping<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09805977975878342, 'fraction_numerical': 0.0036706869428421605, 'mean_word_length': 4.530434782608696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '152836', 'license': 'MIT', 'n_tokens_mistral': 608, 'n_tokens_neox': 572, 'n_words': 207, 'path': 'notebooks/jdc/01-avs-jdc-tags-detection.ipynb', 'repo_name': 'worldbank/wb-nlp-apps', 'size': '1907'}