<jupyter_start><jupyter_text>IntroductionIn this notebook, I directly try to do a hierarchical classification based on the pytorch-based library:https://github.com/Tencent/NeuralNLP-NeuralClassifierPaper:https://aclanthology.org/P19-3015/ Observation- We are able to generate the label taxonomy compatible for the framework- We are also able to generate the compatible data- On a prelimiary **Transformer-based** training, we are able to get a decent classifier within 25 epochs.- The macro-level metric might be lower because of the skewness of the class distribution.> See the final section where I also evaluate a jaccard-based metric<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import os
DRIVE_BASE = "/content/drive/MyDrive/Colab Notebooks/uah-ra/"
KW_PATH = os.path.join(DRIVE_BASE, "data/keywords.txt")
DATA_PATH = os.path.join(DRIVE_BASE, "data/data.csv")
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline
def load_keywords(path):
    res = []
    with open(path) as f:
        text = f.read().strip()
        tags_str = text.split(",")
        res = map(lambda t: [_.strip().lower() for _ in t.split(">")], tags_str)
        res = filter(lambda x: len(x) > 0, res)
        res = list(res)
    return res
KEYWORDS = load_keywords(KW_PATH)
len(KEYWORDS), len(set([kw for kws in KEYWORDS for kw in kws]))
KEYWORDS[:5]<jupyter_output><empty_output><jupyter_text>Tag Analysis<jupyter_code>!pip install loguru
from loguru import logger
from collections import Counter, defaultdict
def get_counts(keywords, level=0):
    kws = map(lambda x: x[level if level<len(x) else len(x)-1], keywords)
    kws = list(kws)
    # kws = list(map(str.lower, kws))
    counter = Counter(kws)
    return counter
def analyze_kws(keywords, topn=10):
    plt.figure(figsize=(15, 8))
    for level in [0, 1, 2, 3, -1]:
        _ = get_counts(KEYWORDS, level=level)
        logger.debug(f"[Level={level}, NKWs={len(_)}] : {_.most_common(10)}")
        df = pd.DataFrame(_.most_common(topn), columns=["kw", "frequency"])
        ax = sns.barplot(
            x="frequency", y="kw",
            data=df,
            linewidth=2.5,
            facecolor=(1, 1, 1, 0),
            errcolor=".2",
            edgecolor=".2"
        )
        plt.title(f"Level={level}, topn={topn}")
        plt.figure(figsize=(15, 8))
", ".join(list(get_counts(KEYWORDS, level=1).keys()))
analyze_kws(KEYWORDS, topn=20)<jupyter_output>2021-07-30 08:23:16.840 | DEBUG    | __main__:analyze_kws:5 - [Level=0, NKWs=10] : [('earth science', 1645), ('breakup date', 1), ('ice duration', 1), ('oceans', 1), ('surface', 1), ('and internal layer geometry', 1), ('earth observation satellites', 1), ('earth remote sensing instruments', 1), ('earth science services', 1), ('timing', 1)]
2021-07-30 08:23:16.890 | DEBUG    | __main__:analyze_kws:5 - [Level=1, NKWs=29] : [('atmosphere', 422), ('oceans', 202), ('land surface', 160), ('cryosphere', 148), ('biosphere', 118), ('terrestrial hydrosphere', 117), ('spectral/engineering', 95), ('human dimensions', 85), ('biological classification', 83), ('agriculture', 65)]
2021-07-30 08:23:16.971 | DEBUG    | __main__:analyze_kws:5 - [Level=2, NKWs=141] : [('snow/ice', 105), ('soils', 92), ('sea ice', 81), ('animals/vertebrates', 70), ('clouds', 62), ('atmospheric chemistry', 61), ('atmospheric radiation', 51), ('glaciers/ice sheets', 44), ('aerosols', 44), ('atmospheric winds', 42)]
2021-07-3[...]<jupyter_text>Data Analysis<jupyter_code>def parse_kws(kw_str, level=2):
    res = kw_str.split(",")
    res = map(lambda kw: [_.strip().lower() for _ in kw.split(">")], res)
    res = map(lambda x: x[level if level<len(x) else len(x)-1], res)
    return list(set(res))

def load_data(path, level=0):
    logger.info(f"Loading data from {path}. [KW Level={level}]")
    df = pd.read_csv(path)
    df["desc"] = df["desc"].apply(str.strip)
    df["labels"] = df["keywords"].apply(lambda x: parse_kws(x, level))
    df["textlen"] = df["desc"].apply(len)
    return df
DATA = load_data(DATA_PATH, level=1)
DATA.shape
DATA.head(10)

def analyze_labels(df):
    df = df.copy()
    labels = [l for ls in df["labels"] for l in ls]
    uniques = set(labels)
    logger.info(f"{len(uniques)} unique labels")
analyze_labels(DATA)
# idx = 2
# _data.iloc[2].keywords_processed
_data = DATA.copy()
_data = _data[_data["textlen"]>0]
_data.shape
# BERT can only process 512 tokens at once
len(_data[_data["textlen"] <= 512]) / len(_data), len(_data[_data["textlen"] <= 1024]) / len(_data)
plt.figure(figsize=(20, 15))
sns.histplot(data=_data, x="textlen", bins=100).set(xlim=(0, 3000))<jupyter_output><empty_output><jupyter_text>Prepare Dataset<jupyter_code>DATA_TO_USE = DATA.copy()
# DATA_TO_USE = DATA_TO_USE[DATA_TO_USE["textlen"]<=500]
DATA_TO_USE.shape
DATA_TO_USE.head()
analyze_labels(DATA_TO_USE)<jupyter_output>2021-07-30 08:24:02.691 | INFO     | __main__:analyze_labels:5 - 29 unique labels<jupyter_text>Generate DatasetCompatible to the library format  Split Dataset<jupyter_code>from sklearn.model_selection import train_test_split
#  X_train, X_test, Y_train, Y_test = train_test_split(DATA_TO_USE["desc"].to_numpy(), LABELS_ENCODED, test_size=0.1, random_state=42)
DATA_TRAIN, DATA_TEST = train_test_split(DATA_TO_USE, test_size=0.1, random_state=42)

#  X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=42)
DATA_TRAIN.head()<jupyter_output><empty_output><jupyter_text>Install Library<jupyter_code>!git clone https://github.com/Tencent/NeuralNLP-NeuralClassifier
%cd "/content/NeuralNLP-NeuralClassifier"
!ls
! pip install -r requirements.txt
! python train.py conf/train.hierar.json
! python predict.py conf/train.hierar.json data/rcv1_test.hierar.json<jupyter_output><empty_output><jupyter_text>Prepare Custom Dataset<jupyter_code>from typing import List<jupyter_output><empty_output><jupyter_text>Generate Taxonomy<jupyter_code>def get_levels(keywords: List[List[str]], level: int) -> List[str]:
    kws = map(lambda x: x[level if level<len(x) else len(x)-1], keywords)
    kws = list(set(kws))
    return kws
def process_single_kws(kws: List[str]) -> List[str]:
    kws = map(lambda x: "_".join(x.split()).upper(), kws)
    kws = list(kws)
    return kws
def generate_taxonomy(keywords: List[List[str]], outpath: str) -> None:
    keywords_processed = []
    for kws in keywords:
        kws = process_single_kws(kws)
        keywords_processed.append(kws)
    keywords = keywords_processed or keywords
    roots = get_levels(keywords, level=0)
    roots = ["Root"] + roots
    logger.debug(f"[Nroots = {len(roots)}] == {roots}")
    logger.info(f"Writing the custom taxonomy to {outpath}")
    with open(outpath, "w") as f:
        f.write(" ".join(roots))
        f.write("\n")
        for kws in keywords:
            f.write(" ".join(kws))
            f.write("\n")
generate_taxonomy(KEYWORDS, outpath="data/custom.taxonomy")<jupyter_output>2021-07-30 09:52:29.508 | DEBUG    | __main__:generate_taxonomy:9 - [Nroots = 11] == ['Root', 'EARTH_OBSERVATION_SATELLITES', 'TIMING', 'EARTH_SCIENCE', 'EARTH_SCIENCE_SERVICES', 'SURFACE', 'EARTH_REMOTE_SENSING_INSTRUMENTS', 'AND_INTERNAL_LAYER_GEOMETRY', 'OCEANS', 'ICE_DURATION', 'BREAKUP_DATE']
2021-07-30 09:52:29.512 | INFO     | __main__:generate_taxonomy:10 - Writing the custom taxonomy to data/custom.taxonomy<jupyter_text>Generate Data<jupyter_code>import json
def tokenize_custom(text: str) -> List[str]:
    return text.split()
def process_keywords_str(keywords_str: str, label_sep=",", hierar_sep=">") -> List[str]:
    keywords = keywords_str.upper().split(label_sep)
    keywords = map(str.strip, keywords)
    keywords = map(lambda x: [_.strip() for _ in x.split(hierar_sep)], keywords)
    keywords = list(keywords)
    return keywords
def generate_data(df: pd.DataFrame, outpath: str) -> None:
    assert "desc" in df
    assert "keywords" in df
    data = []
    for _df in df.itertuples():
        tokens = tokenize_custom(_df.desc)
        keywords_str = _df.keywords
        keywords = process_keywords_str(keywords_str)
        keywords = list(map(process_single_kws, keywords))
        keywords = list(map(lambda kws: "--".join(kws), keywords))
        data.append({
            "doc_label": keywords,
            "doc_token": tokens,
            "doc_keyword": [],
            "doc_topic": []
        })
    if outpath:
        logger.info(f"Writing data to {outpath}")
        with open(outpath, "w") as f:
            for dct in data:
                json.dump(dct, f)
                f.write("\n")
    return data
final_data = generate_data(DATA_TRAIN, "data/custom.hierar.json")
final_data = generate_data(DATA_TEST, "data/custom_test.hierar.json")
len(final_data)<jupyter_output><empty_output><jupyter_text>Generate Config<jupyter_code>! cp "/content/NeuralNLP-NeuralClassifier/conf/train.hierar.json" "/content/NeuralNLP-NeuralClassifier/conf/custom.train.hierar.json"
! rm -rf "/content/NeuralNLP-NeuralClassifier/checkpoint_dir_rcv1"<jupyter_output><empty_output><jupyter_text>Train> Note: macro-score seems low because of skewed class distribution.<jupyter_code>! python train.py conf/custom.train.hierar.json<jupyter_output>Use dataset to generate dict.
Size of doc_label dict is 2393
Size of doc_token dict is 36336
Size of doc_char dict is 119
Size of doc_token_ngram dict is 0
Size of doc_keyword dict is 0
Size of doc_topic dict is 0
Shrink dict over.
Size of doc_label dict is 2393
Size of doc_token dict is 36336
Size of doc_char dict is 119
Size of doc_token_ngram dict is 0
Size of doc_keyword dict is 0
Size of doc_topic dict is 0
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)
[W pthreadpool-cpp.cc:90] Warni[...]<jupyter_text>Evaluate<jupyter_code>!python eval.py conf/custom.train.hierar.json<jupyter_output>/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
Performance is precision: 0.607306, recall: 0.441860, fscore: 0.511538, right: 266, predict: 438, standard: 602.<jupyter_text>Predict<jupyter_code>! python predict.py conf/custom.train.hierar.json data/custom_test.hierar.json<jupyter_output><empty_output><jupyter_text>Inspect Inference<jupyter_code>from pprint import pprint
import os
def inspect_inference(df: pd.DataFrame, predsfile: str, index=0) -> None:
    predictions = []
    if isinstance(predsfile, str) and os.path.exists(predsfile):
        with open(predsfile) as f:
            predictions = f.readlines()
    df = df.copy()
    pred = predictions[index]
    _df = df.iloc[index]
    print(f"Text ==> {_df.desc[:150]}")
    print(f"GT ==> {_df.keywords}")
    print(f"Predictions ==> {pred}")
    print("-"*25)
inspect_inference(
    DATA_TEST,
    "/content/NeuralNLP-NeuralClassifier/predict.txt",
    index = 15
)<jupyter_output>Text ==> CalSub product files are constructed from calibrated radiance files (infrared and microwave) from either the SNPP, Aqua or MetOP-A/B. Nominally one lo
GT ==> EARTH SCIENCE SERVICES > DATA ANALYSIS AND VISUALIZATION > CALIBRATION/VALIDATION,EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX,EARTH SCIENCE > SPECTRAL/ENGINEERING > INFRARED WAVELENGTHS > BRIGHTNESS TEMPERATURE
Predictions ==> EARTH_SCIENCE_SERVICES--DATA_ANALYSIS_AND_VISUALIZATION--CALIBRATION/VALIDATION,EARTH_SCIENCE--ATMOSPHERE--ATMOSPHERIC_RADIATION--RADIATIVE_FLUX,EARTH_SCIENCE--SPECTRAL/ENGINEERING--INFRARED_WAVELENGTHS--BRIGHTNESS_TEMPERATURE

-------------------------<jupyter_text>Custom EvaluateWe try to evaluate the predictions with the ground truth at different **level** in the hierarcy. ObservationThe performance drops as we go down the hierarchy<jupyter_code>from typing import Tuple
def compute_jaccard(tokens1: List[str], tokens2: List[str]) -> float:
    if not tokens1 or not tokens2:
        return 0.0
    tokens1 = set(tokens1)
    tokens2 = set(tokens2)
    return len(tokens1.intersection(tokens2)) / len(tokens1.union(tokens2))

compute_jaccard(["a", "b"], ["b", "c"])
def get_keywords(df: pd.DataFrame, predsfile: str, index=0) -> Tuple[list, list]:
    predictions = []
    if isinstance(predsfile, str) and os.path.exists(predsfile):
        with open(predsfile) as f:
            predictions = f.readlines()
    
    predictions = map(lambda kw: process_keywords_str(kw, label_sep=";", hierar_sep="--"), predictions)
    predictions = list(predictions)
    df = df.copy()
    df["gt"] = df["keywords"].apply(process_keywords_str)

    def _prep_gt(gts) -> List[List[str]]:
        res = map(lambda gt: ["_".join(x.split()) for x in gt], gts)
        return list(res)

    df["gt"] = df["gt"].apply(_prep_gt)

    return df["gt"].tolist(), predictions
GTs, PREDICTIONS = get_keywords(DATA_TEST, "/content/NeuralNLP-NeuralClassifier/predict.txt")
GTs[0], PREDICTIONS[0]
def evaluate_custom(gts, preds, level=0):
    gts = list(map(lambda x: get_levels(x, level=level), gts))
    preds = list(map(lambda x: get_levels(x, level=level), preds))
    
    metrics = map(lambda x: compute_jaccard(x[0], x[1]), zip(gts, preds))
    metrics = list(metrics)
    return metrics
for level in [0, 1, 2, 3, 4]:
    metrics = evaluate_custom(GTs, PREDICTIONS, level=level)
    avg = np.mean(metrics)
    logger.debug(f"[Level={level}] || {avg}")<jupyter_output>2021-07-30 11:11:44.999 | DEBUG    | __main__:<module>:4 - [Level=0] || 0.616111111111111
2021-07-30 11:11:45.007 | DEBUG    | __main__:<module>:4 - [Level=1] || 0.4543849206349207
2021-07-30 11:11:45.013 | DEBUG    | __main__:<module>:4 - [Level=2] || 0.32370591676841676
2021-07-30 11:11:45.020 | DEBUG    | __main__:<module>:4 - [Level=3] || 0.19290151515151516
2021-07-30 11:11:45.024 | DEBUG    | __main__:<module>:4 - [Level=4] || 0.18427083333333333
{'fraction_non_alphanumeric': 0.13896103896103895, 'fraction_numerical': 0.039097744360902256, 'mean_word_length': 4.496243425995492, 'pattern_counts': {'":': 4, '<': 65, '<?xml version=': 0, '>': 90, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '478919', 'license': 'MIT', 'n_tokens_mistral': 5571, 'n_tokens_neox': 5120, 'n_words': 1222, 'path': 'hierarchical_classification.ipynb', 'repo_name': 'NISH1001/earth-science-text-classification', 'size': '14630'}