<jupyter_start><jupyter_text>1. EDA<jupyter_code>import pandas as pd
from tqdm import tqdm
tqdm.pandas()
df = pd.read_csv("data/orig/new_train.csv")
print(len(df))
df.head(2)
df[["poi", "street"]] = df["POI/street"].str.split("/", expand=True)
df.head(2)<jupyter_output><empty_output><jupyter_text>1.1 Distributions<jupyter_code>(df["poi"] == "").value_counts().plot(kind="bar")
(df["street"] == "").value_counts().plot(kind="bar")
(df["POI/street"] == "").value_counts().plot(kind="bar")<jupyter_output><empty_output><jupyter_text>Observations:1. A large proportion (~60%) of the text don't have person names.2. Most of the text (~80%) have address information.3. All text have either person name or address info or both.  1.2 See whether this problem can be modelled as span-based problem<jupyter_code>import re
import json

import numpy as np
from transformers import AutoTokenizer, AutoModelForTokenClassification
tokenizer = AutoTokenizer.from_pretrained("cahya/xlm-roberta-large-indonesian-NER")
patt = re.compile("((?<=.)(?![a-zA-Z0-9]))|((?<![a-zA-Z0-9])(?<=.)(?=.))")
def pre_tokenize(s):
    """Split punctuations, etc."""
    s_tokenized = patt.split(s)
    s_tokenized = list(
        filter(lambda x: x is not None and len(x) > 0 and not x.isspace(), 
               s_tokenized)
    )
    s = " ".join(s_tokenized)
    return s

def transforms(row):
    raw_address, poi, street = row[["raw_address", "poi", "street"]]

    raw_address_tokenized = tokenizer.tokenize(pre_tokenize(raw_address))
    poi_tokenized = tokenizer.tokenize(pre_tokenize(poi))
    street_tokenized = tokenizer.tokenize(pre_tokenize(street))

    # Get the start and end indices of each of [POI, street] in the raw address
    poi_start, poi_end, street_start, street_end = -1, -1, -1, -1

    if len(poi_tokenized) > 0:
        for i, raw_address_i in enumerate(raw_address_tokenized):
            if raw_address_i == poi_tokenized[0]:
                for j in range(1, len(poi_tokenized)):
                    if i + j >= len(raw_address_tokenized):
                        break
                    if raw_address_tokenized[i + j] != poi_tokenized[j]:
                        break
                else:
                    poi_start = i
                    poi_end = i + len(poi_tokenized) - 1
                    break

        if poi_start == -1 or poi_end == -1:
            print(f"POI not found. ID: {row['id']}, raw address: {raw_address_tokenized}, POI: {poi_tokenized}")
            raw_address = np.nan

    if len(street_tokenized) > 0:
        for i, raw_address_i in enumerate(raw_address_tokenized):
            if raw_address_i == street_tokenized[0]:
                for j in range(1, len(street_tokenized)):
                    if i + j >= len(raw_address_tokenized):
                        break
                    if raw_address_tokenized[i + j] != street_tokenized[j]:
                        break
                else:
                    street_start = i
                    street_end = i + len(street_tokenized) - 1
                    break

        if street_start == -1 or street_end == -1:
            print(f"Street not found. ID:{row['id']}, raw address: {raw_address_tokenized}, street: {street_tokenized}")
            raw_address = np.nan

    return pd.Series({"raw_address": raw_address, "poi": poi, "street": street,
                      "raw_address_tok": raw_address_tokenized, 
                      "poi_tok": poi_tokenized, "street_tok": street_tokenized,
                      "poi_start": poi_start, "poi_end": poi_end, 
                      "street_start": street_start, "street_end": street_end})
df_processed = df.progress_apply(transforms, axis=1)
print(len(df_processed))
df_processed.head(2)
# Remove invalid entries
before = len(df_processed)
df_processed = df_processed.dropna(subset=["raw_address"])
print(f"Before: {before}, after: {len(df_processed)}")
df_processed = df_processed.T.to_dict()
print(len(df_processed))
df_processed[0]
with open("data/processed/train_processed.json", "w") as fout:
    json.dump(df_processed, fout, indent=2)<jupyter_output><empty_output><jupyter_text>2. EDA (con't)<jupyter_code>import json

import pandas as pd
with open("data/processed/train_processed.json", "r") as fin:
    df = json.load(fin)

df = pd.DataFrame.from_dict(df, orient="index")
print(len(df))
df.head(3)<jupyter_output>299999<jupyter_text>2.1 Length distribution<jupyter_code>df["raw_address_tok"].apply(len).hist(bins=20)
lengths = df["poi_tok"].apply(len)
lengths = lengths[lengths > 0]
lengths.value_counts().plot(kind="bar")
lengths.value_counts().tail()
lengths = df["street_tok"].apply(len)
lengths = lengths[lengths > 0]
lengths.value_counts().plot(kind="bar")
lengths.value_counts().tail()<jupyter_output><empty_output><jupyter_text>Observations1. Most of POI/street has length of 3 - 5 (tokenized).2. The two distributions (street and POI) are similar.3. There are quite a few of outliers (e.g., length of 49 for POI and length of 52 for street).  2.2 Normalized length distribution<jupyter_code>lengths = df["poi_tok"].apply(len)
s_lengths = df["raw_address_tok"].apply(len)

mask = lengths > 0
lengths = lengths[mask]
s_lengths = s_lengths[mask]

(lengths / s_lengths).hist(bins=20)
lengths = df["street_tok"].apply(len)
s_lengths = df["raw_address_tok"].apply(len)

mask = lengths > 0
lengths = lengths[mask]
s_lengths = s_lengths[mask]

(lengths / s_lengths).hist(bins=20)<jupyter_output><empty_output><jupyter_text>Observations1. The distributions are quite similar.2. There is a significant amount of POIs which occupy the entire sentences. Note that, **however**, the reverse is expected (i.e., street is expected to have this behavior, not the POI).  2.3 Normalized index distribution<jupyter_code>start = df["poi_start"]
end = df["poi_end"]
s_lengths = df["raw_address_tok"].apply(len)

mask = start >= 0
start = start[mask]
end = end[mask]
s_lengths = s_lengths[mask]

(start / (s_lengths - 1 + 1e-10)).hist(bins=20)
# Fraction of POI that starts at the beginning of the sentence
(start == 0).sum() / len(start)
(end / (s_lengths - 1 + 1e-10)).hist(bins=20)
start = df["street_start"]
end = df["street_end"]
s_lengths = df["raw_address_tok"].apply(len)

mask = start >= 0
start = start[mask]
end = end[mask]
s_lengths = s_lengths[mask]

(start / (s_lengths - 1 + 1e-10)).hist(bins=20)
# Fraction of POI that starts at the beginning of the sentence
(start == 0).sum() / len(start)
(end / (s_lengths - 1 + 1e-10)).hist(bins=20)<jupyter_output><empty_output><jupyter_text>Observations1. The start index distributions are very similar. The end index distributions, however, are not.2. There is a significant peak at start index 0 for both street and POI, indicating that a raw address is most likely to start with a street name or a person name, which is expected.3. There is also a small peak at end index 1.0 (last token in the sentence), indicating that these are sometimes written at the end of the adress, which is also expected. This is especially applicable for POI (its peak is significant).  3. Stratified samplingBased on the occurrences of POI and street (`0`: no POI, no street, `1`: no street, `2`: no POI, `3`: have both) and total lengths of the tokenize raw address.<jupyter_code>import os
import math
import json

import pandas as pd
num_folds = 10
src_path = "data/processed/train_processed.json"
dest_dir = "data/processed/splits/"
with open(src_path, "r") as fin:
    df = json.load(fin)

df = pd.DataFrame.from_dict(df, orient="index")
print(len(df))
df.head(2)
df["raw_address_lengths"] = df["raw_address_tok"].apply(len)

street = (df["street_start"].to_numpy() != -1)
poi = (df["poi_start"].to_numpy() != -1) * 2
df["have_street_and_poi"] = street + poi

df.head(2)
folds = [[] for _ in range(num_folds)]
df = df.sample(frac=1.0, random_state=136)

for (_, _), sub_df in df.groupby(["raw_address_lengths", "have_street_and_poi"]):
    size = math.ceil(len(sub_df) / num_folds)

    for i in range(num_folds):
        start = i * size
        end = min((i + 1) * size, len(sub_df))
        folds[i].append(sub_df.iloc[start:end])

folds = [pd.concat(fold, ignore_index=True) for fold in folds]
print([len(fold) for fold in folds])
folds[0].head(2)
# Post process and save
save_path = os.path.join(dest_dir, "fold_{}.json")

for i, df_fold in enumerate(folds):
    df_fold = df_fold[["raw_address", "poi", "street", "raw_address_tok", 
                       "poi_tok", "street_tok", "poi_start", "poi_end", 
                       "street_start", "street_end"]]
    with open(save_path.format(i), "w") as fout:
        json.dump(df_fold.T.to_dict(), fout, indent=2)<jupyter_output><empty_output><jupyter_text>4. Process test data<jupyter_code>import re
import json

import pandas as pd
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForTokenClassification

tqdm.pandas()
tokenizer = AutoTokenizer.from_pretrained("cahya/xlm-roberta-large-indonesian-NER")
patt = re.compile("((?<=.)(?![a-zA-Z0-9]))|((?<![a-zA-Z0-9])(?<=.)(?=.))")

df = pd.read_csv("data/orig/new_test.csv")
print(len(df))
df.head(3)
def pre_tokenize(s):
    """Split punctuations, etc."""
    s_tokenized = patt.split(s)
    s_tokenized = list(
        filter(lambda x: x is not None and len(x) > 0 and not x.isspace(), 
               s_tokenized)
    )
    s = " ".join(s_tokenized)
    return s

def transforms(row):
    raw_address = row["raw_address"]
    raw_address_tokenized = tokenizer.tokenize(pre_tokenize(raw_address))

    return pd.Series({"raw_address": raw_address,
                      "raw_address_tok": raw_address_tokenized})
df_processed = df.progress_apply(transforms, axis=1)
print(len(df_processed))
df_processed.head(2)
df_processed = df_processed.T.to_dict()
print(len(df_processed))
df_processed[0]
with open("data/processed/test_processed.json", "w") as fout:
    json.dump(df_processed, fout, indent=2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13808709644976366, 'fraction_numerical': 0.01408025746756512, 'mean_word_length': 3.8085106382978724, 'pattern_counts': {'":': 12, '<': 42, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '472079', 'license': 'MIT', 'n_tokens_mistral': 3618, 'n_tokens_neox': 3432, 'n_words': 825, 'path': '1_EDA.ipynb', 'repo_name': 'saobangmath/CZ4034-Aspect-Classification-Model', 'size': '9943'}