<jupyter_start><jupyter_text>TREC-6 question classification DatasetObtained from https://cogcomp.seas.upenn.edu/Data/QA/QC/ (Training set 5 + Test set)```@inproceedings{li2002learning,  title={Learning question classifiers},  author={<NAME> and <NAME>},  booktitle={COLING 2002: The 19th International Conference on Computational Linguistics},  year={2002}}```<jupyter_code>import pandas as pd
import os
from tqdm import tqdm
import hashlib

TRAIN_DATASET_LOCATION = "../datasets/trec/train_5500.label"
TEST_DATASET_LOCATION = "../datasets/trec/TREC_10.label"<jupyter_output><empty_output><jupyter_text>Validity check<jupyter_code>VERIFY_CHECKSUM = True
assert os.path.exists(TRAIN_DATASET_LOCATION)
assert os.path.exists(TEST_DATASET_LOCATION)

def verify_checksum(file, checksum):
    with open(file, "rb") as f:
        file_hash = hashlib.md5()
        while chunk := f.read(8192):
            file_hash.update(chunk)
    assert checksum == file_hash.hexdigest()
    print("Checksum matches")
    
if VERIFY_CHECKSUM:
    verify_checksum(TRAIN_DATASET_LOCATION, '073462e3fcefaae31e00edb1f18d2d02')
    verify_checksum(TEST_DATASET_LOCATION, '323a3554401d86e650717e2d2f942589')<jupyter_output>Checksum matches
Checksum matches<jupyter_text>Read data<jupyter_code>def read_trec(path, nrows=None, remove_subcategory=True):
    with open(path, "r") as f:
        lines = []
        nrow = 0
        for line in f:
            category, text = line.strip().split(" ", 1)
            category = category.split(":")[0] if remove_subcategory else category
            lines.append([category, text])
            nrow += 1
            if nrows and nrow >= nrows:
                break
            
    return pd.DataFrame(lines)

read_trec(TRAIN_DATASET_LOCATION, 10, remove_subcategory=False)
df_trec_train = read_trec(TRAIN_DATASET_LOCATION)
df_trec_train.columns = ['target', 'text']
df_trec_train.shape
df_trec_train.head()
1250/5452
vc = df_trec_train.target.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per question category', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Category')
ax.set_xlabel('Count')<jupyter_output><empty_output><jupyter_text>Export File<jupyter_code>PREPROCESSED_FILE = '../datasets/preprocessed/trec_train.csv'
df_trec_train.to_csv(PREPROCESSED_FILE, index=False)<jupyter_output><empty_output><jupyter_text>Detect Non-English texts<jupyter_code>import fasttext
import numpy as np
from tqdm.notebook import tqdm_notebook

tqdm_notebook.pandas()

LANGUAGE_DETECTION_MODEL = "../fasttext/language_identification/lid.176.bin" #https://fasttext.cc/docs/en/language-identification.html
model = fasttext.load_model(LANGUAGE_DETECTION_MODEL)
def get_language(paragraph, k=None, return_all=False, min_certainty=0.25):
    if pd.isna(paragraph):
        return np.nan
    sentences = [line for line in paragraph.split("\n") if line != '']
    if len(sentences) < 1:
        return np.nan
    class_labels, certainties = model.predict(sentences, k)
    # sum certainties per class label and then divide by length -> avg certainty
    class_lookup, indexed_lang = np.unique(np.hstack(class_labels), return_inverse=True)
    summed_certainties = np.bincount(indexed_lang, weights=np.hstack(certainties))/len(sentences)
    if return_all:
        return dict(zip(class_lookup, summed_certainties))
    
    # get most likely lang
    lang_id = np.argmax(summed_certainties)
    if summed_certainties[lang_id] < min_certainty:
        return np.nan
    return class_lookup[lang_id]        
df_trec_train['lang'] = df_trec_train.text.progress_apply(lambda review: get_language(review, k=5))
vc = df_trec_train.lang.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per language', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Language')
ax.set_xlabel('Count')
vc.nlargest(5)
f"{vc['__label__en']/df_trec_train.shape[0]*100:.4f}% of samples are english" 
df_trec_train[df_trec_train.lang != '__label__en']<jupyter_output><empty_output><jupyter_text>Test Set analysis<jupyter_code>df_trec_test = read_trec(TEST_DATASET_LOCATION)
df_trec_test.columns = ['target', 'text']
df_trec_test.shape
df_trec_test.head()
vc = df_trec_test.target.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per question category', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Category')
ax.set_xlabel('Count')<jupyter_output><empty_output><jupyter_text>Export file<jupyter_code>PREPROCESSED_FILE = '../datasets/preprocessed/trec_test.csv'
df_trec_test.to_csv(PREPROCESSED_FILE, index=False)
df_trec_test['lang'] = df_trec_test.text.progress_apply(lambda review: get_language(review, k=5))
vc = df_trec_test.lang.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per language', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Language')
ax.set_xlabel('Count')
vc.nlargest(5)
f"{vc['__label__en']/df_trec_test.shape[0]*100:.4f}% of samples are english"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1303850552802135, 'fraction_numerical': 0.025352649637819292, 'mean_word_length': 5.550561797752809, 'pattern_counts': {'":': 1, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '300053', 'license': 'Apache-2.0', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1950, 'n_words': 335, 'path': 'preprocessing/trec.ipynb', 'repo_name': 'freecraver/zimp_experiments', 'size': '5250'}