<jupyter_start><jupyter_text>Dataset InvestigationExperimental Data for Question Classificationobtained from https://cogcomp.seas.upenn.edu/Data/QA/QC/<jupyter_code>from collections import defaultdict
import requests
import os
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Download dataset and pre-processingRun this cell to download the dataset:Possibilies: `train_5500.label`, `train_4000.label`,`train_3000.label`,`train_2000.label`, `train_1000.label`,`<jupyter_code>filename = 'train_5500.label'

os.makedirs('datasets', exist_ok=True)
url = f'https://cogcomp.seas.upenn.edu/Data/QA/QC/{filename}'
with open(f'datasets/{filename}', 'xb') as file:
    response = requests.get(url, allow_redirects=True)
    file.write(response.content)<jupyter_output><empty_output><jupyter_text>Save queries and class labels to Python lists:<jupyter_code>query = []
label = []

with open(f'datasets/{filename}', 'r', errors='replace') as data:
    for line in data.readlines():
        tokens = line.split()
        label.append(tokens[0])
        query.append(tokens[1:])
        
print(query[0])
print(label[0])
print(len(label))
lengths = [len(q) for q in query]
print(f'max sentence length: {max(lengths)}')

char_lengths = [len(word) for q in query for word in q]
print(f'max word length: {max(char_lengths)}')<jupyter_output>max sentence length: 37
max word length: 28<jupyter_text>Extract classes and subclasses<jupyter_code>label_dict = defaultdict(lambda: [])

for token in label:
    mainc, subc = token.split(':')
    if not subc in label_dict[mainc]:
        label_dict[mainc].append(subc)

print(f'main class count: {len(label_dict.keys())}')
print(f'subclass count: {sum([len([j for j in i]) for i in label_dict.values()])}')
label_dict<jupyter_output>main class count: 6
subclass count: 50<jupyter_text>Class Distribution<jupyter_code>coarse_labels = [i.split(':')[0] for i in label]
fine_labels = [i.split(':')[1] for i in label]

coarse_dict = defaultdict(lambda: 0)
fine_dict = defaultdict(lambda: 0)

for l in coarse_labels:
    coarse_dict[l] += 1

for l in fine_labels:
    fine_dict[l] += 1


x = np.arange(len(coarse_dict.keys()))
plt.bar(x, coarse_dict.values())
plt.xticks(x, coarse_dict.keys())
plt.title('Distribution of Coarse Classes')
plt.ylabel('nr. of queries')
plt.show()

x = np.arange(len(fine_dict.keys()))
sorted_tup = [(i,j) for i,j in fine_dict.items()]
sorted_tup = sorted(sorted_tup, key=lambda x: x[1])
sorted_keys = [i[0] for i in sorted_tup]
sorted_values = [i[1] for i in sorted_tup]

plt.figure(figsize=(20,10))
plt.bar(x, sorted_values)
plt.xticks(x, sorted_keys, rotation=45)
plt.title('Distribution of Fine Classes')
plt.ylabel('nr. of queries')
plt.show()<jupyter_output><empty_output><jupyter_text>Query Length<jupyter_code>query_lengths = [len(q) for q in query]
print(f'Longest query: {max(query_lengths)}')
print(query[max(zip(query_lengths, range(len(query_lengths))))[1]])
print('---')
print(f'Shortest query: {min(query_lengths)}')
print(query[min(zip(query_lengths, range(len(query_lengths))))[1]])<jupyter_output>Longest query: 37
['Why', 'do', 'people', 'get', 'goosebumps', 'when', 'they', 'have', 'something', 'emotional', 'happen', 'to', 'them', ',', 'like', 'when', 'they', 'hear', 'a', 'beautiful', 'piece', 'of', 'music', ',', 'or', 'see', 'something', 'beautiful', ',', 'or', 'get', 'aroused', 'by', 'someone', 'they', 'love', '?']
---
Shortest query: 3
['Define', 'cosmology', '.']<jupyter_text>Unique Tokens<jupyter_code>unique_tokens = set()

for q in query:
    unique_tokens.update(q)
            
print(f'Number of unique tokens: {len(unique_tokens)}')<jupyter_output>Number of unique tokens: 9448<jupyter_text>Sequence Length Distribution<jupyter_code>query_lengths = [len(q) for q in query]
seq_dict = defaultdict(lambda: 0)
for length in query_lengths:
    seq_dict[length] += 1
    
seq_tup = [(i,j) for i,j in seq_dict.items()]
seq_tup = sorted(seq_tup, key=lambda x: x[0])
sorted_lengths = [i[0] for i in seq_tup]
sorted_cnt = [i[1] for i in seq_tup]

x = np.arange(len(sorted_lengths))
plt.figure(figsize=(20,10))
plt.bar(x, sorted_cnt)
plt.xticks(x, sorted_lengths)
plt.title('Distribution of Sequence Lengths')
plt.ylabel('nr. of queries')
plt.xlabel('sequence length')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16497227356746766, 'fraction_numerical': 0.015711645101663587, 'mean_word_length': 6.387372013651877, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '405828', 'license': 'MIT', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1613, 'n_words': 357, 'path': 'notebooks/dataset.ipynb', 'repo_name': 'XlightwolfX/DL_NLP_project', 'size': '4328'}