<jupyter_start><jupyter_text>dontpatronizeme_pcl<jupyter_code>data_pcl = pd.read_csv("./dontpatronizeme_v1.4/dontpatronizeme_pcl.tsv", sep="\t", names=['par_id','art_id','keyword','country_code','text','label'], skiprows=3)
data_pcl.head()
data_pcl['text']
data_pcl['label']<jupyter_output><empty_output><jupyter_text>Each paragraph has been annotated by two annotators as 0 (No PCL), 1 (borderline PCL) and 2 (contains PCL). The combined annotations have been used in the following graded scale:	0 -> Annotator 1 = 0 AND Annotator 2 = 0	1 -> Annotator 1 = 0 AND Annotator 2 = 1 OR Annotator 1 = 1 AND Annotator 2 = 0	2 -> Annotator 1 = 1 AND Annotator 2 = 1	3 -> Annotator 1 = 1 AND Annotator 2 = 2 OR Annotator 1 = 2 AND Annotator 2 = 1	4 -> Annotator 1 = 2 AND Annotator 2 = 2<jupyter_code>x = data_pcl['text'].to_numpy(dtype=str)
y = data_pcl['label'].to_numpy(dtype=np.int)
y.shape, y.min(), y.max(), y.mean(), y.var()
n_per_annotation = {}
for annotation in y:
    n_per_annotation[annotation] = n_per_annotation.get(annotation, 0) + 1
print(n_per_annotation)
plt.bar(n_per_annotation.keys(), n_per_annotation.values())
plt.grid()<jupyter_output>{0: 8529, 1: 947, 2: 144, 3: 458, 4: 391}<jupyter_text>The experiments reported in the paper consider the following tag grouping: 	- {0,1}   = No PCL	- {2,3,4} = PCL<jupyter_code>y_binary = np.array((y>1.5), dtype=int)
y_binary.shape, y_binary.sum(), y_binary.sum()/len(y_binary)
n_per_binary = {}
n_per_binary[0] = len(y_binary) - y_binary.sum()
n_per_binary[1] = y_binary.sum()
print(n_per_binary)
plt.bar([0.5, 3], n_per_binary.values(), width=[1.9, 2.9], label='Binary label')
plt.bar(n_per_annotation.keys(), n_per_annotation.values(), label="Annotation")
plt.title("Number of examples per label")
plt.legend()
plt.grid()
n_words = np.array([len(text.split()) for text in x])
plt.hist(n_words)
plt.yscale('log')
plt.ylabel("Nb paragraphs")
plt.xlabel("Nb words in paragraph")
plt.grid()
# Mininmum, maximum, and average number of words per example
n_words.min(), n_words.max(), n_words.mean().round(3)
n_words_per_label = {}
n_words_per_label[0] = []
n_words_per_label[1] = []
for i,label in enumerate(y_binary):
    n_words_per_label[label].append(n_words[i])
    
n_words_per_label[1] = np.array(n_words_per_label[1])
n_words_per_label[0] = np.array(n_words_per_label[0])
# Average number of words and standard deviation for examples WITHOUT PCL
n_words_per_label[0].sum() / len(n_words_per_label[0]), n_words_per_label[0].std()
# Average number of words and standard deviation for examples WITH PCL
n_words_per_label[1].sum() / len(n_words_per_label[1]), n_words_per_label[1].std()
bars = [n_words_per_label[0].sum() / len(n_words_per_label[0]), n_words_per_label[1].sum() / len(n_words_per_label[1])]
errors = [n_words_per_label[0].std(), n_words_per_label[1].std()]
plt.bar(["NO PCL", "PLC"],bars, yerr=errors, capsize=10)
plt.title("Average size of paragraph and standard deviation")
plt.ylabel("Number of words")
plt.grid()
# Medians
np.median(n_words_per_label[0]), np.median(n_words_per_label[1])
plt.hist(n_words_per_label[0], label="NO PCL")
plt.hist(n_words_per_label[1], label="PCL")
plt.yscale('log')
plt.yscale('log')
plt.legend()
plt.title("Histogram of paragraph sizes in the dataset")
plt.xlabel("Number of words per paragraph")
plt.ylabel("Number of paragraphs")
plt.grid()<jupyter_output><empty_output><jupyter_text>dontpatronizeme_categories<jupyter_code>names = ["par_id", "art_id", "text", "keyword", "country_code", "span_start", "span_finish", "span_text", "pcl_category", "number_of_annotators"]
data_categories = pd.read_csv("./dontpatronizeme_v1.4/dontpatronizeme_categories.tsv", sep="\t", names=names, skiprows=3)
data_categories.head()
n_example_per_category = {}
for cat in data_categories["pcl_category"]:
    n_example_per_category[cat] = n_example_per_category.get(cat, 0) + 1
n_example_per_category
plt.bar(n_example_per_category.keys(), n_example_per_category.values())
plt.xticks(rotation=90)
plt.grid()
n_categories_per_example = np.zeros(len(data_pcl), dtype=int)
for i in sorted(data_categories['par_id'] - 1):
    n_categories_per_example[i] = n_categories_per_example[i] +1
(n_categories_per_example!=0).sum() == y_binary.sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15550857411322527, 'fraction_numerical': 0.02654451491660794, 'mean_word_length': 7.110476190476191, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '360894', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1644, 'n_words': 328, 'path': 'playing_with_data.ipynb', 'repo_name': 'TibiGG/NLP_CW', 'size': '4257'}