<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/overfit-ir/persian-twitter-ner.git
import os
tweets1 = []
tweet = []
with open('persian-twitter-ner/twitter_data/kappa_test_data/test_data1.txt', 'r') as file:
    for line_number, line in enumerate(file.readlines()):
        if line != '\n':
            tweet.append(line.replace('\n', '').split('\t'))
        else:
            tweets1.append(tweet)
            tweet = []
tweets2 = []
tweet = []
with open('persian-twitter-ner/twitter_data/kappa_test_data/test_data12.txt', 'r') as file:
    for line_number, line in enumerate(file.readlines()):
        if line != '\n':
            tweet.append(line.replace('\n', '').split('\t'))
        else:
            tweets2.append(tweet)
            tweet = []
tags1 = [x[1] for tweet in tweets1 for x in tweet ]
tags2 = [x[1] for tweet in tweets2 for x in tweet ]
diff = [0 if x==y else 1 for x,y in zip(tags1, tags2)]
sum(diff)
len([x  for x in tags1 if x != 'O'])
len(tags1)<jupyter_output><empty_output><jupyter_text>KAPPA<jupyter_code>from sklearn.metrics import confusion_matrix, cohen_kappa_score
import pandas as pd
import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt
array = confusion_matrix(tags1, tags2,['B-PER', 'I-PER', 'B-ORG', 'B-LOC', 'I-ORG', 'B-NAT', 'B-EVE',
       'B-POG', 'I-NAT', 'I-EVE', 'I-LOC', 'I-POG'])
df_cm = pd.DataFrame(array, index = ['B-PER', 'I-PER', 'B-ORG', 'B-LOC', 'I-ORG', 'B-NAT', 'B-EVE',
       'B-POG', 'I-NAT', 'I-EVE', 'I-LOC', 'I-POG'],
                  columns = ['B-PER', 'I-PER', 'B-ORG', 'B-LOC', 'I-ORG', 'B-NAT', 'B-EVE',
       'B-POG', 'I-NAT', 'I-EVE', 'I-LOC', 'I-POG'])
plt.figure(figsize = (20,14))
sn.heatmap(df_cm, annot=True, cmap='Blues')
cohen_kappa_score(tags1, tags2, ['B-PER', 'I-PER', 'B-ORG', 'B-LOC', 'I-ORG', 'B-NAT', 'B-EVE',
       'B-POG', 'I-NAT', 'I-EVE', 'I-LOC', 'I-POG'])
cohen_kappa_score(tags1, tags2, ['B-PER', 'I-PER', 'B-ORG', 'O', 'B-LOC', 'I-ORG', 'B-NAT', 'B-EVE',
       'B-POG', 'I-NAT', 'I-EVE', 'I-LOC', 'I-POG'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22372227579556414, 'fraction_numerical': 0.013982642237222759, 'mean_word_length': 3.3684210526315788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '566353', 'license': 'MIT', 'n_tokens_mistral': 907, 'n_tokens_neox': 884, 'n_words': 182, 'path': 'quality_of_tagging.ipynb', 'repo_name': 'overfit-ir/parstwiner', 'size': '2074'}