<jupyter_start><jupyter_text>Cleaning Up (& Stats About It) - For each annotator:   - How many annotation files?   - How many txt files?   - Number of empty .ann files   - How many non-empty .ann files have a `TranscriptionError_Document`/`DuplicatePage` tag?   - How many .ann files have ONLY one of those two tags and are empty o/w? -> remove if so      => remove corresponding .txt files     => create new corpus<jupyter_code>def get_all_files(annotator):
    """ collapsing folder structure per annotator"""
    data_dir = "../Data/"
    ann_dir = data_dir+annotator+"/"
    for cur_dir in glob(ann_dir+"/6*"):
        txt_files = sorted(glob(cur_dir+"/*.txt"))
        ann_files = sorted(glob(cur_dir+"/*.ann"))
        yield from zip(txt_files, ann_files)
        
        
def has_error_tag(any_string):
    """Return strings with error tags"""
    return "TranscriptionError_Document" in any_string or\
                "DuplicatePage" in any_string


def remove_error_tag_lines(ann_file_content):
    return [line for line in ann_file_content.strip().split("\n") 
             if not has_error_tag(line)] 
annotators = "A  B  C  <NAME>".split()
results = {}

print("Total Annotation Files Per Annotator\n")
for anno in annotators:
    empty = []
    cur_keep = []

    error_tag = []
    error_tag_but_non_empty = []

    ann_files = list(get_all_files(anno))
    print(anno, len(ann_files))

    for txt, ann in ann_files:
        with open(ann) as handle:
            contents = handle.read()
            
            if not contents.strip():
                empty.append((txt, ann))  
            elif has_error_tag(contents):
                
                error_tags_removed = remove_error_tag_lines(
                                contents
                        )
                
                if error_tags_removed == []:
                    error_tag.append((txt, ann))
                else:
                    error_tag_but_non_empty.append((txt, ann))                
            else:
                cur_keep.append((txt, ann))
                
    
    results[anno] = [cur_keep, empty, error_tag, error_tag_but_non_empty]
    
from tabulate import tabulate

stats = pd.DataFrame([
    [k, sum(map(len, v))]+
    [len(v[0])+len(v[-1])]+
    list(map(len, v)) for k, v in results.items()
    
],
columns=["Annotator", "Total", "Keep",
         "Non-empty-No error", "Empty", "Error", "Err.&Non-Empty"]).set_index("Annotator")
print(stats)
stats_T = pd.melt(stats[["Total", "Empty", "Keep", "Error"]].reset_index(), 
                  id_vars=["Annotator"], value_name="Number")

plt.figure(figsize=(10, 7))
sns.barplot(data=stats_T, x='Annotator', y="Number", hue="variable")
keep = {anno: v[0]+v[-1] for anno, v in results.items()}

{k: len(v) for k, v in keep.items()}
# keep<jupyter_output><empty_output><jupyter_text>Make New Corpusby copying files<jupyter_code>from shutil import copy2

already_copied = True

if not already_copied:
    from tqdm import tqdm 
    os.makedirs('Keep')

    for anno, ls in tqdm(keep.items()):
        cur_dir = f"Keep/{anno}"
        os.makedirs(cur_dir)

        for txt, ann in ls:
            copy2(txt, cur_dir)
            copy2(ann, cur_dir)
else:
    print("Already copied, doing nothing!")<jupyter_output>Already copied, doing nothing!<jupyter_text>Pairwise Intersections of Annotation Files<jupyter_code>def only_names(file_list):
    "returns only names of files in a particular list"
    return [ann.split("/")[-1] for txt, ann in file_list]


ls = []
for a1, fs1 in keep.items():
    for a2, fs2 in keep.items():
        if not a1 == a2:
            
            names1, names2 = only_names(fs1), only_names(fs2)
            inter = set(names1) & set(names2) #names of files are identical
            val = len(inter)/len(names1)
            
            total_names1 = only_names(tup for ls in results[a1] for tup in ls)
            total_names2 = only_names(tup for ls in results[a2] for tup in ls)
            
            total_inter = set(total_names1) & set(total_names2)
            total_val = len(total_inter)/len(total_names1)
    
            jacc_val = len(set(names1).intersection(set(names2)))/len(set(names1).union(set(names2)))
            jacc_val_2 = len(set(total_names1).intersection(set(total_names2)))/len(set(total_names1).union(set(total_names2)))
            
            
            
            ls.append([a1, a2, len(inter), val, 
                       len(total_inter), total_val, jacc_val, jacc_val_2])
            
            
inter_stats = pd.DataFrame(ls, 
                           columns=["Anno1", "Anno2", 
                                    "Intersection", "normed_Intersection",
                                   "total_Intersection", "total_normed_Intersection", "Jaccard_distance", "Jaccard_Distance_2"])
# inter_stats<jupyter_output><empty_output><jupyter_text>Jaccard Distance to Understand Overlap Pages between Annotators<jupyter_code>inter_stats_T = inter_stats.pivot_table(
                values="Jaccard_distance",
    index="Anno1", columns="Anno2"
)

sns.heatmap(inter_stats_T*100, annot=True, cmap="YlGnBu")
_ = plt.title("Before Clean Up: Jaccard Distance (percentage)")

plt.show()

inter_stats_T = inter_stats.pivot_table(
                values="Jaccard_Distance_2",
    index="Anno1", columns="Anno2"
)

sns.heatmap(inter_stats_T*100, annot=True, cmap="YlGnBu")
_ = plt.title("After Clean Up: Jaccard Distance (percentage)")

plt.show()


# inter_stats_T = inter_stats.pivot_table(
#                 values="Intersection",
#     index="Anno1", columns="Anno2"
# )

# sns.heatmap(inter_stats_T, 
#             annot=True, cmap="YlGnBu")

# _ = plt.title("Before Clean Up: Raw Counts")<jupyter_output><empty_output><jupyter_text>**Conclusion**: Each pair of annotators annotated on average have 6% overlap (over the total documents they annotated together).  Check Tag Distributions<jupyter_code>def get_lines(ann_file):
    with open(ann_file) as handle:
        for l in handle:
            if not l.strip(): continue
            yield l.strip().split("\t")

def get_entities(ann_file):
    for line in get_lines(ann_file):
        if line[0].startswith("T") and len(line) >= 2:
            tag_type, tag, string = line
            yield tag.split()[0]


        
ents = {a: [e for txt, ann in files for e in get_entities(ann)]
       for a, files in keep.items()}
from collections import Counter

entity_stats = pd.DataFrame(
    [[a, e, c] for a in ents for e, c in Counter(ents[a]).items() if not e in ["DuplicatePage", "Noteworthy", "TranscriptionError_Document"]],
    columns=["Annotator", "EntityType", "Count"]
)



plt.figure(figsize=(10, 7))
_ = sns.barplot(data=entity_stats, x='Annotator', y="Count", hue="EntityType")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12527569475077194, 'fraction_numerical': 0.009851492427584178, 'mean_word_length': 2.682728749323227, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '1197', 'license': 'MIT', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2213, 'n_words': 509, 'path': 'data_2/Notebooks/JaccardDistanceAnalysis.ipynb', 'repo_name': 'budh333/UnSilence_VOC', 'size': '6807'}