<jupyter_start><jupyter_text>Properties<jupyter_code>input_path = "./candidates/output/candidates_similarity.with_categories.20000.csv"
output_path = "./similarity_categories_11-6-2021.xmi"<jupyter_output><empty_output><jupyter_text>Install  Install Package to write CAS XMI files See https://github.com/dkpro/dkpro-cassis<jupyter_code>#pip install dkpro-cassis<jupyter_output>Requirement already satisfied: dkpro-cassis in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (0.5.1)
Requirement already satisfied: lxml>=4.5.0 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (4.6.2)
Requirement already satisfied: attrs>=19.3.0 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (20.3.0)
Requirement already satisfied: sortedcontainers>=2.1.0 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (2.3.0)
Requirement already satisfied: toposort>=1.5 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (1.6)
Requirement already satisfied: more-itertools>=8.2.0 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (8.7.0)
Requirement already satisfied: deprecation>=2.0.7 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from dkpro-cassis) (2.1.0)
Requirement already satisfied: impo[...]<jupyter_text>Installing Pandas for better csv/tsv support<jupyter_code>#pip install pandas<jupyter_output>Requirement already satisfied: pandas in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (1.2.2)
Requirement already satisfied: python-dateutil>=2.7.3 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from pandas) (2.8.1)
Requirement already satisfied: pytz>=2017.3 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from pandas) (2021.1)
Requirement already satisfied: numpy>=1.16.5 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from pandas) (1.20.1)
Requirement already satisfied: six>=1.5 in /Users/mika/.pyenv/versions/3.8.7/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)
Note: you may need to restart the kernel to use updated packages.<jupyter_text>Skript  Import TSV Input Files<jupyter_code>import pandas as pd
segments = pd.read_csv(input_path,sep=',')
segments<jupyter_output><empty_output><jupyter_text>Load Typesystem<jupyter_code>from cassis import *
import numpy as np

with open('typesystem.xml', 'rb') as f:
    typesystem = load_typesystem(f)<jupyter_output><empty_output><jupyter_text>Create CAS & Sofa<jupyter_code># This creates automatically the view `_InitialView`
cas = Cas(typesystem=typesystem)

# Sentence Annotation Type
Sentence = typesystem.get_type('webanno.custom.Sentence')
SentenceRelation = typesystem.get_type('webanno.custom.SentenceRelation')

# Creating Sentence Annotation & Sofa
sofa = ""
# Store Sentence generated IDs
alreadyIncludedSentences = []

def createId(doc_id, sent_start, sent_end):
    return "doc_" + str(doc_id) + "_start_" + str(sent_start) + "_end_" + str(sent_end)

# get Sentence Annotation by id
def searchForSentenceAnnotation(id):
    global cas
    for s in cas.select('webanno.custom.Sentence'):
        if s.sent_id == id:
            return s

relations = []
def createAnnotation_pairInput(x):
    global cas
    global relations
    # Add Sentence to Sofa & remember begin & end
    def createAnnotation(doc_id, sent_start, sent_end, text, url, title):
        global sofa
        global cas
        internalID = createId(doc_id, sent_start, sent_end)
        # If Sentence Annotation is not already created, create and add to CAS
        if internalID not in alreadyIncludedSentences:
            b = len(sofa)
            e = len(sofa + text)
            sofa += text
            alreadyIncludedSentences.append(internalID)
            cas.add_annotation(Sentence(begin=b, end=e, doc_id=doc_id, sent_id=internalID, sent_start=sent_start, sent_end=sent_end, url=url, title=title))
        return internalID            
    # apply for both sentences
    id1 = createAnnotation(x["a_doc_id"], x["a_start"], x["a_end"], x["a_text"], x["a_url"], x["a_title"])
    id2 = createAnnotation(x["b_doc_id"], x["b_start"], x["b_end"], x["b_text"], x["b_url"], x["b_title"])
    sent1 = searchForSentenceAnnotation(id1)
    sent2 = searchForSentenceAnnotation(id2)
    # create Relation Annotation
    relations.append(SentenceRelation(begin=sent1.begin, end=sent1.end, Dependent=sent1, Governor=sent2, label="unset"))
    relations.append(SentenceRelation(begin=sent2.begin, end=sent2.end, Dependent=sent2, Governor=sent1, label="unset"))
    
segments.apply(createAnnotation_pairInput, axis=1)

cas.sofa_string = sofa
cas.sofa_mime = "text/plain"

# Save changes to cas
for relation in relations:
    cas.add_annotation(relation)<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>cas.to_xmi(output_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1317953861584754, 'fraction_numerical': 0.0324974924774323, 'mean_word_length': 4.80442374854482, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '330291', 'license': 'MIT', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1772, 'n_words': 334, 'path': 'notebooks/import.ipynb', 'repo_name': 'DFKI-NLP/semantic-storytelling', 'size': '4985'}