<jupyter_start><jupyter_text>Resolving issuesThe original preco files contain lots of unicode quotation marks, as opposed to the standardized "". This results in difficulties parsing, and the original dataset needs to be transformed, as well as updating the mention indices.<jupyter_code>from unidecode import unidecode
weird = set()
gum, lit, onto, prec = datasets

dataset=prec

with open(dataset, "r", encoding="utf8") as data:
    all_docs = data.readlines()
    for doc in all_docs:
        doc = json.loads(doc)
        tkns = doc["tokens"]
        for tok in tkns:
            tok = unidecode(tok)
            if len(tok) == 1 and not tok.isalnum():
                weird.add(tok)
weird<jupyter_output><empty_output><jupyter_text>Lemmas and shit<jupyter_code># retokenize the pre-tokenized input
joined = ' '.join(tokens)
doc = nlp(joined, disable=disabled)
doc
spacytokens = [t.text for t in doc]
for sp, norm in zip(spacytokens, tokens):
    print(sp, "\t", norm)
lemmed_spacy = [t if t.lemma_ == "-PRON-" else t.lemma_ for t in doc]
lemmed_norm = [t if t.lemma_ == "-PRON-" else t.lemma_ for t in doc]
for orig, lem in zip(doc, lemmed):
    print(orig.text, "\t", lem)
_tokens = tokens.copy()
_clusters = clusters.copy()
DISABLED = ["tagger", "parser", "ner", "textcat"]
def alter_cluster_mentions_by_index(index, direction):
    for c_idx in range(len(_clusters)):
        for m_idx in range(len(_clusters[c_idx])):
            try:
                m1, m2 = _clusters[c_idx][m_idx]
            except Exception:
                print(_clusters[c_idx][m_idx])
            if m1 > index:
                m1 += direction * 1
            if m2 > index:
                m2 += direction * 1
            _clusters[c_idx][m_idx] = [m1, m2]

def remove_invalid_cluster_tokens():
    # start by creating a spacy tokenization
    spacydoc = nlp(' '.join(_tokens), disable=DISABLED)
    spacytoks = [t.text for t in spacydoc]

    latex_quotations = ["``"]
    parsed_tokens = []
    
    for index, token in enumerate(_tokens):
        if token == " ":
            alter_cluster_mentions_by_index(index, -1)

        else:
            # fix latex quotations.
            if token in latex_quotations:
                token = "''"
            elif token != spacytoks[index]:
                # e.g. 've becomes ' and ve, as separate tokens.
                # fix this by expanding index
                #alter_cluster_mentions_by_index(index, 1)
                pass
            parsed_tokens.append(token)
    print(parsed_tokens)
remove_invalid_cluster_tokens()
viz.raw_render(_tokens, _clusters)
print_clusters(tokens, clusters)
viz.raw_render(tokens, clusters)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10809806835066865, 'fraction_numerical': 0.005200594353640416, 'mean_word_length': 2.6941015089163236, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '445546', 'license': 'MIT', 'n_tokens_mistral': 881, 'n_tokens_neox': 861, 'n_words': 243, 'path': 'DatasetVerification_Preco.ipynb', 'repo_name': 'ph10m/CorefLite', 'size': '2692'}