<jupyter_start><jupyter_text>enriched (main) Persons with Places* write Persons (main) to .txt file* use this file to annotate LOC within prodigy* train a LOC-model* run it and annotate Persons<jupyter_code>import ast
import random
from spacytei.train import batch_train
persons = Person.objects.filter(is_main_person__isnull=False)
persons.count()
items = [x.written_name for x in persons]
items = sorted(iter(items), key=lambda k: random.random())
filename = "person__main.txt"
with open(filename, 'w', encoding="utf-8") as f:
    for x in items:
        f.write("{}".format(x) + '\n')
file = r"C:\Users\pandorfer\Documents\Redmine\prodigy\invs\invs_loc.jsonl"
with open(file) as f:
    TRAIN_DATA = f.readlines()
train_data = [ast.literal_eval(x) for x in TRAIN_DATA]
print(len(train_data))
train_data[4]
batch_train(train_data=train_data, output_dir='./data/main_loc', new_label='LOC', eval_split=0.2, n_iter=8)
import spacy
nlp = spacy.load(r"./data/main_loc")
for x in persons:
    doc = nlp("{}".format(x.written_name))
    for ent in doc.ents:
        if ent.label_ == "LOC":
            place, _ = Place.objects.get_or_create(
                name="{}".format(ent.text)
            )
            x.belongs_to_place.add(place)
            break
    x.save()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13590733590733592, 'fraction_numerical': 0.003861003861003861, 'mean_word_length': 3.946564885496183, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '667247', 'license': 'MIT', 'n_tokens_mistral': 476, 'n_tokens_neox': 464, 'n_words': 106, 'path': 'add_places.ipynb', 'repo_name': 'reading-in-the-alps/rita-invs', 'size': '1295'}