<jupyter_start><jupyter_text>Language Model Loading<jupyter_code>from french_crs.text2corefchains import udpipe_spacy_lang_model, mentions2chains
model=udpipe_spacy_lang_model("../external language models/french-ud-2.0-170801.udpipe","fr_core_news_md")<jupyter_output><empty_output><jupyter_text>Text Parsing<jupyter_code>model.tokenize("à mettre en œuvre de le protectionnisme intelligent à mettre en avant de le patriotisme économique pour donner un avantage à les entreprises françaises dans la commande publique voilà tout cela . Le patriotisme économique qui n ’ a jamais été mis en œuvre le protectionnisme intelligent la défiscalisation de les heures supplémentaires la suppression de le travail détaché la baisse de les charges mais exclusivement pour les TPE PME . Il met en place un patriotisme économique un protectionnisme intelligent il dit à les constructeurs américains si vous voulez aller faire vos voitures à l ’ étranger construire une voiture à l’ étranger alors vous paierez une taxe en les réimportant à les Etats-Unis . D ’ autant que évidemment ce que fait Trump m ’ intéresse et pour cause puisqu ’ il met en place la politique que j’ appelle de mes vœux depuis très longtemps et notamment la politique de patriotisme économique de protectionnisme intelligent")    
model.parse_sentences_json()
model.update_json_sents_with_spacy_entities(operation="in")
model.update_json_sents_with_mentions(operation="==", relative_path_mention_detector="..")<jupyter_output><empty_output><jupyter_text>Pairs & Features Building<jupyter_code>chains_generator=mentions2chains(model.mentions_ids_in_json_sents)
chains_generator.generate_mention_pairs(window_size=30, iteration=3)
chains_generator.generate_json_mention_pairs(model.mentions_ids_in_json_sents,model.sentences_json)
chains_generator.json_mention_pairs2dataframe(save_file=True, file_path="./demo_output/unseen_data.xlsx")<jupyter_output><empty_output><jupyter_text>Coreference Prediction<jupyter_code>chains_generator.json_mention_pairs2dataframe(save_file=True, file_path="./demo_output/unseen_data.xlsx")
chains_generator.init_data_model(model_name="../pre-trained language models/Model_ANCOR_Representative_FULL.model", 
                                 input_method=".xlsx",
                                 file_path="./demo_output/unseen_data.xlsx",
                                 column_outcome="Prediction",
                                 threshold=0.5)

chains_generator.columns_drop_list = ["Left_ID", "Right_ID"]

chains_generator.apply_model_to_dataset()<jupyter_output><empty_output><jupyter_text>Coreference Chains Building<jupyter_code>from french_crs.pairs2chains import chains_builder

chains = chains_builder("./demo_output/unseen_data.xlsx",
                        "./demo_output/unseen_data.xlsx",
                        "Prediction",
                        "Prediction",
                        "./demo_output/",
                        0.4)

chains.generate_gold_model_json_output(mode="test")
chains.coref_chains_pred_json<jupyter_output>SCORCH json formats saved at following addresses:
/Users/mehdi.mirzapour/French-CRS/demo/demo_output/coref_chains_gold.json
/Users/mehdi.mirzapour/French-CRS/demo/demo_output/coref_chains_pred.json<jupyter_text>Coreference Chains Visualizer<jupyter_code>chains.context_visualizer(model.sentences_json,
                          context_left_window=3,
                          context_right_window=3,
                          theme_left_window=5,
                          theme_right_window=4,
                          file_path="./demo_output/context_visualized.xlsx",
                          path_to_stop_words="../french_crs/StopWords-FR.xlsx")<jupyter_output><empty_output><jupyter_text>Experiment I : Running through 12 models<jupyter_code>import glob
from french_crs.pairs2chains import chains_builder

threshold=0.5


PATH="../pre-trained language models/"

files_list=[file[file.rfind("/")+1:] for file in glob.glob(PATH+"*.model")]

files_list=['Model_ANCOR_balanced_DIRECTE.model',
 'Model_ANCOR_window_INDIRECTE.model',
 'Model_ANCOR_window_DIRECTE.model',
 'Model_ANCOR_representative_ANAPHORE.model',
 'Model_ANCOR_balanced_Full_Soon.model',
 'Model_ANCOR_Balanced_FULL.model',
 'Model_ANCOR_balanced_INDIRECTE.model',
 'Model_ANCOR_window_ANAPHORE.model',
 'Model_ANCOR_representative_INDIRECTE.model',
 'Model_ANCOR_Balanced_ANAPHORE.model',
 'Model_ANCOR_representative_Full.model',
 'Model_ANCOR_representative_DIRECTE.model',
 'Model_ANCOR_window_Full.model']

for file in files_list:
    
    chains_generator=mentions2chains(model.mentions_ids_in_json_sents)
    chains_generator.generate_mention_pairs(window_size=30, iteration=3)
    chains_generator.generate_json_mention_pairs(model.mentions_ids_in_json_sents,model.sentences_json)
    chains_generator.json_mention_pairs2dataframe(save_file=True, file_path="./unseen_data.xlsx")
    
    chains_generator.init_data_model(model_name="../pre-trained language models/"+file, 
                                 input_method=".xlsx",
                                 file_path="./unseen_data.xlsx",
                                 column_outcome="Prediction",
                                 threshold=threshold)
    
    chains_generator.columns_drop_list = ["Left_ID", "Right_ID"]


    chains_generator.apply_model_to_dataset()
    

    chains = chains_builder("./demo_output/unseen_data.xlsx",
                            "./demo_output/unseen_data.xlsx",
                            "Prediction",
                            "Prediction",
                            "./demo_output/",
                            0.4)

    chains.generate_gold_model_json_output(mode="test")
    
    print("---------------------------------------------------------------")
    print("Model: "+file)
    for chain in chains.coref_chains_pred_json:
        print(chain)
        print(chains.coref_chains_pred_json[chain])
    print("---------------------------------------------------------------")<jupyter_output>SCORCH json formats saved at following addresses:
/Users/mehdi.mirzapour/French-CRS/demo/demo_output/coref_chains_gold.json
/Users/mehdi.mirzapour/French-CRS/demo/demo_output/coref_chains_pred.json
---------------------------------------------------------------
Model: Model_ANCOR_balanced_DIRECTE.model
0
[['patriotisme économique', 4, [38, 39]], ['place', 3, [4]], ['le protectionnisme intelligent', 2, [13, 14, 15]], ['Le patriotisme économique', 2, [1, 2, 3]], ['un patriotisme économique', 3, [5, 6, 7]], ['un protectionnisme intelligent', 3, [8, 9, 10]], ['protectionnisme intelligent', 4, [41, 42]], ['un avantage', 1, [19, 20]], ['le patriotisme économique', 1, [14, 15, 16]], ['place', 4, [21]], ['le protectionnisme intelligent', 1, [6, 7, 8]]]
1
[['il', 3, [11]], ['Il', 3, [1]], ['il', 4, [18]]]
2
[['les entreprises françaises', 1, [22, 23, 24]], ['les constructeurs américains', 3, [14, 15, 16]]]
3
[['vous', 3, [18]], ['vous', 3, [35]]]
4
[['une taxe', 3, [37, 38]], ['une voiture', 3,[...]<jupyter_text>Experiment II : Running through different thresholds<jupyter_code>from french_crs.pairs2chains import chains_builder
threshold=1.05

for iterate in range(0,20):
    threshold-=0.05
    chains = chains_builder("./unseen_data.xlsx",
                            "./unseen_data.xlsx",
                            "Prediction",
                            "Prediction",
                            "./demo_output/",
                            threshold)
    coreference_chain_built=chains.generate_coreference_chains(chains.gold_dataframe, 
                                                               chains.gold_column, 
                                                               mode="test")
    
    print("---------------------------------------------------------------")
    print("Threshold :"+str(round(threshold,2)))
    for chain in coreference_chain_built:
        print(chain)
        print(coreference_chain_built[chain])
    print("---------------------------------------------------------------")<jupyter_output>---------------------------------------------------------------
Threshold :1.0
---------------------------------------------------------------
---------------------------------------------------------------
Threshold :0.95
0
[['Le patriotisme économique', 2, [1, 2, 3]], ['le patriotisme économique', 1, [14, 15, 16]]]
1
[['le protectionnisme intelligent', 2, [13, 14, 15]], ['le protectionnisme intelligent', 1, [6, 7, 8]]]
2
[['il', 3, [11]], ['Il', 3, [1]], ['il', 4, [18]]]
3
[['que', 4, [7]], ['que', 4, [24]]]
4
[['la politique', 4, [35, 36]], ['la politique', 4, [22, 23]]]
---------------------------------------------------------------
---------------------------------------------------------------
Threshold :0.9
0
[['Le patriotisme économique', 2, [1, 2, 3]], ['le patriotisme économique', 1, [14, 15, 16]]]
1
[['le protectionnisme intelligent', 2, [13, 14, 15]], ['le protectionnisme intelligent', 1, [6, 7, 8]]]
2
[['il', 3, [11]], ['Il', 3, [1]], ['il', 4, [18]]]
3
[['vous', 3, [18]],[...]
{'fraction_non_alphanumeric': 0.18154761904761904, 'fraction_numerical': 0.02568342151675485, 'mean_word_length': 2.607554671968191, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '519122', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2955, 'n_tokens_neox': 2728, 'n_words': 542, 'path': 'demo/Text2CoreferenceChains_UDPipe.ipynb', 'repo_name': 'mehdi-mirzapour/TALAD-CRS', 'size': '9072'}