<jupyter_start><jupyter_text>Food Disease dataset<jupyter_code>df_train = pd.read_csv("/Users/adamkovacs/data/food-disease-dataset/splits/cause_folds/fold0/train.csv", sep=",", quotechar='"')
df_dev = pd.read_csv("/Users/adamkovacs/data/food-disease-dataset/splits/cause_folds/fold0/val.csv", sep=",", quotechar='"')
import re

def extract_entities(df):
    sen = re.sub(re.escape(df.term1), 'XXX', df.sentence, flags=re.IGNORECASE)
    sen = re.sub(re.escape(df.term2), 'YYY', sen, flags=re.IGNORECASE)
    return sen.encode('ascii', errors='ignore').decode('utf-8')
    

df_train['preprocessed_sen'] = df_train.apply(extract_entities, axis=1)
df_train['treat_label'] = df_train.is_treat.replace({1: 'TREAT', 0: 'NOT'})
df_train['cause_label'] = df_train.is_cause.replace({1: 'CAUSE', 0: 'NOT'})

df_dev['preprocessed_sen'] = df_dev.apply(extract_entities, axis=1)
df_dev['treat_label'] = df_dev.is_treat.replace({1: 'TREAT', 0: 'NOT'})
df_dev['cause_label'] = df_dev.is_cause.replace({1: 'CAUSE', 0: 'NOT'})
from xpotato.dataset.dataset import Dataset
from xpotato.models.trainer import GraphTrainer<jupyter_output><empty_output><jupyter_text>Detecting treat<jupyter_code>train_rows = df_train.iterrows()
dev_rows = df_dev.iterrows()
train_sentences = [(row[1].preprocessed_sen, row[1].treat_label) for row in train_rows]
dev_sentences = [(row[1].preprocessed_sen, row[1].treat_label) for row in dev_rows]

train_dataset = Dataset(train_sentences, label_vocab={"TREAT":1, "NOT": 0}, lang='en_bio')
train_dataset.set_graphs(train_dataset.parse_graphs(graph_format="ud"))

dev_dataset = Dataset(dev_sentences, label_vocab={"TREAT":1, "NOT": 0}, lang='en_bio')
dev_dataset.set_graphs(dev_dataset.parse_graphs(graph_format="ud"))
train_df = train_dataset.to_dataframe()
dev_df = dev_dataset.to_dataframe()
from xpotato.dataset.utils import save_dataframe

save_dataframe(train_df, 'food_train_dataset_treat_ud.tsv')
save_dataframe(dev_df, 'food_dev_dataset_treat_ud.tsv')<jupyter_output><empty_output><jupyter_text>Detecting cause<jupyter_code>train_rows = df_train.iterrows()
dev_rows = df_dev.iterrows()

train_sentences = [(row[1].preprocessed_sen, row[1].cause_label) for row in train_rows]
dev_sentences = [(row[1].preprocessed_sen, row[1].cause_label) for row in dev_rows]

train_dataset_cause = Dataset(train_sentences, label_vocab={"CAUSE":1, "NOT": 0})
train_dataset_cause.set_graphs(train_dataset.graphs)

dev_dataset_cause = Dataset(dev_sentences, label_vocab={"CAUSE":1, "NOT": 0})
dev_dataset_cause.set_graphs(dev_dataset.graphs)
train_df = train_dataset.to_dataframe()
dev_df = dev_dataset.to_dataframe()
save_dataframe(train_df, 'food_train_dataset_cause_fourlang.tsv')
save_dataframe(dev_df, 'food_dev_dataset_cause_fourlang.tsv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15171171171171172, 'fraction_numerical': 0.011171171171171172, 'mean_word_length': 9.094545454545454, 'pattern_counts': {'":': 8, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '97538', 'license': 'MIT', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1068, 'n_words': 143, 'path': 'features/food/food.ipynb', 'repo_name': 'GKingA/POTATO', 'size': '2775'}