<jupyter_start><jupyter_text>Create dataset in SQUAD format> Create json files for question answering<jupyter_code>#export
from fastai.core import *<jupyter_output><empty_output><jupyter_text>Here is sentiment extraction dataset with `text`, `selected_text` and `sentiment` given. This problem can also be formulated as a question answering problem. We wil consider `text` as `context`, `sentiment` as `question`, and `selected_text` as answer.<jupyter_code>data_path = Path("../data/")
train_df = pd.read_csv(data_path/'train.csv')
train_df = train_df.dropna().reset_index(drop=True)
test_df = pd.read_csv(data_path/'test.csv')
train_df.head(2)
test_df.head(2)<jupyter_output><empty_output><jupyter_text>The following is how a [SQUAD](https://rajpurkar.github.io/SQuAD-explorer/) data sample looks like.<jupyter_code>#hide
squad_sample = {
    "version": "v2.0",
    "data": [
        {
            "title": "Beyonc\u00e9",
            "paragraphs": [
                {
                    "qas": [
                        {
                            "question": "When did Beyonce start becoming popular?",
                            "id": "56be85543aeaaa14008c9063",
                            "answers": [
                                {
                                    "text": "in the late 1990s",
                                    "answer_start": 269
                                }
                            ],
                            "is_impossible": False
                        }
                    ],
                    "context": "Beyonc\u00e9 <NAME>-Carter (/bi\u02d0\u02c8j\u0252nse\u026a/ bee-YON-say) (born September 4, 1981) is an American singer, songwriter, record producer and actress. Born and raised in Houston, Texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny's Child. Managed by her father, Mathew Knowles, the group became one of the world's best-selling girl groups of all time. Their hiatus saw the release of Beyonc\u00e9's debut album, Dangerously in Love (2003), which established her as a solo artist worldwide, earned five Grammy Awards and featured the Billboard Hot 100 number-one singles \"Crazy in Love\" and \"Baby Boy\"."
                }
            ]
        }
    ]
}
squad_sample
row = train_df.iloc[np.random.choice(range(len(train_df)))]
text_id = row['textID']
context = row['text']
answer = row['selected_text']
question = row['sentiment']
text_id, context, answer, question
#export
def get_answer_start(context, answer):
    len_a = len(answer)
    for i, _ in enumerate(context):
        if context[i:i+len_a] == answer: return i
    raise Exception("No overlapping segment found")
# test
start_idx = get_answer_start(context, answer)
assert context[start_idx:start_idx+len(answer)] == answer
#export
def generate_qas_dict(text_id, context, answer, question, impossible=False):
    qas_dict = {}
    qas_dict['question'] = question
    qas_dict['id'] = text_id
    qas_dict['is_impossible'] = impossible
    
    if answer is None: qas_dict['answers'] = []
    else: 
        answer_start = get_answer_start(context, answer)
        qas_dict['answers'] = [{"text":answer, "answer_start":answer_start}]
    return qas_dict
generate_qas_dict(text_id, context, answer, question)
#export
def create_squad_from_df(df):
    data_dicts = []
    for _, row in df.iterrows():
        text_id = row['textID']
        context = row['text']
        answer =  row['selected_text'] if 'selected_text' in row else None
        question = row['sentiment']

        qas_dict = generate_qas_dict(text_id, context, answer, question)
        data_dict = {"paragraphs" : [{"qas" : [qas_dict], "context":context}]}
        data_dict['title'] = text_id
        data_dicts.append(data_dict)
    return {"version": "v2.0", "data": data_dicts}
train_squad_dict = create_squad_from_df(train_df)
os.makedirs("../squad_data", exist_ok=True)
#export
def save_dict_as_json(d, fname):
    with open(fname, "w") as f: f.write(json.dumps(d))
save_dict_as_json(train_squad_dict, "../squad_data/train_squad_data.json")
#export
def read_json_as_dict(fname):
    return json.loads(open(fname).read())
train_squad_dict = read_json_as_dict("../squad_data/train_squad_data.json")
np.random.choice(train_squad_dict['data'])<jupyter_output><empty_output><jupyter_text>Create KFold Stratified Data<jupyter_code>from sklearn.model_selection import StratifiedKFold
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=42)
SQUAD_DATA_PATH = Path("../squad_data/")
SQUAD_DATA_PATH.ls()
for foldnum,(trn_idx, val_idx) in enumerate(skf.split(train_df, train_df.sentiment.values)):
    trn_fold_df = train_df.iloc[trn_idx]
    val_fold_df = train_df.iloc[val_idx]
    train_squad_dict = create_squad_from_df(trn_fold_df)
    valid_squad_dict = create_squad_from_df(val_fold_df)
    save_dict_as_json(train_squad_dict, SQUAD_DATA_PATH/f"train_squad_data_{foldnum}.json")
    save_dict_as_json(valid_squad_dict, SQUAD_DATA_PATH/f"valid_squad_data_{foldnum}.json")
len(train_squad_dict['data']), len(valid_squad_dict['data'])
test_squad_dict =  create_squad_from_df(test_df)
save_dict_as_json(test_squad_dict, SQUAD_DATA_PATH/f"test_squad_data.json")<jupyter_output><empty_output><jupyter_text>Check json files<jupyter_code>i = 0
val_ids = []
for i in range(5):
    val_dict = read_json_as_dict(SQUAD_DATA_PATH/f"valid_squad_data_{i}.json")
    val_ids.append(set(map(lambda o:o['paragraphs'][0]['qas'][0]['id'], val_dict['data'])))
for ids in val_ids: print(len(ids))
# all validation ids are unique across files
u = set.intersection(*val_ids)
assert len(u) == 0<jupyter_output><empty_output><jupyter_text>export<jupyter_code>from nbdev.export import notebook2script
notebook2script()<jupyter_output>Converted 00_core.ipynb.
Converted 01-squad-utils.ipynb.
{'fraction_non_alphanumeric': 0.11760707002039429, 'fraction_numerical': 0.013766145479265806, 'mean_word_length': 3.6817820206841687, 'pattern_counts': {'":': 17, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '355781', 'license': 'Apache-2.0', 'n_tokens_mistral': 2071, 'n_tokens_neox': 1987, 'n_words': 413, 'path': 'nbs/01-squad-utils.ipynb', 'repo_name': 'KeremTurgutlu/tse', 'size': '5893'}