<jupyter_start><jupyter_text>Install dependencies<jupyter_code>!pip install datasets<jupyter_output>Collecting datasets
[?25l  Downloading https://files.pythonhosted.org/packages/1a/38/0c24dce24767386123d528d27109024220db0e7a04467b658d587695241a/datasets-1.1.3-py3-none-any.whl (153kB)
[K     |████████████████████████████████| 163kB 5.5MB/s 
[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.6/dist-packages (from datasets) (2.23.0)
Requirement already satisfied: dataclasses; python_version < "3.7" in /usr/local/lib/python3.6/dist-packages (from datasets) (0.8)
Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from datasets) (1.1.4)
Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from datasets) (0.3.3)
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from datasets) (1.18.5)
Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.6/dist-packages (from datasets) (4.41.1)
Collecting pyarrow>=0.17.1
[?25l  Downloading https://files.pythonhos[...]<jupyter_text>Python script for preprocessing the original swag + cosmos_qa datasets. Run this to make the functions available.<jupyter_code># -*- coding: utf-8 -*-
"""
Created on Mon Nov 23 19:18:39 2020

@author: nickg
"""
import json
import pandas as pd
from tqdm import tqdm
from datasets import load_dataset
    
# Write a single json'd data entry to file
def write_jsonl_entry(entry, jsonl_file):
    json.dump(entry, jsonl_file)
    jsonl_file.write('\n')
    
# A two-for-one data formatter for both swag and hellaswag datasets.
# Splits for swag: train, val
# Splits for hellaswag: train, validation
def swag2quail(split, prefix=""):
    is_hella = prefix=="hella"
    itr_container = pd.read_csv("swag/{}.csv".format(split)).sample(frac=1).reset_index(drop=True).iterrows() \
                                              if not is_hella else enumerate(load_dataset('hellaswag')[split].shuffle()) 
    elem_indices = ['source_id', 'ctx_a', 'ctx_b', 'label'] if is_hella else ['fold-ind', 'sent1', 'sent2', 'label']
    ending_funct = lambda e, k : e['endings'][k] if is_hella else e['ending{}'.format(k)]
    
    with open("swag/{}swag_{}.jsonl".format(prefix, split), mode='w', encoding='utf-8') as f:
        for id_, swag_entry in tqdm(itr_container):
            quail_entry = {"id": str(swag_entry[elem_indices[0]]),
                           "context": swag_entry[elem_indices[1]],
                           "question": swag_entry[elem_indices[2]],
                           "question_type": 'Subsequent_state',
                           "answers": [ending_funct(swag_entry,k) for k in range(4)],
                           "correct_answer_id": str(swag_entry[elem_indices[-1]]) }
            write_jsonl_entry(quail_entry, f)
         
# Convert cosmos_qa to quail format. Questions for which the correct answer contains
# "None of the above" are unanswerable questions in this dataset.
# Splits: train, validation
def cosmos2quail(split):
    is_unanswerable = lambda e : "None of the above" in e["answer{}".format(e['label'])]
    cosmos = load_dataset('cosmos_qa')[split].shuffle()

    with open("cosmos_qa/cosmos_qa_{}.jsonl".format(split), mode='w', encoding='utf-8') as f:
        for cosmos_entry in tqdm(cosmos):
            quail_entry = {"id": str(cosmos_entry['id']),
                           "context": cosmos_entry['context'],
                           "question": cosmos_entry['question'],
                           "question_type": 'Unanswerable' if is_unanswerable(cosmos_entry) else 'Causality',
                           "answers": [cosmos_entry['answer{}'.format(k)] for k in range(4)],
                           "correct_answer_id": cosmos_entry['label'] }
            write_jsonl_entry(quail_entry, f)
    
def process_dataset(dataset, split):
    print("\nProcessing {}[{}]...".format(dataset, split))
    if 'swag' in dataset:
        swag2quail(split, dataset.split('swag')[0])
    elif 'cosmos_qa' in dataset:
        cosmos2quail(split)
    else:
        print("Unknown dataset: {}".format(dataset))<jupyter_output><empty_output><jupyter_text>Get quail formatted cosmos_qa datasets:<jupyter_code>!mkdir cosmos_qa
process_dataset(dataset='cosmos_qa', split='train')
process_dataset(dataset='cosmos_qa', split='validation')<jupyter_output>mkdir: cannot create directory ‘cosmos_qa’: File exists

Processing cosmos_qa[train]...<jupyter_text>Download SWAG dataset<jupyter_code>!mkdir swag
!wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/test.csv -O swag/test.csv
!wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/train.csv -O swag/train.csv
!wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/val.csv -O swag/val.csv<jupyter_output>mkdir: cannot create directory ‘swag’: File exists
--2020-11-24 06:43:38--  https://raw.githubusercontent.com/rowanz/swagaf/master/data/test.csv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 192.168.3.11, 192.168.3.11, 172.16.31.10, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|192.168.3.11|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 7817885 (7.5M) [text/plain]
Saving to: ‘swag/test.csv’

swag/test.csv       100%[===================>]   7.46M  28.8MB/s    in 0.3s    

2020-11-24 06:43:39 (28.8 MB/s) - ‘swag/test.csv’ saved [7817885/7817885]

--2020-11-24 06:43:39--  https://raw.githubusercontent.com/rowanz/swagaf/master/data/train.csv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 192.168.3.11, 192.168.3.11, 172.16.31.10, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|192.168.3.11|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length[...]<jupyter_text>Get quail formatted swag and hellaswag datasets:<jupyter_code># Swag - not available in huggingface/datasets. CSV files downloaded from github into ./swag
data = process_dataset(dataset='swag', split='train')
process_dataset(dataset='swag', split='val')
# Hellaswag is available directly from huggingface/datasets. Ref: https://rowanzellers.com/hellaswag/
process_dataset(dataset='hellaswag', split='train')
process_dataset(dataset='hellaswag', split='validation')
import json
import glob
import random

train = []
n_final_train_samples = 20000

with open('cosmos_qa/cosmos_qa_train.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      train.append(line)

with open('swag/hellaswag_train.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      train.append(line)

with open('swag/swag_train.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      train.append(line)

random.shuffle(train)
with open('train.jsonl','w') as w:
  for itr, entry in enumerate(train):
    if itr >= n_final_train_samples:
      break
    entry = json.loads(entry)
    entry['correct_answer_id'] = str(entry['correct_answer_id'])
    w.write(json.dumps(entry))
    w.write('\n')

val = []
n_final_val_samples = 10000

with open('cosmos_qa/cosmos_qa_validation.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      val.append(line)

with open('swag/hellaswag_validation.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      val.append(line)

with open('swag/swag_val.jsonl','r') as f:
    for line in f:
      line = line.strip('\n')
      val.append(line)

random.shuffle(val)
with open('val.jsonl','w') as w:
  for itr, entry in enumerate(val):
    if itr >= n_final_val_samples:
      break
    entry = json.loads(entry)
    entry['correct_answer_id'] = str(entry['correct_answer_id'])
    w.write(json.dumps(entry))
    w.write('\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1509967845659164, 'fraction_numerical': 0.041414790996784565, 'mean_word_length': 3.7414634146341466, 'pattern_counts': {'":': 12, '<': 20, '<?xml version=': 0, '>': 25, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '42759', 'license': 'Apache-2.0', 'n_tokens_mistral': 2908, 'n_tokens_neox': 2700, 'n_words': 559, 'path': 'notebooks/cse576/CSE576_data_preprocessing.ipynb', 'repo_name': 'ngdodd/transformers', 'size': '7789'}