<jupyter_start><jupyter_text>CONDA<jupyter_code># download the CONDA dataset
!mkdir datasets
!mkdir datasets/conda
!wget -P datasets/conda/ -c 'https://raw.githubusercontent.com/usydnlp/CONDA/main/data/CONDA_train.csv'
!wget -P datasets/conda/ -c 'https://raw.githubusercontent.com/usydnlp/CONDA/main/data/CONDA_valid.csv'
!wget -P datasets/conda/ -c 'https://raw.githubusercontent.com/usydnlp/CONDA/main/data/CONDA_test.csv'
import pandas as pd

# import the dataset into pandas dataframe
conda_train = pd.read_csv('datasets/conda/CONDA_train.csv')
conda_val = pd.read_csv('datasets/conda/CONDA_valid.csv')
conda_test = pd.read_csv('datasets/conda/CONDA_test.csv')
conda_train.head(6)
# check for any null items
print("train set:")
print(conda_train.isnull().sum())
print("\nval set:")
print(conda_val.isnull().sum())
print("\ntest set:")
print(conda_test.isnull().sum())
# remove rows with nan values
conda_train.dropna(inplace=True)
conda_val.dropna(inplace=True)
conda_test.dropna(inplace=True)
# create a function to convert the conda dataset into jointBERT input
def create_input_file(data, column, file_name):
    # create a list of the slot text and tokens
    texts = list(data[column])

    # create a list of slot labels
    slot_label = ['(C)','(D)','(O)','(P)','(S)','(T)','(SEPA)',
                  '(C),','(D),','(O),','(P),','(S),','(T),','(SEPA),']

    # extract the text from the slot text and tokens
    all_new_text = []
    for text in texts:
      try: split_text = text.split()
      except: print(text) 
      new_text = []
      for stext in split_text:
        if stext not in slot_label:
          new_text.append(stext)
      join_new_text = " ".join(new_text)
      all_new_text.append(join_new_text)

    # write the text into seq.in file for text input to jointBERT model
    with open(file_name, 'w') as f:
        for item in all_new_text:
            f.write("%s\n" % item)
# create input files for the train set
!mkdir datasets/conda/train
create_input_file(conda_train, 'slotTokens', 'datasets/conda/train/seq.in')
create_input_file(conda_train, 'slotClasses', 'datasets/conda/train/seq.out')
create_input_file(conda_train, 'intentClass', 'datasets/conda/train/label')
# create input files for the val set
!mkdir datasets/conda/val
create_input_file(conda_val, 'slotTokens', 'datasets/conda/val/seq.in')
create_input_file(conda_val, 'slotClasses', 'datasets/conda/val/seq.out')
create_input_file(conda_val, 'intentClass', 'datasets/conda/val/label')
# create input files for the test set (there is no slot/intent labels for the test set)
!mkdir datasets/conda/test
create_input_file(conda_test, 'utterance', 'datasets/conda/test/sample_pred_in.txt')
# create a function to convert the conda dataset into jointBERT input
def create_input_file2(data, column):
    # create a list of the slot text and tokens
    texts = list(data[column])

    # create a list of slot labels
    slot_label = ['(C)','(D)','(O)','(P)','(S)','(T)','(SEPA)',
                  '(C),','(D),','(O),','(P),','(S),','(T),','(SEPA),']

    # extract the text from the slot text and tokens
    all_new_text = []
    for text in texts:
      try: split_text = text.split()
      except: print(text) 
      new_text = []
      for stext in split_text:
        if stext not in slot_label:
          new_text.append(stext)
      join_new_text = " ".join(new_text)
      all_new_text.append(join_new_text)
    
    return all_new_text
train_text = create_input_file2(conda_train, 'slotTokens')
train_text = ['BOS ' + text + ' EOS\t' for text in train_text]
train_text[:5]
train_slot = create_input_file2(conda_train, 'slotClasses')
train_slot[:5]
train_label = create_input_file2(conda_train, 'intentClass')
train_label[:5]
result = list(zip(train_text, train_slot))
result = ["".join(text) for text in result]
result[:5]
result_w_intent = list(zip(result, train_label))
result_w_intent = [" ".join(text) for text in result_w_intent]
result_w_intent[:5]
with open('datasets/conda/conda.train.w-intent.iob', 'w') as f:
  for item in result_w_intent:
    f.write("%s\n" % item)

with open('datasets/conda/conda.train.iob', 'w') as f:
  for item in result:
    f.write("%s\n" % item)
val_text = create_input_file2(conda_val, 'slotTokens')
val_text = ['BOS ' + text + ' EOS\t' for text in val_text]
val_slot = create_input_file2(conda_val, 'slotClasses')
val_label = create_input_file2(conda_val, 'intentClass')

result = list(zip(val_text, val_slot))
result = ["".join(text) for text in result]
result_w_intent = list(zip(result, val_label))
result_w_intent = [" ".join(text) for text in result_w_intent]

with open('datasets/conda/conda.dev.w-intent.iob', 'w') as f:
  for item in result_w_intent:
    f.write("%s\n" % item)

with open('datasets/conda/conda.dev.iob', 'w') as f:
  for item in result:
    f.write("%s\n" % item)<jupyter_output><empty_output><jupyter_text>LOW<jupyter_code># Code to download file into Colaboratory:
!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# download the low dataset
!mkdir datasets/low

# train
id = '<KEY>'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/train.csv')

# train label
id = '<KEY>'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/train_label.csv')

# val
id = '<KEY>'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/val.csv')

# val label
id = '<KEY>'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/val_label.csv')

# test
id = '<KEY>JRErw0BmzQ_OLWPTcbCczFMo'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/test.csv')

# test label
id = '1-40KJhMjrv7Yxkk-A6NYEMkbPl9sjv_4'
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('datasets/low/test_label.csv')
import pandas as pd

# import the dataset into pandas dataframe
train_text = pd.read_csv('datasets/low/train.csv', index_col=0)
train_label = pd.read_csv('datasets/low/train_label.csv', index_col=0)
val_text = pd.read_csv('datasets/low/val.csv', index_col=0)
val_label = pd.read_csv('datasets/low/val_label.csv', index_col=0)
test_text = pd.read_csv('datasets/low/test.csv', index_col=0)
test_label = pd.read_csv('datasets/low/test_label.csv', index_col=0)
train_text.head()
train_label.head()
print(train_text.shape)
print(train_label.shape)
# concat the text and the label files
train_data = pd.concat([train_text,train_label],axis=1,join='inner')
val_data = pd.concat([val_text,val_label],axis=1,join='inner')
test_data = pd.concat([test_text,test_label],axis=1,join='inner')

# drop the 'Unnamed: 12' column
train_data.drop(['Unnamed: 12'], axis=1, inplace=True)
val_data.drop(['Unnamed: 12'], axis=1, inplace=True)
test_data.drop(['Unnamed: 12'], axis=1, inplace=True)
print(train_data.shape)
print(val_data.shape)
print(test_data.shape)
train_data.head()
# check for any null items
print("train set:")
print(train_data.isnull().sum())
print("\nval set:")
print(val_data.isnull().sum())
print("\ntest set:")
print(test_data.isnull().sum())
# remove rows with nan values
train_data.dropna(inplace=True)
val_data.dropna(inplace=True)
print("train set:")
print(train_data.isnull().sum())
print("\nval set:")
print(val_data.isnull().sum())
# delete texts that contain '#ERROR!'
train_data.drop(train_data['1'][train_data['1']=='#ERROR!'].index, inplace=True)
val_data.drop(val_data['1'][val_data['1']=='#ERROR!'].index, inplace=True)
test_data.drop(test_data['1'][test_data['1']=='#ERROR!'].index, inplace=True)
# check the intent labels
print(set(train_data['Label']))
print(set(val_data['Label']))
print(set(test_data['Label']))
# remove the val data with 'T' intent label
val_data.drop(val_data['Label'][val_data['Label']=='T'].index, inplace=True)
# 'I' and 'I ' intent labels in train set should be the same as 'I'
labels = [label.strip().upper() for label in list(train_data['Label'])]
train_data['Label']=labels
print(set(train_data['Label']))
print(set(val_data['Label']))
print(set(test_data['Label']))
print(train_data.shape)
print(val_data.shape)
print(test_data.shape)
# create a function to remove rows in which the length of the text != the length of the slot tokens
def remove_error(data):
    text_slot = list(zip(list(data['1']),list(data['10'])))

    to_remove = []
    for i, text in enumerate(text_slot):
      if len(str(text[0]).split())!=len(text[1].split()):
        to_remove.append(i)
        print("index:",i)
        print("text:",text[0])
        print("slot label:",text[1])

    to_remove_index = [data.iloc[[n]].index[0] for n in to_remove]
    data.drop(to_remove_index, inplace=True)
    print(data.shape)
# remove rows in train set 
remove_error(train_data)
# remove rows in val set 
remove_error(val_data)
# remove rows in test set 
remove_error(test_data)
# check the proportion of intent labels in the dataset
print("train E:",train_data[train_data['Label']=='E']['Label'].count()/train_data.shape[0])
print("train I:",train_data[train_data['Label']=='I']['Label'].count()/train_data.shape[0])
print("train A:",train_data[train_data['Label']=='A']['Label'].count()/train_data.shape[0])
print("train O:",train_data[train_data['Label']=='O']['Label'].count()/train_data.shape[0])
print("train_data length:",train_data.shape[0])
print()
print("val E:",val_data[val_data['Label']=='E']['Label'].count()/val_data.shape[0])
print("val I:",val_data[val_data['Label']=='I']['Label'].count()/val_data.shape[0])
print("val A:",val_data[val_data['Label']=='A']['Label'].count()/val_data.shape[0])
print("val O:",val_data[val_data['Label']=='O']['Label'].count()/val_data.shape[0])
print("val_data length:",val_data.shape[0])
print()
print("test E:",test_data[test_data['Label']=='E']['Label'].count()/test_data.shape[0])
print("test I:",test_data[test_data['Label']=='I']['Label'].count()/test_data.shape[0])
print("test A:",test_data[test_data['Label']=='A']['Label'].count()/test_data.shape[0])
print("test O:",test_data[test_data['Label']=='O']['Label'].count()/test_data.shape[0])
print("test_data length:",test_data.shape[0])
# create a function to convert the low dataset into jointBERT input
def create_input(data,set):
    seq_in = list(data['1'])
    with open('datasets/low/'+set+'/seq.in', 'w') as f:
        for item in seq_in:
            f.write("%s\n" % item)

    seq_out = list(data['10'])
    with open('datasets/low/'+set+'/seq.out', 'w') as f:
        for item in seq_out:
            f.write("%s\n" % item)

    label = list(data['Label'])
    with open('datasets/low/'+set+'/label', 'w') as f:
        for item in label:
            f.write("%s\n" % item)
# create the directory
!mkdir datasets/low/train
!mkdir datasets/low/val
!mkdir datasets/low/test

# create the input files for jointBERT
create_input(train_data,'train')
create_input(val_data,'val')
create_input(test_data,'test')
data = train_data
seq_in = list(data['1'])
seq_out = list(data['10'])
label = list(data['Label'])
seq_in = ['BOS ' + text + ' EOS\t' for text in seq_in]
result = list(zip(seq_in, seq_out))
result = ["".join(text) for text in result]
result_w_intent = list(zip(result, label))
result_w_intent = [" ".join(text) for text in result_w_intent]

with open('datasets/low/low.train.w-intent.iob', 'w') as f:
  for item in result_w_intent:
    f.write("%s\n" % item)

with open('datasets/low/low.train.iob', 'w') as f:
  for item in result:
    f.write("%s\n" % item)
data = val_data
seq_in = list(data['1'])
seq_out = list(data['10'])
label = list(data['Label'])
seq_in = ['BOS ' + text + ' EOS\t' for text in seq_in]
result = list(zip(seq_in, seq_out))
result = ["".join(text) for text in result]
result_w_intent = list(zip(result, label))
result_w_intent = [" ".join(text) for text in result_w_intent]

with open('datasets/low/low.dev.w-intent.iob', 'w') as f:
  for item in result_w_intent:
    f.write("%s\n" % item)

with open('datasets/low/low.dev.iob', 'w') as f:
  for item in result:
    f.write("%s\n" % item)
data = test_data
seq_in = list(data['1'])
seq_out = list(data['10'])
label = list(data['Label'])
seq_in = ['BOS ' + text + ' EOS\t' for text in seq_in]
result = list(zip(seq_in, seq_out))
result = ["".join(text) for text in result]
result_w_intent = list(zip(result, label))
result_w_intent = [" ".join(text) for text in result_w_intent]

with open('datasets/low/low.test.w-intent.iob', 'w') as f:
  for item in result_w_intent:
    f.write("%s\n" % item)

with open('datasets/low/low.test.iob', 'w') as f:
  for item in result:
    f.write("%s\n" % item)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17309786836782323, 'fraction_numerical': 0.006457133966080598, 'mean_word_length': 5.395522388059701, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '587718', 'license': 'Apache-2.0', 'n_tokens_mistral': 4760, 'n_tokens_neox': 4684, 'n_words': 974, 'path': 'data/conda_low_pre_processing.ipynb', 'repo_name': 'usydnlp-conda/usydnlp-conda.github.io', 'size': '12970'}