<jupyter_start><jupyter_text><jupyter_code>! git --version
! git clone https://github.com/Alenush/style_transfer_sirius2021summer/
%cd style_transfer_sirius2021summer/data/scripts/
import glob
import os
import re
from itertools import chain

import pandas as pd
from tqdm import tqdm

def fetch_lines():
    lines = []
    for fname in glob.glob('./**/*.txt'):
        with open(fname, 'r') as f:
            lines.append(f.readlines())
    return lines
%cd russian
russian_episodes = fetch_lines()
%cd ../english
english_episodes = fetch_lines()
assert len(english_episodes) == len(russian_episodes), "Eng and Rus n of series mismatch"
! cp -R "/content/drive/MyDrive/Colab Notebooks/friends-classification" /content
% cd /content/friends-classification/
if not os.path.exists('excel_files'): 
    os.mkdir('excel_files')
for i in range(len(english_episodes)):
    left = len(english_episodes[i])
    right = len(russian_episodes[i])
    if left != right:
        excel = pd.concat(
        [pd.Series(english_episodes[i], name='eng'),
        pd.Series(russian_episodes[i], name='rus')],
        axis=1)
        #excel = pd.DataFrame(dict(eng=english_episodes[i], rus=russian_episodes[i]))
        excel.to_excel(f'excel_files/{i}.xlsx')
        print( f"Mismatch in number of lines in ep {i}, eng: {left}, rus: {right}")
    
import shutil
shutil.make_archive('excel_files', 'zip', 'excel_files')<jupyter_output><empty_output><jupyter_text>Then I've manually aligned texts in Excel and put the results into `excel_files_edited` directory.<jupyter_code>for i in range(len(english_episodes)):
    left = len(english_episodes[i])
    right = len(russian_episodes[i])
    if left != right:
        df = pd.read_excel(f'excel_files_edited/{i}.xlsx')
        eng = df['eng'].to_list()
        rus = df['rus'].to_list()
        assert len(eng) == len(rus) and len(eng) == left, \
          "Mismatch in edited file"
        english_episodes[i] = eng
        russian_episodes[i] = rus
        print( f"New number of lines in ep {i}, eng:"
         f"{len(english_episodes[i])}, rus: {len(russian_episodes[i])}")
    
#pd.set_option('max_colwidth', None)
df = pd.read_excel(f'excel_files_edited/198.xlsx')
df[:-60].tail(60)
from itertools import chain
import pandas as pd
russian_meta_lines = list(chain(*russian_episodes))
russian_lines = list(
    map(lambda line: re.split(r"^.*?[А-Я]+:\s?|\n", line)[1], russian_meta_lines)
)
russian_lines = list(
    map(lambda line: re.sub(r'"', "", line), russian_lines)
)
english_meta_lines = list(chain(*english_episodes))
assert len(english_meta_lines) == len(russian_meta_lines), "Length mismatch between ENG and RUS scripts"
english_lines = list(
    map(lambda line: re.split(r"^[А-Я]+:\s?|\n", line)[1], english_meta_lines)
)
english_lines = list(
    map(lambda line: re.sub(r'"', "", line), english_lines)
)
df_train = pd.read_csv('train_data.csv').rename({'Category': 'label'}, axis=1)
df_train.other_speaker.fillna('', inplace=True)
df_val = pd.read_csv('val_data.csv')
df_val.other_speaker.fillna('', inplace=True)
df_test = pd.read_csv('test.csv')
df_test.other_speaker.fillna('', inplace=True)
print(df_train.shape, df_val.shape, df_test.shape)
"""from tqdm import tqdm
from numba import njit, jit
@njit
def get_lines(lines, russian_lines):
    common_lines = []
    for line in lines:
      for reference in russian_lines:
        if reference.find(line) > -1:
          common_lines.append(line)
          break
    return common_lines
get_lines(df_train.friend_response.to_list()[:100], list(russian_lines))"""
      
def get_true_lines(lines):
    common_lines = [] #pd.Series([''] * len(df_train))
    found_indices = [] #pd.Series([-1] * len(df_train)
    negatives = []
    for line in tqdm(lines):
        for i, reference in enumerate(russian_lines):
            if line in reference:
                common_lines.append(reference)
                found_indices.append(i)
                break
        else:
            curr = re.sub(r"[\.\!\?]", " ", line)
            curr = re.sub(" +", ".*?", curr)
            #print(curr, line)
            for i, reference in enumerate(russian_lines):
                if re.findall(curr, reference):
                    common_lines.append(reference)
                    found_indices.append(i)
                    break
            else: 
                negatives.append(line)
                common_lines.append("")
                found_indices.append(-1)
    if not negatives: print('No missing lines, found all.')
    return common_lines, found_indices, negatives

def supplement_dataset(df):
    df = df.copy()
    print('Processing friend_response:')
    df['true_friend_response'], df['inds_friend_response'], _ \
    = get_true_lines(df['friend_response'])
    df['inds_other_speaker'] = df['inds_friend_response'] - 1
    df['true_other_speaker'] = df['inds_other_speaker'].apply(lambda x: russian_lines[x])
    df['eng_friend_response'] = df['inds_friend_response'].apply(lambda x: english_lines[x])
    df['eng_other_speaker'] = df['inds_other_speaker'].apply(lambda x: english_lines[x])
    mask = df['inds_friend_response'] != -1
    print(f"\n{mask.sum()} items were found")
    return df[mask]
df_val_aug = supplement_dataset(df_val)
df_test_aug = supplement_dataset(df_test)
df_train_aug = supplement_dataset(df_train)
df_val_aug.query('inds_friend_response == -1')
df_test_aug.query('inds_friend_response != inds_other_speaker + 1')
df_val_aug.sample(20)
def compose_eng_ds(df_orig, df_aug):
    eng_df = df_orig.copy()
    eng_df.other_speaker = df_aug.eng_other_speaker
    eng_df.friend_response = df_aug.eng_friend_response
    return eng_df
def compose_rus_aug_ds(df_orig, df_aug):
    rus_df = df_orig.copy()
    rus_df.other_speaker = df_aug.true_other_speaker
    rus_df.friend_response = df_aug.true_friend_response
    return rus_df
df_train_eng = compose_eng_ds(df_train, df_train_aug)
df_val_eng = compose_eng_ds(df_val, df_val_aug)
df_test_eng = compose_eng_ds(df_test, df_test_aug)

df_train_true_rus = compose_rus_aug_ds(df_train, df_train_aug)
df_val_true_rus = compose_rus_aug_ds(df_val, df_val_aug)
df_test_true_rus = compose_rus_aug_ds(df_test, df_test_aug)
if not os.path.exists('english'): 
    os.mkdir('english')
df_train_eng.to_csv('english/df_train_eng.csv')
df_val_eng.to_csv('english/df_val_eng.csv')
df_test_eng.to_csv('english/df_test_eng.csv')

if not os.path.exists('russian_true'): 
    os.mkdir('russian_true')
df_train_true_rus.to_csv('russian_true/df_train_true_rus.csv')
df_val_true_rus.to_csv('russian_true/df_val_true_rus.csv')
df_test_true_rus.to_csv('russian_true/df_test_true_rus.csv')
shutil.make_archive('english', 'zip', 'english')
shutil.make_archive('russian_true', 'zip', 'russian_true')
df_val_eng
df_val<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13560817767318722, 'fraction_numerical': 0.004559494043241653, 'mean_word_length': 4.120481927710843, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '189147', 'license': 'MIT', 'n_tokens_mistral': 2528, 'n_tokens_neox': 2502, 'n_words': 436, 'path': "Kaggle Sber's Friends Dialogues Classification Challenge/eng_original_model/eng_orig_preparation.ipynb", 'repo_name': 'andrewargatkiny/contests', 'size': '6799'}