<jupyter_start><jupyter_text>Удаление стоп-слов, стемминг, оставляем по одному комментарию<jupyter_code>from nltk import download
download('stopwords')

import nltk
nltk.download('wordnet')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
from nltk.stem.snowball import SnowballStemmer 
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import stopwords
stemmer = SnowballStemmer('english')
st_wr = set(stopwords.words('english'))<jupyter_output><empty_output><jupyter_text>Лемматайзер:<jupyter_code>import nltk
from nltk.corpus import wordnet

wnl = WordNetLemmatizer()

# function to convert nltk tag to wordnet tag
def nltk_tag_to_wordnet_tag(nltk_tag):
    if nltk_tag.startswith('J'):
        return wordnet.ADJ
    elif nltk_tag.startswith('V'):
        return wordnet.VERB
    elif nltk_tag.startswith('N'):
        return wordnet.NOUN
    elif nltk_tag.startswith('R'):
        return wordnet.ADV
    else:          
        return None

def lemmatize_sentence(lemmatizer, sentence):
    #tokenize the sentence and find the POS tag for each token
    nltk_tagged = nltk.pos_tag(nltk.word_tokenize(sentence))  
    #tuple of (token, wordnet_tag)
    wordnet_tagged = map(lambda x: (x[0], nltk_tag_to_wordnet_tag(x[1])), nltk_tagged)
    lemmatized_sentence = []
    for word, tag in wordnet_tagged:
        if tag is None:
            #if there is no available tag, append the token as is
            lemmatized_sentence.append(word)
        else:        
            #else use the tag to lemmatize the token
            lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))
    return " ".join(lemmatized_sentence)

# example
lemmatize_sentence(wnl, 'kirill is playing with fire')<jupyter_output><empty_output><jupyter_text>Тут из нескольких комментариев выбирается самый короткий (обосновано тем, что тогда он с большей вероятностью встречается часто == больше похож на метку). Отсекаются пробелы и всякие пустые метки.<jupyter_code>import re
def clear_tag(tag):
    ready_tag = tag.replace('_', ' ')
    ready_tag = re.sub('[^A-Za-z0-9 ]+', '', ready_tag)
    ready_tag = ready_tag.strip()                    # removes spaces in the start and in the end
    ready_tag = ready_tag.replace(' ', '_')
    return ready_tag
from tqdm import tqdm
all_to_del = set(['', "['']", ' ', '  ', "[' ']", '        ', '     ', '    ', '   ', "['', ' ']", '       ', '      ', \
             '             ', '         ', '          ', "']",\
                 "['   ", "['", "[' ", "['  ", "['    ", "['    "])
rows_to_del = []
for i in tqdm(range(data.shape[0])):
    my_string = data.loc[i, 'tag']
    
    if (my_string is not np.nan) and my_string is not None:
        
        min_len = len(data.loc[i, 'tag']) + 10
        min_el = None
        s = set(list(map(str, my_string.split("', '"))))
        
        for el in s:
            
            # getting rid of punctuation, spaces, adding "_"
            el = clear_tag(el)
            
            if el not in all_to_del and el is not None and len(el) < 100:
                
                # getting rid of stop words
                el = [w.lower() for w in el.split('_') if w.lower() not in st_wr]
                el = '_'.join(el)
                
                # stemming 
                el = stemmer.stem(el)
                
                # lemmatizing
                # el = lemmatize_sentence(wnl, el)
                
                if len(el) < min_len:
                    min_len = len(el)
                    min_el = el

        if min_el is not None:
            data.loc[i, 'tag'] = min_el
        else:
            rows_to_del.append(i)
    else:
        rows_to_del.append(i)<jupyter_output>100%|██████████| 221803/221803 [00:53<00:00, 4167.51it/s]<jupyter_text>Было данных:<jupyter_code>data.shape<jupyter_output><empty_output><jupyter_text>Осталось:<jupyter_code>data = data.drop(rows_to_del)
data.shape
data = data.reset_index()
data.drop(columns=['index'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>________  Удаление пустых меток<jupyter_code>data = data[data['tag'] != '']
data = data[data['tag'] != '_']
data = data[data['tag'] != '__']
data = data[data['tag'] != '___']
data = data[data['tag'] != '____']
data = data[data['tag'] != '_____']<jupyter_output><empty_output><jupyter_text>Оставим самые популярные метки На данный момент обработки имеем уникаьных комментариев:<jupyter_code>data['tag'].nunique()<jupyter_output><empty_output><jupyter_text>Оставим 5000 самых популярных<jupyter_code>from collections import Counter

mc = Counter(data['tag']).most_common(5000)
strange_to_del = []<jupyter_output><empty_output><jupyter_text>Если уникальных блоков кода с этим комментарием меньше 75, то удалим эту метку т.к. это скорее всего скопированный комментарий. Этот процесс проводится среди самых популярных 5000 тэгов.<jupyter_code>for el in tqdm(mc):
    if data[data['tag']==el[0]]['code_block'].nunique() < 75:
        strange_to_del.append(el[0])
for el in tqdm(strange_to_del):
    data = data[data['tag'] != el]<jupyter_output>100%|██████████| 4904/4904 [01:25<00:00, 57.40it/s]<jupyter_text>Здесь теряется некоторое количество данных и остается:<jupyter_code>print(data.shape)
print(data['tag'].nunique())<jupyter_output>(118655, 2)
66058<jupyter_text>Посмотрим на самые популярные 20 меток:<jupyter_code>Counter(data['tag']).most_common(20)
num_tags_to_drop = data['tag'].nunique() - 20
num_tags_to_drop<jupyter_output><empty_output><jupyter_text>Удаление самых непопулярные (nunique - 20) меток:<jupyter_code>least_c_tags = set(Counter(data['tag']).most_common()[-num_tags_to_drop:])
for el in tqdm(least_c_tags):
    data = data[data['tag'] != el[0]]<jupyter_output>100%|██████████| 66038/66038 [10:27<00:00, 105.32it/s]<jupyter_text>Осталось:<jupyter_code>print(data.shape)
print(data['tag'].nunique())
ready_data = \
data.reset_index().drop(columns=['index'], axis=1)
mc = Counter(ready_data['tag']).most_common()
#with open("most_common_50_1.txt", 'w') as output:
#    for row in mc:
#        output.write(str(row) + '\n')<jupyter_output><empty_output><jupyter_text>На данном этапе метки вышли не самыми лучшими, поэтому давайте попробуем доработать их самостоятельно: 1) Вариант для стемминга (финальный)<jupyter_code>ready_data['tag'].unique()
jut_inds = ready_data[ready_data['tag'] == 'specify_none_want_read_whole_fil'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)
jut_inds = ready_data[ready_data['tag'] == 'create_x'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)
jut_inds = ready_data[ready_data['tag'] == 'uncom'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)



ready_data.loc[ready_data['tag'] == 'data_preprocessing', 'tag'] = 'preprocessing'
ready_data.loc[ready_data['tag'] == 'data_load', 'tag'] = 'load_data'
ready_data.loc[ready_data['tag'] == 'data', 'tag'] = 'load_data'
ready_data.loc[ready_data['tag'] == 'importing_data', 'tag'] = 'load_data'
ready_data.loc[ready_data['tag'] == 'reading_data', 'tag'] = 'load_data'
ready_data.loc[ready_data['tag'] == 'eda', 'tag'] = 'exploratory_data_analysi'
ready_data.loc[ready_data['tag'] == 'data_explor', 'tag'] = 'exploratory_data_analysi'


jut_inds = ready_data[ready_data['tag'] == 'code'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)
jut_inds = ready_data[ready_data['tag'] == '1'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)

ready_data.loc[ready_data['tag'] == 'importing_librari', 'tag'] = 'import_librari'
ready_data.loc[ready_data['tag'] == 'import', 'tag'] = 'import_librari'
ready_data.loc[ready_data['tag'] == 'librari', 'tag'] = 'import_librari'
ready_data.loc[ready_data['tag'] == 'training_model', 'tag'] = 'import_librari'<jupyter_output><empty_output><jupyter_text>2) Вариант без стемминга, без лемматизации<jupyter_code>#ready_data.loc[ready_data['tag'] == 'data_preprocessing', 'tag'] = 'preprocessing'
#ready_data.loc[ready_data['tag'] == 'modelling', 'tag'] = '
#ready_data.loc[ready_data['tag'] == 'model_building', 'tag'] = 'model_building'
#ready_data.loc[ready_data['tag'] == 'data_visualization', 'tag'] = 'visualization'
#ready_data.loc[ready_data['tag'] == 'data', 'tag'] = 'load_data'
#ready_data.loc[ready_data['tag'] == 'exploratory_data_analysis', 'tag'] = 'eda'
#ready_data.loc[ready_data['tag'] == 'training', 'tag'] = 'train'

#jut_inds = ready_data[ready_data['tag'] == 'just_uncomment_them'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)
#jut_inds = ready_data[ready_data['tag'] == 'check_your_answer'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)
#jut_inds = ready_data[ready_data['tag'] == 'check_your_answers'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)
#jut_inds = ready_data[ready_data['tag'] == 'your_code_here'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)<jupyter_output><empty_output><jupyter_text>3) Вариант с использованием только лемматизации:<jupyter_code>#ready_data.loc[ready_data['tag'] == 'check_your_answers', 'tag'] = 'check_your_answer'
#ready_data.loc[ready_data['tag'] == 'data_preprocessing', 'tag'] = 'preprocessing'
#ready_data.loc[ready_data['tag'] == 'training', 'tag'] = 'train'
#ready_data.loc[ready_data['tag'] == 'exploratory_data_analysis', 'tag'] = 'eda'
#ready_data.loc[ready_data['tag'] == 'data', 'tag'] = 'load_data'
#ready_data.loc[ready_data['tag'] == 'model_building', 'tag'] = 'model'

#jut_inds = ready_data[ready_data['tag'] == 'just_uncomment_them'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)
#jut_inds = ready_data[ready_data['tag'] == 'your_code_here'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)'''<jupyter_output><empty_output><jupyter_text>Вариант для стемминга:<jupyter_code>#ready_data.loc[ready_data['tag'] == 'data', 'tag'] = 'load_data'
#ready_data.loc[ready_data['tag'] == 'train_model', 'tag'] = 'train'
#ready_data.loc[ready_data['tag'] == 'fit_model', 'tag'] = 'train'
#ready_data.loc[ready_data['tag'] == 'data_preprocess', 'tag'] = 'preprocess'
#ready_data.loc[ready_data['tag'] == 'check_answers', 'tag'] = 'check_answer'
#ready_data.loc[ready_data['tag'] == 'model', 'tag'] = 'model_building'
#ready_data.loc[ready_data['tag'] == 'data_visu', 'tag'] = 'visual'
#ready_data.loc[ready_data['tag'] == 'model_building', 'tag'] = 'model_build'

#jut_inds = ready_data[ready_data['tag'] == 'uncomment_them'].index
#ready_data.drop(index=jut_inds, axis=0, inplace=True)'''<jupyter_output><empty_output><jupyter_text>Результаты:<jupyter_code>print(ready_data.shape)
print(ready_data['tag'].nunique())
Counter(ready_data['tag']).most_common()<jupyter_output>(10246, 2)
18<jupyter_text>Восстановление баланса классов:<jupyter_code>todel_mc1 = np.random.choice(
    np.array(ready_data[ready_data['tag'] == 'linear_algebra'].index),
    1859 - 1200,
    replace = False
)

ready_data.drop(index=todel_mc1, inplace=True, axis=0)
print(ready_data.shape)
print(ready_data['tag'].nunique())
Counter(ready_data['tag']).most_common()
jut_inds = ready_data[ready_data['tag'] == 'check_answ'].index
ready_data.drop(index=jut_inds, axis=0, inplace=True)
print(ready_data.shape)
print(ready_data['tag'].nunique())
Counter(ready_data['tag']).most_common()<jupyter_output>(8873, 2)
17<jupyter_text>____________ Разделим на train, test, val в соотношении 8.5 / 1.5 / 1.5 Поделим нацело:<jupyter_code>(ready_data.shape[0] - 7) / 11
ready_data = ready_data.drop(ready_data.tail(7).index)<jupyter_output><empty_output><jupyter_text>То есть размер обучающей выборки:<jupyter_code>ready_data.shape[0] / 11 * 8<jupyter_output><empty_output><jupyter_text>Разделение на train, test, validation с помощью StratifiedShuffleSplit<jupyter_code>from sklearn.model_selection import StratifiedShuffleSplit
SEED = 42001

split = StratifiedShuffleSplit(n_splits=1, test_size=3/11, random_state=SEED)
for train_index, test_valid_index in split.split(\
                                                 ready_data.drop('tag', axis=1), \
                                                 ready_data['tag']\
                                                ):
    train_set = ready_data.iloc[train_index]
    test_valid_set = ready_data.iloc[test_valid_index]

split2 = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=SEED)
for test_index, valid_index in split2.split(test_valid_set, test_valid_set['tag']):
    test_set = test_valid_set.iloc[test_index]
    valid_set = test_valid_set.iloc[valid_index]
train_set = train_set.reset_index()
test_set = test_set.reset_index()
valid_set = valid_set.reset_index()<jupyter_output><empty_output><jupyter_text>______ Далее все '\n' заменяются на '\n\t', а также вначале каждого сниппета приписывается'def tag():\n\t'Каждый сниппет записывается в **отдельный файл** с названием номер_итерации.py в каталог my_train, my_test или my_valid (требование к формату code2vec).<jupyter_code>def add_tabs(exmpl):
    new_exmpl = str()
    for i in range(len(exmpl)):
        if (exmpl[i]=='\n'):
            new_exmpl += '\n\t'
        else:
            new_exmpl += exmpl[i]
    return new_exmpl<jupyter_output><empty_output><jupyter_text>train:<jupyter_code>for i in range(train_set.shape[0]):
        exmpl = train_set.loc[i]['code_block'][:-1]
        exmpl = add_tabs(exmpl)
        print('def ' + str(train_set.loc[i]['tag'])+'():\n\t' + exmpl
              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_train/func'+str(i)+'.py', 'w'))<jupyter_output><empty_output><jupyter_text>test:<jupyter_code>for i in range(test_set.shape[0]):
        exmpl = test_set.loc[i]['code_block'][:-1]
        exmpl = add_tabs(exmpl)
        print('def ' + str(test_set.loc[i]['tag'])+'():\n\t' + exmpl
              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_test/func'+str(i)+'.py', 'w'))<jupyter_output><empty_output><jupyter_text>valid:<jupyter_code>for i in range(valid_set.shape[0]):
        exmpl = valid_set.loc[i]['code_block'][:-1]
        exmpl = add_tabs(exmpl)
        print('def ' + str(valid_set.loc[i]['tag'])+'():\n\t' + exmpl
              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_val/func'+str(i)+'.py', 'w'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1613653995345229, 'fraction_numerical': 0.015374850130474646, 'mean_word_length': 4.778321108394458, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '528836', 'license': 'MIT', 'n_tokens_mistral': 5659, 'n_tokens_neox': 5584, 'n_words': 836, 'path': 'pre-preprocessing.ipynb', 'repo_name': 'AmmarHarrat/Test_code2vec', 'size': '14179'}