<jupyter_start><jupyter_text>Тематика последующего обращения (NN)<jupyter_code>import pandas as pd
import numpy as np
import time
import matplotlib.pyplot as plt
data = pd.read_csv('dialogs_2.csv')
data.head()
data_one_day = data.dropna(subset=['client_id']).copy()
data_one_day = data_one_day[~data_one_day['client_id'].isnull()]
data_one_day = data_one_day.sort_values(by=['client_id'])
data_one_day = data_one_day[data_one_day['client_id'] > 2000]
data_one_day = data_one_day[~data_one_day['text'].isnull()]
data_one_day.iloc[203]
len(np.unique(data_one_day['client_id']))
'интернет': ['скорость', 'интернет', 'сайт', 'знач', 'антен', 'скорость', 'тормозит', 'грузит', 'онлайн', '4g', '3g', '2g'],
'услуги': ['услуг', 'снял', 'отключ', 'подпис'],
'тариф': ['тариф', 'стоимост', 'плат'],<jupyter_output><empty_output><jupyter_text>Препроцессинг<jupyter_code>alphabet = {i:k for k, i in enumerate(' абвгдежзийклмнопрстуфхцчшщъыьэюя')}
topic = []
boc = []
for i in range(len(data_one_day)):
    flag = 1
    text_poverka = data_one_day.iloc[i]['text'].lower()
    tmp_boc = np.zeros(50)
    for j in range(min(50, len(text_poverka))):
        if text_poverka[j] in alphabet:
            tmp_boc[j] = alphabet[text_poverka[j]]
    boc.append(tmp_boc)
    for el in ['скорость', 'интернет', 'сайт', 'знач', 'антен', 'скорость', 'тормозит', 'грузит', 'онлайн', '4g', '3g', '2g']:
        if el in text_poverka:
            topic.append(1)
            flag = 0
            break
    if flag:
        for el in ['услуг', 'снял', 'отключ', 'подпис']:
            if el in text_poverka:
                topic.append(2)
                flag = 0
                break
    if flag:
        for el in ['тариф', 'стоимост', 'плат']:
            if el in text_poverka:
                topic.append(3)
                flag = 0
                break
    if flag:
        topic.append(0)
data_topic = data_one_day.iloc[:len(topic)].copy()
data_topic['tipic'] = topic
data_topic['boc'] = boc
data_topic.head()
train = []
label = []
for client_id in np.unique(data_topic['client_id']):
    tmp = data_topic[data_topic['client_id'] == client_id]
    train.append(np.array(tmp['boc'][:-1]))
    label.append(np.array(tmp['tipic'][1:]))
train_x = []
for i in range(len(train)):
    train_x.extend(train[i])
train_x = np.array(train_x)
train_x[:5]
label_x = []
for i in range(len(label)):
    label_x.extend(label[i])
label_x = np.array(label_x)
label_x[:5]
len_test = int(len(train_x) / 5)
test = train_x[-len_test:]
test_label = label_x[-len_test:]
train = train_x[:-len_test]
label = label_x[:-len_test]
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import to_categorical
y_binary = to_categorical(label)
y_binary[1]<jupyter_output><empty_output><jupyter_text>Модель<jupyter_code>model = Sequential()
model.add(Dense(units=15, activation='relu'))
model.add(Dense(units=4, activation='softmax'))
model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Обучение модели<jupyter_code>model.fit(train, y_binary, epochs=5, batch_size=32)
y_test = to_categorical(test_label)
loss_and_metrics = model.evaluate(test, y_test, batch_size=32)<jupyter_output>2000/2000 [==============================] - 0s 117us/step<jupyter_text>Качество модели<jupyter_code>loss_and_metrics<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16642250219748023, 'fraction_numerical': 0.015235862877234104, 'mean_word_length': 4.5602605863192185, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '297865', 'license': 'MIT', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1392, 'n_words': 228, 'path': 'predict_next_topic(nn).ipynb', 'repo_name': 'rhangelxs/tele2-scheduling', 'size': '3413'}