<jupyter_start><jupyter_text>DATASET PROCESING Загружаем граф:<jupyter_code>graph = pd.read_csv("duo_edge.csv")
graph.dropna(inplace=True)
graph['adjacent'] = graph['adjacent'].astype("str")
graph.head() # помогите меня держут в рабстве, мне не платят деньги и не дают еду. я хочу домой...<jupyter_output><empty_output><jupyter_text>Генерируем edge list:<jupyter_code>def get_index_tuples(df):
    lst = []
    for i in tqdm(range(len(df))):
        tmp = df['adjacent'].iloc[i].split(',')
        for edge in tmp:
            lst.append([i, int(edge)])
            
    return lst
lst = get_index_tuples(graph)
lst = pd.DataFrame(lst)
lst
edge_data = pd.DataFrame(
    {
        "source": list(lst[0]),
        "target": list(lst[1])
    }
)
edge_data<jupyter_output><empty_output><jupyter_text>Дропаем лишнее (структурное описание/копии индекса/артефакты):<jupyter_code>node_data = pd.read_csv('node_data.csv')
node_data = graph.drop(["edge_id", "adjacent", "Unnamed: 0", "style_<pandas.io.formats.style.Styler object at 0x000001BA9BC98AC0>"], axis = 1)<jupyter_output><empty_output><jupyter_text>Создаем граф:<jupyter_code>G = sg.StellarGraph(
    {"street": node_data}, {"connection": edge_data}
)
print(G.info())<jupyter_output>StellarGraph: Undirected multigraph
 Nodes: 231688, Edges: 1149492

 Node types:
  street: [231688]
    Features: float32 vector, length 100
    Edge types: street-connection->street

 Edge types:
    street-connection->street: [1149492]
        Weights: all 1 (default)
        Features: none<jupyter_text>CONCATENATE EMBEDDINGS Загружаем файл с путями:<jupyter_code>node_data.head()
cors = []
for i in tqdm(range(len(node_data))):
    cors.append([node_data.lat.iloc[i], node_data.lon.iloc[i]])
cors[:10]
import numpy as np
from sklearn.neighbors import BallTree, DistanceMetric

X = np.array([[np.radians(x) for x in list_] for list_ in cors])
tree = BallTree(X, leaf_size=3, metric = DistanceMetric.get_metric('haversine'))             

dist, idx = tree.query([cors[0]], k = 1)
c = 2 * asin(sqrt(a)) 
r = 6371
node_data.lat.iloc[178405], node_data.lon.iloc[178405]
for i in tqdm(range(len(samples))):
    edges_id=[]
    ok = True
    if i not in ids_to_remove:
        for cor in samples[i]['cors']:
            dist, idx = tree.query([[cor[0]* np.pi / 180, cor[1]* np.pi / 180]], k = 1)
            if dist*6371000 > 100:
                ids_to_remove.append(i)
                ok = False
                break
            edges_id.append(idx[0][0])
        if ok:
            samples[i]['edges'] = edges_id
len(ids_to_remove), len(samples)
ids_to_remove = list(set(ids_to_remove))
samples_copy = samples.copy()
for i in ids_to_remove[::-1]:
    samples_copy.pop(i)
for samp in samples_copy:
    del samp['cors']
samples_copy[5]
import pickle
with open('samples.pkl', 'rb') as f:
    samples = pickle.load(f)
routes = pd.read_csv("dgi_sage_128_5_relu_1100e_0.33l.csv", index_col = 0)
routes.head()<jupyter_output><empty_output><jupyter_text>Дропаем лишние копии индекса (если они имеются):<jupyter_code>c_routes = routes.drop(["index", "level_0","Unnamed: 0.1","Unnamed: 0.1.1","Unnamed: 0.1.1.1","Unnamed: 0.1.1.1.1"], axis = 1)
#c_routes = routes.drop(["Unnamed: 0.1","Unnamed: 0.1.1"], axis = 1)
c_routes<jupyter_output><empty_output><jupyter_text>Дропаем лишние признаки:<jupyter_code>X_pre = c_routes.drop(["real_dist","pred_dist","rebuildCount", "speed", "time", "length", "route_type","start_timestamp"], axis = 1)<jupyter_output><empty_output><jupyter_text>Проверяем индексацию:<jupyter_code>X_pre = X_pre.reset_index().drop(["index"], axis = 1)
import multiprocessing as mp
import psutil
X_pre.loc[3298, "edges"].replace("'", "")<jupyter_output><empty_output><jupyter_text>Процессим edges: (будет исправлено позднее)<jupyter_code>for i in tqdm(range(len(X_pre.loc[:, "edges"]))):
    X_pre.loc[i, "edges"] = X_pre.loc[i, "edges"].replace("'", "")
def summ(first, second):
    return [x + y for x, y in zip(first, second)]
def zerolistmaker(n):
    return [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,]<jupyter_output><empty_output><jupyter_text>Размер эмбеддинга:<jupyter_code>emb_len = 128<jupyter_output><empty_output><jupyter_text>Определяем размер батча для распараллеливания построения эмбеддинга подграфа:<jupyter_code>node_embeddings = pd.read_csv("best_emb.csv", index_col = 0)
workers_count = 20
part_const = int(len(samples_copy)/workers_count)
part_const
X_pre = samples_copy
def spawn():
    dataframe_list = []
    procs = list()
    n_cpus = psutil.cpu_count()
    manager = mp.Manager()
    return_list = manager.list()
    
    for cpu in range(workers_count):
        d = dict(border = part_const*(cpu+1))
        p = mp.Process(target=run_child, args=(d, return_list))
        p.start()
        procs.append(p)
    for p in procs:
        p.join()
        print('joined')
    
    return return_list

def run_child(border, return_list):
    emb = []
    for i in tqdm(range(border['border'] - part_const, border['border'])):
        path_emb = zerolistmaker(emb_len)
#        print(X_pre.loc[i, "edges"])
        arr = X_pre[i]["edges"]
        for j in range(len(arr)):
            ind = graph[graph["edge_id"] == int(arr[j])].index
            if (len(ind) != 0):
                ind = graph[graph["edge_id"] == int(arr[j])].index[0]
            else:
                print(str(i) + " " + str(j) + " " + arr[j] + " "+ str(graph[graph["edge_id"] == int(arr[j])].index[0]))
            path_emb = summ(node_embeddings.iloc[ind], path_emb)
        emb.append(path_emb)
    return_list.append([(int(border['border'])-part_const)/part_const, emb])
 #   print(border['border'])
#    df = pd.DataFrame(dataframe[(int(border['border'])-part_const):border['border']]).reset_index()
#    return_list.append([(int(border['border'])-part_const)/part_const,processing.flatternize(16, filepath = "", dataframe = df)])
a = spawn()
graph = node_data<jupyter_output><empty_output><jupyter_text>Собираем данные:<jupyter_code>normal_container_s = []
for i in tqdm(range(len(a))):
    normal_container_s.append(a[i])
a_ = sorted(normal_container_s,key=lambda x: (x[0],x[1]))
a_c = a_.copy() 
a_ = sorted(normal_container_s,key=lambda x: (x[0],x[1]))
result = a_[0][1]
for i in range(1, len(a_)):
    result += a_[i][1]<jupyter_output><empty_output><jupyter_text>Технические проверки:<jupyter_code>X_pre_c = X_pre.drop(X_pre[X_pre["snow"] == -1].index)
set(X_pre_c["day_period"])<jupyter_output><empty_output><jupyter_text>Объединяем данные путей с эмбеддингами, завершаем подготовку данных к обучению:<jupyter_code>X_pre_c = pd.DataFrame(samples_copy)
X_pre_c.head()
X = X_pre_c.join(pd.DataFrame(result)).dropna().drop(["edges"], axis = 1).reset_index()
# wind_dir_classes = pd.get_dummies(X.wind, prefix='wind_dir_class')
# day_classes = pd.get_dummies(X.day_period, prefix='day_class')
Y = X["time"]
X = X.drop(["time"], axis = 1)
X.shape<jupyter_output><empty_output><jupyter_text>Модель 1:<jupyter_code>len(X)
from sklearn.linear_model import ElasticNetCV
model = ElasticNetCV()
prediction = model.fit(X.iloc[0:9000, :], list(Y[0:9000]), epochs=100)
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
a_RMSE = mean_squared_error(Y[9000:], prediction.predict(X.iloc[9000:, :]))
a_MAE = mean_absolute_error(Y[9000:], prediction.predict(X.iloc[9000:, :]))
pd.DataFrame([[a_RMSE, a_MAE]], index = ["Abakan"], columns = ["RMSE", "MAE"])<jupyter_output><empty_output><jupyter_text>Модель 2:<jupyter_code>import tensorflow as tf

from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
modelNN = tf.keras.models.Sequential()
modelNN.add(tf.keras.Input(shape=(53,)))
modelNN.add(tf.keras.layers.Dense(128, activation='relu'))
modelNN.add(tf.keras.layers.Dense(128, activation='relu'))
modelNN.add(tf.keras.layers.Dense(1))
modelNN.compile(
    optimizer=tf.optimizers.Adam(learning_rate=0.1),
    loss='mean_absolute_error')
train_x = np.asarray(X.iloc[:9000])
train_y = np.asarray(Y[:9000])
test_x = np.asarray(X.iloc[9000:])
test_y = np.asarray(Y[9000:])
from  tensorflow.keras.utils import normalize, to_categorical
train_x = normalize(train_x, axis=1)
test_x =  normalize(test_x, axis=1)
historyNN = modelNN.fit(train_x, train_y, epochs = 100)
from stellargraph.utils import plot_history

plot_history(historyNN)
result = modelNN.evaluate(
    test_x,
    test_y, verbose=0)
result
from sklearn.metrics import mean_absolute_error
print(mean_absolute_error(test_y, modelNN.predict(test_x)))
a_RMSE<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16130897317298798, 'fraction_numerical': 0.0311054579093432, 'mean_word_length': 5.108050847457627, 'pattern_counts': {'":': 4, '<': 65, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '572026', 'license': 'MIT', 'n_tokens_mistral': 3605, 'n_tokens_neox': 3527, 'n_words': 557, 'path': 'algorithms/regression.ipynb', 'repo_name': 'Vloods/TransTTE_demo', 'size': '8648'}