<jupyter_start><jupyter_text>Что тут есть?cross_section - можно думать, что это логарифм площади  relative_lateral_velocity - скорость относительно радара, радиальная ее часть  absolute_radial_velocity - скорость относительно мираbelongs_to_object - попала ли точка по нашему мненияю на машину  Эвристика №1![](./img/credo.jpg)Машины большие и едут быстро<jupyter_code>def big_cars_sol(ds):
    y_hat = np.zeros(len(ds.values))
    dummy_mask = ds.absolute_radial_velocity > 2
    y_hat[dummy_mask] = 1
    return y_hat.astype(int)

y_hat = big_cars_sol(ds)
print('precission', precision_score(ds.belongs_to_object.values, y_hat))
print('recall', recall_score(ds.belongs_to_object.values, y_hat))
print('f1', f1_score(ds.belongs_to_object.values, y_hat))<jupyter_output>precission 0.3082339135536867
recall 0.11514575874917805
f1 0.1676595744680851<jupyter_text>Эвристика №2Машины отражают по несколько точек<jupyter_code>def kkn2_sol(ds):
    dummy_mask = ds.absolute_radial_velocity > 2
    dummy_cars = ds[dummy_mask][['x', 'y']].values

    y_hat = np.zeros(len(ds.values))
    if dummy_cars.shape[0] >= 2:
        index = NearestNeighbors(metric='l2')
        index.fit(dummy_cars)
        d, _ = index.kneighbors(ds[['x', 'y']], n_neighbors=2)
        y_hat[(d < 2.0)[:, 0]] = 1

    return y_hat.astype(int)

y_hat = kkn2_sol(ds)
print('precission', precision_score(ds.belongs_to_object.values, y_hat))
print('recall', recall_score(ds.belongs_to_object.values, y_hat))
print('f1', f1_score(ds.belongs_to_object.values, y_hat))
import matplotlib.pyplot as plt
from IPython import display
import time
# ax = plt.gca()
# for seq in sorted(ds.seq.unique()):
#     ax.clear()
#     scene = ds[ds.seq == seq]
#     plt.scatter(scene.x, scene.y, s=5, c=scene.relative_radial_velocity)
#     ax.set_xlim((450,490))
#     ax.set_ylim((555,580))
#     display.clear_output(wait=True)
#     display.display(plt.gcf())
#     time.sleep(0.01)<jupyter_output><empty_output><jupyter_text>Tree<jupyter_code>stamp_order = ds.groupby('seq').apply(lambda x: x.stamp.max()).index.to_list()
assert stamp_order == sorted(stamp_order), 'Seqs ordered by number.'
def add_features(scene):
    index = NearestNeighbors(metric='l2', n_jobs=8)
    train_points = scene[scene.absolute_radial_velocity > 2]
    index.fit(train_points[['x', 'y']].values)
    scene = scene.copy()
    for radius in tqdm.tqdm((2, 4, 8, 16, 32, 64, 128, 256)):
        dist, ind = index.radius_neighbors(scene[['x', 'y']].values, radius=radius)
        f1, f2, f3 = [], [], []
        for d1, i1 in zip(dist, ind):
            if len(i1):
                points = train_points.iloc[i1, :]
                f1.append(points.relative_radial_velocity.mean())
                f2.append(points.absolute_radial_velocity.mean())
                f3.append(np.mean(d1))
            else:
                f1.append(None)
                f2.append(None)
                f3.append(None)
        
        scene['ave_rel_vel_{}'.format(radius)] = f1
        scene['ave_abs_vel_{}'.format(radius)] = f2
        scene['ave_dist_{}'.format(radius)] = f3
    return scene
scenes = []
for seq in tqdm.tqdm(sorted(ds.seq.unique())):
    scene = ds[ds.seq == seq]
    scenes.append(scene)
n = len(scenes)
train = scenes[:int(0.8 * n)]
val = scenes[int(0.8 * n): int(0.9 * n)]
test = scenes[int(0.9 * n):]
assert len(train) + len(val) + len(test) == n
len(train), len(val), len(test)
def to_ds(scenes):
    scene_list = []
    for scene in scenes:
        scene_list.append(scene.values)
    ds = np.concatenate(scene_list, axis=0)
    ds = pd.DataFrame(ds, columns=scenes[0].columns)
    ds = add_features(ds)
    ds.drop(['id', 'stamp', 'seq'], axis=1, inplace=True)
    return ds
train_ds, val_ds, test_ds = to_ds(train), to_ds(val), to_ds(test)
train_ds.shape, val_ds.shape, test_ds.shape
LABEL = 'belongs_to_object'
LABEL_COLUMS = ['x', 'y', 'object_label', 'belongs_to_object']
import catboost

def learn(X_train, X_val, y_train, y_val):
    train_pool = catboost.Pool(X_train, y_train)
    clf = catboost.CatBoostClassifier(
        custom_loss=['AUC', 'Accuracy'],
        n_estimators=3000,
        depth=6,
        learning_rate=1e-3,
        colsample_bylevel=0.25,
        l2_leaf_reg=1,
    )
    clf.fit(
        train_pool,
        early_stopping_rounds=10,
        use_best_model=True, 
        eval_set=(X_val.values, y_val.values),
        plot=True,
        verbose=False,
    )
    return clf

X_train = train_ds.drop(LABEL_COLUMS, axis=1)
y_train = train_ds[LABEL]


X_val = val_ds.drop(LABEL_COLUMS, axis=1)
y_val = val_ds[LABEL]
cls = learn(X_train, X_val, y_train, y_val)
X_val = val_ds.drop(LABEL_COLUMS, axis=1)
y_val = val_ds[LABEL].values.astype(int)
THRESHOLD = 0.29  # Fit best based on val part.
y_hat = (cls.predict_proba(X_val)[:, 1] > THRESHOLD).astype(int)
print('precission', precision_score(y_val, y_hat))
print('recall', recall_score(y_val, y_hat))
print('f1', f1_score(y_val, y_hat))
X_test = test_ds.drop(LABEL_COLUMS, axis=1)
y_test = test_ds[LABEL].values.astype(int)
y_hat = (cls.predict_proba(X_test)[:, 1] > THRESHOLD).astype(int)
print('precission', precision_score(y_test, y_hat))
print('recall', recall_score(y_test, y_hat))
print('f1', f1_score(y_test, y_hat))
for c, i in zip(X_test.columns, cls.feature_importances_):
    print(c, i)<jupyter_output>probability 0.0
relative_radial_velocity 5.320843881517552
cross_section 0.8000596895139328
absolute_radial_velocity 2.7539740577954905
ave_rel_vel_2 14.013927826852647
ave_abs_vel_2 2.7538102477289623
smth_2 1.3816029717406286
ave_rel_vel_4 7.160274823966891
ave_abs_vel_4 5.387840357047283
smth_4 1.7774249562865014
ave_rel_vel_8 7.296915757466398
ave_abs_vel_8 3.3673964815670603
smth_8 3.592945820278413
ave_rel_vel_16 2.777925587574162
ave_abs_vel_16 2.4155976509714154
smth_16 9.995984167380719
ave_rel_vel_32 1.4304359304887935
ave_abs_vel_32 0.8423985675796444
smth_32 10.328079398153406
ave_rel_vel_64 1.8416405064230363
ave_abs_vel_64 2.717261260716759
smth_64 2.059725562128506
ave_rel_vel_128 3.063384472498454
ave_abs_vel_128 2.673593678495085
smth_128 1.2503192947877924
ave_rel_vel_256 1.0317774900736367
ave_abs_vel_256 0.44518319862781686
smth_256 1.5196763623389944
{'fraction_non_alphanumeric': 0.13093131735563776, 'fraction_numerical': 0.10567797973299019, 'mean_word_length': 4.3788927335640135, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '265696', 'license': 'MIT', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2584, 'n_words': 416, 'path': 'radar.ipynb', 'repo_name': 'stasbel/self-driving-cars', 'size': '6217'}