<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>def is_nan_stat(_dir, _ord=None):
    
    def list_cusum(lst):
    
        out = []
        tmp = []
        k = 0

        for i in range(0, len(lst)):

            for j in range(k, i + 1):

                tmp.append(lst[j])
                k = j + 1


            out.append(deepcopy(tmp))

        return out



    isna = {}
    df_nan = {}
    mn_size = 1e9
    
    for i in tqdm(range(0 , len(_dir))):

        x = pd.read_csv(_dir[i])
        mn_size = min(mn_size, len(x))
        
        df_nan[i] = (x.isna().sum().values == len(x)).astype('int')
        
        tmp = x.isna().sum()
        
        if i == 0:
            
            for col in x.keys():
                
                isna[col] = 0
        
        for col in tmp.keys():
            
            isna[col] += tmp[col]
        
    
    keys = _ord
    
    if _ord is None:
        
        isna = [(j, i) for i, j in isna.items()]

        isna.sort()

        keys = [j for i,j in isna]
    
    else:
        
        isna = [(isna[keys[i]], keys[i]) for i in range(len(keys))]
        
    ckeys = list_cusum(keys)
    
    values = [i for i, j in isna]
    values = np.array(values)
    
    cusum_nan = np.cumsum(values)
    size = np.repeat(mn_size * len(_dir), 10)

    worst_case_size = size - cusum_nan
    worst_case_size[worst_case_size < 0] = 0
    
    choices_size = list(zip(worst_case_size, ckeys))
    
    nan = np.array(list(df_nan.values()))
    
    return choices_size, nan # nan/sensor [from 1 to 10 in order] for each dataframe
# test_choices, test_nan = is_nan_stat(test_dir, _ord=None)

# features = np.where(test_nan.sum(axis=0) == 0)[0] + 1
# features = list(map(lambda f: f'sensor_{f}', features))

# print(features, '-- test size :', test_choices[1][0])

features = ['sensor_4', 'sensor_6']<jupyter_output><empty_output><jupyter_text>Sampling<jupyter_code>def get_id(_dir):
    
    return os.path.splitext(_dir)[0].split('/')[-1]

def get_traget(_dir):
    
    return metadata.query(f'segment_id == {get_id(_dir)}').values[0, 1]

def get_sample(_dir, sample_size=10, typ='train', seed=None):
    
    sample = None
    skip_counts = 0
    
    seg_id = []
    
    for i in tqdm(range(len(_dir))):
        
        if seed:
            seed += i
        
        x = pd.read_csv(_dir[i])[features].dropna()
                
        if len(x) < sample_size:
            
            if typ == 'test':
                
                raise ValueError(f'len(x) < sample_size, len(x) = {len(x)}, sample_size = {sample_size}, check features choice (nan)')
            
            else:
                
                skip_counts +=1
                
        x = x.sample(n=sample_size, random_state=seed)
        
        _id = get_id(_dir[i])
        _id = [_id] * len(x)
        
        seg_id +=_id

        if typ == 'train':
            
            y = get_traget(_dir[i])
        
        if i == 0:
            
            sample = x
            
            if typ == 'train':
                
                sample['target'] = [y] * len(x)
            
        else:
            
            tmp = x
            
            if typ == 'train':
                
                tmp['target'] = [y] * len(x)
            
            sample = pd.concat([sample, tmp], axis=0)
    
    sample.insert(loc=0, column='segment_id', value=seg_id)
    sample.reset_index(drop=True, inplace=True)
    
    counts = len(_dir) - skip_counts
    
    return sample, counts
sample_size = 1000
seed = 1

test, _ = get_sample(_dir=test_dir, sample_size=sample_size, typ='test', seed=seed)
train, counts = get_sample(_dir=train_dir, sample_size=sample_size, typ='train', seed=seed)
print('valid sample :', len(np.unique(test['segment_id'])) == len(test_dir))<jupyter_output>valid sample : True<jupyter_text>Model $minimize : \frac{1}{m} \sum_{i=0}^{m} \ (\hat{y} * scale - y)^2$> $where: \hat{y} = w_0 * d_0 + w_1 * d_1 + .... + w_k * d_k$> $subject \ to: 0 \leq w_j \leq 1$ ---> (Could be converted into implicit constraints using sigmoid function)<jupyter_code>def standardize(x):
    
    return (x - x.mean(axis=0)) / x.std(axis=0)
X = train.to_numpy().reshape(counts, sample_size, 4)
Y = X[:, 0, -1, None].astype('float32')

X = X[:, :, 1:-1].astype('float32')

x_test = test.to_numpy().reshape(len(test_dir), sample_size, 3)
seg_id = x_test[:, 0, 0]

x_test = x_test[:, :, 1:].astype('float32')

standardized = standardize(X)
standardized_test = standardize(x_test)

print(X.shape)
print(Y.shape)

print('\ntest shape : ', x_test.shape)

# x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.1)

# print(x_train.shape)
# print(x_val.shape)
offset = 1e3
scale = 1e6
step = 0.1

mx = (train['target'].max() + offset) / scale

durations_dist = np.arange(1 , mx, step=step).astype('float32')
durations_dist = durations_dist[None, :]

dist_size = durations_dist.shape[-1]

print('shape : ', durations_dist.shape)
class WeightedSum(tf.keras.layers.Layer):
    
    def call(self, inputs):
        
        out = K.sum(inputs * durations_dist, axis=-1)
        out = out[:, None]
        
        return out
inputs = Input(shape=[sample_size, 2])

conv1d = Conv1D(filters=1, kernel_size=sample_size-1, activation='relu')(inputs)

batch_norm = BatchNormalization()(conv1d)

lstm = LSTM(units=1, activation='tanh', return_sequences=True)(batch_norm)
lstm = LSTM(units=dist_size, activation='sigmoid')(lstm) # weights

dropout = Dropout(0.2)(lstm)

outputs = WeightedSum()(dropout)

model = Model(inputs=inputs, outputs=outputs)

model.summary()
def mse_loss(y_true, y_pred):
    
    loss = (y_pred - (y_true / scale)) ** 2
    
    return K.mean(loss)
    
def metrice(y_true, y_pred):
    
    msa = K.abs((y_pred * scale - y_true))
    
    return K.mean(msa)
opt = tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)
model.compile(optimizer=opt, loss=mse_loss, metrics=metrice)
history = model.fit(X, Y, steps_per_epoch=256, epochs=100)
y_test = model.predict(x_test)
y_test *= scale
y_test.min(), y_test.max()
submission = {}

submission['segment_id'] = seg_id.squeeze()
submission['time_to_eruption'] = y_test.squeeze()

submission = pd.DataFrame(submission)

submission.to_csv('submission.csv', encoding='utf-8', index=False)
submission.head(2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14918597370068878, 'fraction_numerical': 0.017063243581715716, 'mean_word_length': 2.3120787973043027, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '611252', 'license': 'MIT', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2342, 'n_words': 455, 'path': 'deep-learning/ingv-volcanic-eruption-weighted-sum-model.ipynb', 'repo_name': 'm-zayan/notebooks', 'size': '6388'}