<jupyter_start><jupyter_text>2-D Synth<jupyter_code>dataset_name = 'data/simulated/hawkes_synthetic_random_2d_20191130-180837.pkl'  # run dpp.data.list_datasets() to see the list of available datasets
dataset = np.load(dataset_name,allow_pickle = True)

## Modify the dataset for IFTPL

sequences = []

# for i  in range(len(dataset['timestamps'])):
for i  in range(400):
    sequence = {'t_start':0,'t_end' :200, 'arrival_times':dataset['timestamps'][i].tolist(),'marks' :dataset['types'][i].tolist()}
    
    sequences.append(sequence)
    
simulated_data = {'sequences':sequences,'num_marks':2}

np.save('data/simulated/sample_hawkes',simulated_data,allow_pickle = True)

dataset_name = 'data/simulated/hawkes_synthetic_random_2d_20191130-180837.pkl'  # run dpp.data.list_datasets() to see the list of available datasets
dataset = np.load(dataset_name,allow_pickle = True)

## Modify the dataset for IFTPL

sequences = []

# for i  in range(len(dataset['timestamps'])):
for i  in range(3200):
    sequence = {'t_start':0,'t_end' :200, 'arrival_times':dataset['timestamps'][i].tolist(),'marks' :dataset['types'][i].tolist()}

    sequences.append(sequence)
    
simulated_data = {'sequences':sequences,'num_marks':2}

with open('data/simulated/train.pkl', 'wb') as handle:
    pickle.dump(simulated_data, handle, protocol=pickle.HIGHEST_PROTOCOL)

## Modify the dataset for IFTPL

sequences = []

# for i  in range(len(dataset['timestamps'])):
for i  in range(3200,3600):
    sequence = {'t_start':0,'t_end' :200, 'arrival_times':dataset['timestamps'][i].tolist(),'marks' :dataset['types'][i].tolist()}

    sequences.append(sequence)
    
simulated_data = {'sequences':sequences,'num_marks':2}

with open('data/simulated/valid.pkl', 'wb') as handle:
    pickle.dump(simulated_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
## Modify the dataset for IFTPL

sequences = []

# for i  in range(len(dataset['timestamps'])):
for i  in range(3600,4000):
    sequence = {'t_start':0,'t_end' :200, 'arrival_times':dataset['timestamps'][i].tolist(),'marks' :dataset['types'][i].tolist()}

    sequences.append(sequence)
    
simulated_data = {'sequences':sequences,'num_marks':2}

with open('data/simulated/test.pkl', 'wb') as handle:
    pickle.dump(simulated_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
def convert_sahp_data_format_to_itfl(dataset,num_marks = 2,t_end = 6,t_start = 0):
    
    dataset 
    sequences= []
    t_max = 0
    t_min = 10e10
    for i  in range(len(dataset['timestamps'])):
        m = max(dataset['timestamps'][i])
        mi = min(dataset['timestamps'][i])
        if m>t_max:
            t_max = m
        if mi<t_min:
            t_min = mi
        sequence = {'t_start':t_start,'t_end' :t_end, 'arrival_times':dataset['timestamps'][i].tolist(),'marks' :dataset['types'][i].tolist()}

        sequences.append(sequence)
    
    itfl_data = {'sequences':sequences,'num_marks':num_marks}
    print('T_min:{}, T_max:{} '.format(t_min,t_max))
    return itfl_data<jupyter_output><empty_output><jupyter_text>Stackoverflow<jupyter_code>dataset = 'stackoverflow'
process_dim =22 

train_path = 'data/' + dataset + '/train_manifold_format.pkl'
dev_path = 'data/' + dataset + '/dev_manifold_format.pkl'
test_path = 'data/' + dataset + '/test_manifold_format.pkl'

with open(train_path, 'rb') as f:
    train_hawkes_data = pickle.load(f)
with open(dev_path, 'rb') as f:
    dev_hawkes_data = pickle.load(f)
with open(test_path, 'rb') as f:
    test_hawkes_data = pickle.load(f)

    
train_data = convert_sahp_data_format_to_itfl(train_hawkes_data,process_dim,t_end = 1390,t_start=1325)
dev_data = convert_sahp_data_format_to_itfl(dev_hawkes_data,process_dim,t_end = 1390,t_start=1325)
test_data = convert_sahp_data_format_to_itfl(test_hawkes_data,process_dim,t_end = 1390,t_start=1325)


with open('data/stackoverflow/test.pkl', 'wb') as handle:
    pickle.dump(test_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/stackoverflow/valid.pkl', 'wb') as handle:
    pickle.dump(dev_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/stackoverflow/train.pkl', 'wb') as handle:
    pickle.dump(train_data, handle, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output>T_min:1325.3760986328125, T_max:1388.5341796875 
T_min:1325.3760986328125, T_max:1388.5341796875 
T_min:1325.3795166015625, T_max:1388.5323486328125<jupyter_text>Mimic 2<jupyter_code>dataset = 'mimic'
process_dim =75 

train_path = 'data/' + dataset + '/train_manifold_format.pkl'
dev_path = 'data/' + dataset + '/dev_manifold_format.pkl'
test_path = 'data/' + dataset + '/test_manifold_format.pkl'

with open(train_path, 'rb') as f:
    train_hawkes_data = pickle.load(f)
    
with open(dev_path, 'rb') as f:
    dev_hawkes_data = pickle.load(f)
with open(test_path, 'rb') as f:
    test_hawkes_data = pickle.load(f)
    
    
    
train_data = convert_sahp_data_format_to_itfl(train_hawkes_data,process_dim,t_end = 6)
dev_data = convert_sahp_data_format_to_itfl(dev_hawkes_data,process_dim,t_end = 6)
test_data = convert_sahp_data_format_to_itfl(test_hawkes_data,process_dim,t_end = 6)

with open('data/mimic/test.pkl', 'wb') as handle:
    pickle.dump(test_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/mimic/valid.pkl', 'wb') as handle:
    pickle.dump(dev_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/mimic/train.pkl', 'wb') as handle:
    pickle.dump(train_data, handle, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output>T_min:0.0, T_max:5.615384578704834 
T_min:0.0, T_max:5.884615421295166 
T_min:0.0, T_max:5.42307710647583<jupyter_text>Retweet<jupyter_code>dataset = 'retweet'
process_dim =3 

train_path = 'data/' + dataset + '/train_manifold_format.pkl'
dev_path = 'data/' + dataset + '/dev_manifold_format.pkl'
test_path = 'data/' + dataset + '/test_manifold_format.pkl'

with open(train_path, 'rb') as f:
    train_hawkes_data = pickle.load(f)
with open(dev_path, 'rb') as f:
    dev_hawkes_data = pickle.load(f)
with open(test_path, 'rb') as f:
    test_hawkes_data = pickle.load(f)

    
train_data = convert_sahp_data_format_to_itfl(train_hawkes_data,process_dim,t_end = 604800,t_start=0)
dev_data = convert_sahp_data_format_to_itfl(dev_hawkes_data,process_dim,t_end = 604800,t_start=0)
test_data = convert_sahp_data_format_to_itfl(test_hawkes_data,process_dim,t_end = 604800,t_start=0)


with open('data/retweet/test.pkl', 'wb') as handle:
    pickle.dump(test_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/retweet/valid.pkl', 'wb') as handle:
    pickle.dump(dev_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('data/retweet/train.pkl', 'wb') as handle:
    pickle.dump(train_data, handle, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output>T_min:1.0, T_max:604798.0 
T_min:1.0, T_max:604786.0 
T_min:1.0, T_max:604739.0
{'fraction_non_alphanumeric': 0.1431087626596213, 'fraction_numerical': 0.04579480405107882, 'mean_word_length': 5.848241206030151, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '269723', 'license': 'MIT', 'n_tokens_mistral': 2920, 'n_tokens_neox': 2706, 'n_words': 406, 'path': 'Modify Datasets.ipynb', 'repo_name': 'yamacisik/ifl-tpp', 'size': '6813'}