<jupyter_start><jupyter_text>import test file and transform for model<jupyter_code>test  = pd.read_json("/Users/romainbrule/Documents/Cours/Master2/KAGGLE/stanford-covid-vaccine/test.json",lines=True)
def expend_feature_test(df):
    """
    Return a dataframe with expension of sequence for test set prediction 
    
    Args:
        df (Dataframe): same format as train
        
    Returns:
        sub_df: a dataframe with: number of rows = seq_scored 
        columns name = [id, base, base_structure_type, base_predicted_loop_type]
    """
    if (df.shape[0] != df.id.nunique()):
        print('repetition in RNA sequnence, clean dataframe first')
        return # dose the same as retunr None, which exit the function 
    
    else:
        col_names = ['id','id_seqpos','base', 'base_structure_type', 'base_predicted_loop_type']

        #dataframe creation using list of lists
        # loop for each unique sequence
        sub_data = []
        for row_i in df.index:
            #loop for the legth of sequnece score (trian length is different from test)
            serie_i = df.loc[row_i] #panda series object
            seq_length  = serie_i['seq_length']
            for seq_i in range (seq_length):
                seq_data = [serie_i['id'],serie_i['id'] + '_' + str(seq_i), 
                            serie_i['sequence'][seq_i],
                            serie_i['structure'][seq_i], serie_i['predicted_loop_type'][seq_i]] 
                sub_data.append(seq_data)

    sub_df = pd.DataFrame(sub_data, columns =col_names,  dtype = float) 
    return sub_df
    
def count(x,colonne):
    return (colonne==x).sum()

def add_features(seq): # Coumpt the numerous of the group
    # Concerning nucleotides
    gr_base=1
    seq['count_gr_base']=gr_base
    for i in range (1,seq.shape[0]):
        if ((seq.id.iloc[i]!=seq.id.iloc[i-1]) or (seq.base.iloc[i]!=seq.base.iloc[i-1])):
            gr_base=gr_base+1
        seq.count_gr_base[i]=gr_base
    #Count the number of nucleotide which are the same 
    seq['N_base']=seq['count_gr_base'].apply(lambda x: count(x,seq['count_gr_base']))
    
    # Concerning loop_type
    gr_loop_type=1
    seq['count_gr_loop_type']=gr_loop_type
    for i in range (1,seq.shape[0]):
        if ((seq.id.iloc[i]!=seq.id.iloc[i-1]) or (seq.base_predicted_loop_type.iloc[i]!=seq.base_predicted_loop_type.iloc[i-1])):
            gr_loop_type=gr_loop_type+1
        seq.count_gr_loop_type[i]=gr_loop_type
    #Count the number of letters in the group 
    seq['N_loop_type']=seq['count_gr_loop_type'].apply(lambda x: count(x,seq['count_gr_loop_type']))
    
    #Concerning the number of point in 'base_structure_type'
    Number_point=seq[seq.base_structure_type=='.'].groupby(["id"])['base_structure_type'].count()
    seq['N_point']=Number_point[0]
    idx=0
    for i in range (1,seq.shape[0]):
        if (seq.id.iloc[i]!=seq.id.iloc[i-1]):
            idx+=1
        seq.loc[i,['N_point']]=Number_point[idx]
    return seq

#changes, plus besoin du filtre SN 
def fianle_transform_test (dataframe):

    #use expend to change feature 
    data_filter_seq = expend_feature_test(dataframe)
    
    #add feature 
    data_filter_seq = add_features(data_filter_seq)
    
    #make feature onehot encoding
    one_hot_base = pd.get_dummies(data_filter_seq['base'],drop_first=True) #drop "A"
    one_hot_loop = pd.get_dummies(data_filter_seq['base_predicted_loop_type'],drop_first=True) # drop "B"
    one_hot_struct = pd.get_dummies(data_filter_seq['base_structure_type'],drop_first=True)# drop "("
    
    #final dataset
    data_filter_seq = data_filter_seq.drop(["base", "base_predicted_loop_type","base_structure_type"], axis = 1)
    df = pd.concat([data_filter_seq,one_hot_base, one_hot_loop ,one_hot_struct], axis = 1)
    return df

seq_test = fianle_transform_test(test)
#seq_test.to_csv('test_interm.csv', index=False)
letters_nucleotide_1=['C','G','U']

def ajout_N_nucleotide(data):
    for i in letters_nucleotide_1:
        col = np.where(data[i]==1, data['N_base'], 0) 
        data ["N_base_"+i] = col
    col_A=np.where(data.N_base_C+data.N_base_G+data.N_base_U==0,data.N_base,0)
    data["N_base_A"]=col_A
    return data

letters_loop_type_1=['E','H','I','M','S','X']

def ajout_N_predicted_loop(data):
    for i in letters_loop_type_1:
        col = np.where(data[i]==1, data['N_loop_type'], 0) 
        data ["N_"+i] = col
    col_B=np.where(data.N_E+data.N_H+data.N_I+data.N_M+data.N_S+data.N_X==0,data.N_loop_type,0)
    data["N_B"]=col_B
    return data
seq_test=ajout_N_nucleotide(seq_test)
seq_test=ajout_N_predicted_loop(seq_test)
seq_test.to_csv('test_final.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13123135918193438, 'fraction_numerical': 0.007243289305496378, 'mean_word_length': 3.5939334637964775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '646935', 'license': 'Apache-2.0', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1727, 'n_words': 305, 'path': 'datas/Test_file.ipynb', 'repo_name': 'DataCampM2DSSAF/suivi-du-data-camp-Sun-Brul-', 'size': '4694'}