<jupyter_start><jupyter_text>Load unique tweet tokens from file and pad them to match the same size<jupyter_code>import pandas as pd
import numpy as np
import time
import gzip<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>N_ROWS = 50000
PAD = "\t0"<jupyter_output><empty_output><jupyter_text>Read from file the maximum number of tokens per tweet<jupyter_code>with open("tweet_tokens/max_tweet_length.txt", "r") as f:
    max_tweet_length = int(f.readline())
max_tweet_length = 280
max_tweet_length<jupyter_output><empty_output><jupyter_text>Pad all the tweets to match with length = max_length and save them![new_bert_cazzeggio.ipynb](attachment:new_bert_cazzeggio.ipynb)<jupyter_code>TWEET_ID = "tweet_features_tweet_id"
TWEET_TOKENS = "tweet_features_text_tokens"

TOKENS_FILE = "../tweet_tokens/subsample/text_tokens_clean_days_1_unique.csv"
RESULT_PATH = "../tweet_tokens/subsample/text_tokens_clean_days_1_unique_padded.csv"

MAX_TWEET_ID = 73549798
result_file = open(RESULT_PATH, "w+")<jupyter_output><empty_output><jupyter_text>Write the header (columns names)<jupyter_code>result_file.write(TWEET_ID + ',' + TWEET_TOKENS + "\n")<jupyter_output><empty_output><jupyter_text>Open files to be read and perform the padding (and save the results)<jupyter_code>tokens_file = open(TOKENS_FILE, "r")
def save_tweet(index, text):
    line = index + ','
    for item in text:
        line += item 
%%time

# ABOUT ~1h 50min MINUTES EXECUTION ON THE WHOLE DATASET

# COMMENTED CODE IS FOR DEBUGGING (TO CHECK THAT ALL THE OUTPUTS HAVE THE SAME LENGTH)

# ignore header
line = tokens_file.readline()

finished = False
i = 0

while not finished:  # and i < 10:
    
    if i % 1000000 == 0:
        print(i)
    
    line = str(tokens_file.readline())
    
    if line != '':
        
        #print(line)
        
        line = line.replace("\\n",'').replace('\n','').replace("\\t",'\t')
        splitted_line = line.split(',')
        splitted_tokens = splitted_line[1].split('\t')  # take only the tokens (splitted_line[0] is the tweet id)
        
        #splitted_line = []
        for j in range(len(splitted_tokens), max_tweet_length):
            line += PAD
            #splitted_line.append(PAD)
        
        line += '\n'
        
        #if (len(splitted_line)+len(splitted_tokens) != max_tweet_length):
        #    print('Error')
        #    break
        
        result_file.write(line)
        
    else:
        finished = True

    i += 1
tokens_file.close()

result_file.close()<jupyter_output><empty_output><jupyter_text>Check the dataset<jupyter_code>%%time

df = pd.read_csv(RESULT_PATH,
                    names=[TWEET_ID],
                    #compression='gzip',
                    #nrows=N_ROWS,
                    header=0,
                    index_col=0)
df
df = pd.read_csv(TWEET_ID_FILE,
                    names=[TWEET_ID],
                    compression='gzip',
                    #nrows=N_ROWS,
                    header=0,
                    index_col=0)
len(df[TWEET_ID].unique())<jupyter_output><empty_output><jupyter_text>Older code for padding using pandas (should be no more useful)<jupyter_code>def save_chunk(chunk, path):
    chunk.to_csv(path, 
                 columns=[COLUMN_NAME],
                 header=None,
                 mode='a',
                 compression='gzip')
%%time

for chunk in pd.read_csv(PATH,
                            chunksize=CHUNKSIZE,
                            names=[TWEET_ID],
                            compression='gzip',
                            nrows=N_ROWS,
                            header=0,
                            index_col=0):
    
    for row in chunk[TWEET_TOKENS]:

        for i in range(len(row), max_tweet_length):
            row.append(PAD)

        if (len(row) != max_tweet_length):
            print('Error')
            break
dataframe.to_csv(PATH_TWEETS_PADDED,
                 header=None,
                 mode='w',
                 compression='gzip')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10747083643584016, 'fraction_numerical': 0.010424422933730455, 'mean_word_length': 2.4473909324208725, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '341050', 'license': 'Apache-2.0', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1344, 'n_words': 253, 'path': 'BERT/Notebooks/pad_tweet_tokens.ipynb', 'repo_name': 'MaurizioFD/recsys-challenge-2020-twitter', 'size': '4029'}