<jupyter_start><jupyter_text>In the next cell we have an example of reading the content of 1 json file and displaying ID of each tweet<jupyter_code>with open('Arabic_tweets_negative_20190413.json') as json_test:
    for line in json_test:
        json_text_test = json.loads(line)
        print(json_text_test['id'])
        print(index)
with open('Arabic_tweets_negative_20190413.json') as json_test:
    for index, line in enumerate(json_test):
        json_text_test = json.loads(line)
        print(json_text_test['id'])
        print(index)<jupyter_output><empty_output><jupyter_text>Here we create a dataframe from 1 json file, where we have ID,Text,createdAt<jupyter_code>import pandas as pd
import re
df_test_one = pd.DataFrame(columns=['id', 'text', 'createdAt'])

with open('Arabic_tweets_negative_20190413.json') as json_test:
    for index, line in enumerate(json_test):
        json_text_test = json.loads(line)
        id_test = json_text_test['id']
        text_test = json_text_test['text']
        created_at_test = json_text_test['created_at']
        
        df_test_one.loc[index] = [id_test,text_test,created_at_test]
        
df_test_one
tweets_df = pd.DataFrame(columns=['id', 'text', 'createdAt'])
tweet_count_total=0
for file_index, js in enumerate(json_files):
    tweet_count_per_file=0
    with open(os.path.join(path_to_json, js)) as json_file:
        print(f"Opened File {file_index}")
        for tweet_index, line in enumerate(json_file):
            tweet = json.loads(line)
            tweet_id = tweet['id']
            #here we do some cleaning for text, keeping only arabic letters and numbers and 1 whitespace
            tweet_text = re.sub(r'[^0-9\u0600-\u06ff\u0750-\u077f\ufb50-\ufbc1\ufbd3-\ufd3f\ufd50-\ufd8f\ufd50-\ufd8f\ufe70-\ufefc\uFDF0-\uFDFD- ]+', '', tweet['text'])
            tweet_created_at = tweet['created_at']
            
            tweets_df.loc[tweet_count_total] = [tweet_id,tweet_text,tweet_created_at]
            
            tweet_count_total+=1
            tweet_count_per_file+=1
        print(f"Tweet count for file {file_index} is  : {tweet_count_per_file} ")
        print(f"Total number of tweets : {tweet_count_total}")
        print(f"Ended file {file_index}\n ")
        index_current_file = tweet_count_total - tweet_count_per_file
        tweets_df.iloc[ index_current_file:tweet_count_total ,:].to_csv(''.join(["./unzipped_dataset/dataset_csv/",str(js), '_cleaned','.csv']))
        print("Wrote dataframe of file {file_index} to CSV")

        
tweets_df.to_csv('./unzipped_dataset/dataset_csv/tweets_drop_cols_cleaned_text.csv')
tweets_df
tweets_df.to_csv('./unzipped_dataset/dataset_csv/tweets_drop_cols_cleaned_text.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11257309941520467, 'fraction_numerical': 0.021929824561403508, 'mean_word_length': 3.7189655172413794, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '179905', 'license': 'Unlicense', 'n_tokens_mistral': 1022, 'n_tokens_neox': 978, 'n_words': 172, 'path': '1-Data_Collection/Arabic_Tweets_Dataset_Creation.ipynb', 'repo_name': 'belhajManel/News-Prediction-And-Classification-In-Arabic', 'size': '2736'}