<jupyter_start><jupyter_text>Necessary imports<jupyter_code># Importing the classes from 'gydelt'
from gydelt.gydelt import GetData, ProcessData 

# Creating object of GetData
GD = GetData()<jupyter_output><empty_output><jupyter_text>Reading from a file<jupyter_code># Reading from a csv file (obtained from GKG Exporter)
data = GD.read_from_file(path='sample data/fromGKG.txt', parse_dates=['Date'])
data.head(5)<jupyter_output><empty_output><jupyter_text>Preprocessing the data using the wrapper function present in the ProcessData<jupyter_code># Creating object of ProcessData
PD = ProcessData(data_frame=data)
# Calling the wrapper function to pre-process the whole data
processed_data_1 = PD.pre_process()
processed_data_1.head(5)<jupyter_output>Time taken for pre-processing the data --> 0.49 seconds<jupyter_text>Flattening a column<jupyter_code># Flattening(One-hot Encoding) the 'Countries' column
processed_data_2 = PD.flat_column(columns=['Countries'])
processed_data_2.head(5)<jupyter_output>Time taken for flattening the column(s) --> 0.58 seconds<jupyter_text>Sample Usecase - To plot the average 'Tone' over a month for articles with mentions of both countries - India and China<jupyter_code># How can flattening(one-hot encoding) be useful ?
include_India = processed_data_2['India'] == 1
include_China = processed_data_2['China'] == 1

# It makes filtering way more simpler (in case of the data that GDELT provides)
required_data = processed_data_2[include_India & include_China]

print('\n{} records have the mentions of both India and China'.format(required_data.shape[0]))
required_data.head(5)
# Also, since each component of 'ToneData' has been seperated, visualization becomes a lot more simpler
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
%matplotlib inline

# Resampling the 'Tone' (first component of 'ToneData') on Monthly basis
resampled = required_data[['Date', 'Tone']].resample(rule='M', on='Date').apply(np.mean)

# Plotting the resampled data
resampled.plot(figsize=(10, 7), kind='line')<jupyter_output><empty_output><jupyter_text>Saving the processed data<jupyter_code># Saving the data frame (using the function in ProcessData)
PD.save_data_frame()

# (or)
# GD.save_data_frame(data_frame=processed_data_2)<jupyter_output>Success: Data frame saved as - C:\Users\<NAME>\gydelt\tutorial\Result(2017-08-29 14.52.09).csv
{'fraction_non_alphanumeric': 0.10260723296888141, 'fraction_numerical': 0.0159798149705635, 'mean_word_length': 7.1194539249146755, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '457308', 'license': 'MIT', 'n_tokens_mistral': 825, 'n_tokens_neox': 776, 'n_words': 217, 'path': 'tutorial/Example.ipynb', 'repo_name': 'MrinalJain17/gydelt', 'size': '2383'}