<jupyter_start><jupyter_text>Data analysis for Almentor Facebook Data   read required libraries<jupyter_code>import pandas as pd
import seaborn as sns
import regex as re
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import datetime<jupyter_output><empty_output><jupyter_text>Read and optemize files  reading text data file<jupyter_code>## open the file and read it 
with open("C:\\Users\\20812018100700\\working\\work\\rd_part\\result_full.txt", encoding='utf-8')as f:
    result = f.read()
## split .txt file 
result = re.split("done [0-9$,.%]+\d*\n", result)<jupyter_output><empty_output><jupyter_text>reading orignal data frame<jupyter_code>## read DataFrame
df = pd.read_csv("C:\\Users\\20812018100700\\working\\work\\rd_part\\test2_data.csv")
## drop null values
df = df[df["message"].notnull()]
## reset the index of DF
df.reset_index(drop=True, inplace=True)
## slice the DF to the len of our data
df = df.iloc[:len(result), :]
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 15200 entries, 0 to 15199
Data columns (total 4 columns):
 #   Column        Non-Null Count  Dtype 
---  ------        --------------  ----- 
 0   created_time  15200 non-null  object
 1   message       15200 non-null  object
 2   id            15200 non-null  object
 3   from          91 non-null     object
dtypes: object(4)
memory usage: 475.1+ KB<jupyter_text>extract values from text file<jupyter_code>## store values in dict 
data = {
        "Translate": [],
        "is_human" : [],
        "sentment" : [],
        "positive" : [], 
        "neutral"  : [],
        "negative" : []
        }
## loop for each value in result list extract is_human 
## if not human: will extract the Translation, Sentment overall rank, sentment score
## else the sentment is nutral/no translation
for i in range(len(result)): 
    iz = re.search('(?<=is human: )(.*)', result[i]).groups()[0]
    data["is_human"].append(iz[0])
    if (iz == "True"):
        data["sentment"].append("neutral")
        data["Translate"].append(result[i].split(" \n\n")[0])
        h = ['positive=0.00', 'neutral=1.00', 'negative=0.00 ']
        for j in h:
            s = j.split("=")
            data[s[0]].append(s[1])        
                                   
    else:                          
        m = re.search('(?<=Document Sentiment: )(.*)', result[i]).groups()
        data["sentment"].append(m[0])
        
        n = re.search('(?<=Translate: )(.*)', result[i]).groups()
        data["Translate"].append(n[0])
        
        h = str(re.search('(?<=Overall scores: )(.*)', result[i]).groups()[0]).split("; ")
        for j in h:
            s = j.split("=")
            data[s[0]].append(s[1])
        
## convert dict to dataFrame
df1= pd.DataFrame(data)
df1.sample(10)<jupyter_output><empty_output><jupyter_text>merge the 2 dataframes to 1 df<jupyter_code>## merge 2 dataFrames into the index
ndf = pd.merge(df, df1, left_index=True, right_index=True)
ndf.head(25)<jupyter_output><empty_output><jupyter_text>save dataframe to .csv file<jupyter_code>ndf.to_csv("concatnated_DF", index=False)<jupyter_output><empty_output><jupyter_text>read the data again<jupyter_code>ndf = pd.read_csv("C:\\Users\\20812018100700\\working\\work\\rd_part\\concatnated_DF.csv")<jupyter_output><empty_output><jupyter_text>explore some data<jupyter_code>## extract date from created_time
ndf["created_time"] = ndf["created_time"].str.slice(0, 7)
ndf["created_time"]
ndf.is_human.unique()
ndf.query('is_human =="N" and sentment == "negative"').sample(10)<jupyter_output><empty_output><jupyter_text>explore and fix the data<jupyter_code>## drop unused col
ndf.drop("from", axis=1, inplace = True)
ndf.info()
ndf["created_time"] = ndf["created_time"].str.slice(0, 7)
ndf["created_time"] = pd.to_datetime(ndf["created_time"])
ndf["date"] = ndf["created_time"].dt.date
ndf.info()
ndf.query('date == datetime.date(2021, 9, 1)')
ndf.drop(index=17, inplace = True)
ndf.reset_index(drop=True, inplace=True)
ndf.sentment.unique()
## convert rate ti float
ndf["positive"] = ndf["positive"].astype('float')
ndf["neutral"] = ndf["neutral"].astype('float')
ndf["negative"] = ndf["negative"].astype('float')
## checking distripution of the rate
ndf.positive.describe()
ndf.negative.describe()
plot = plt.subplots(figsize = (16,10))
plot = sns.countplot(data = ndf, x ="date",hue = "sentment", palette = 'Paired')
_ = plt.xticks(rotation=45)<jupyter_output><empty_output><jupyter_text>select non-human comments<jupyter_code>## select not human data to explore
info_df = ndf.query('is_human != "T"')
print(info_df.info())
info_df.head()
## select non natural comments
info_df = ndf.query('neutral <0.5')
info_df.reset_index(drop=True, inplace=True)
print(info_df.info())
info_df.head()
info_df.positive.describe()
info_df.negative.describe()
fig, plot = plt.subplots(figsize = (16,10))
plot = sns.lineplot(data=info_df, x="created_time", y="negative", color = "r", alpha = 0.5)
plot = sns.lineplot(data=info_df, x="created_time", y="positive", color = "g", alpha = 0.5)
plt.title('rate ber month')
plt.ylabel("value")
fig, plot = plt.subplots(figsize = (16,10))
plt.scatter(x = info_df["negative"], y =info_df["positive"] )<jupyter_output><empty_output><jupyter_text>the corelation between positivity and negativity is linear :O   lets check what happen in 5/2021<jupyter_code>negative_df = info_df.query('date == datetime.date(2021, 5, 1) and sentment == "negative"')
negative_df.sample(10)<jupyter_output><empty_output><jupyter_text>more than 80% of comment about "Gaza" and considerd as negative  extract "Gaza" commetns and fix the rate<jupyter_code>negative_df = negative_df[negative_df["message"].str.find("#GazaUnderAttack")!= -1]
negative_df<jupyter_output><empty_output><jupyter_text>replace old and new values<jupyter_code>n_negative = negative_df["positive"].copy()
n_positive =  negative_df["negative"].copy()
negative_df["negative"] = n_negative
negative_df["positive"] = n_positive
negative_df["sentment"] = "positive"
negative_df<jupyter_output><empty_output><jupyter_text>readd data to orignal DataFrame<jupyter_code>info_df.loc[negative_df.index, :] = negative_df.copy()
info_df.loc[negative_df.index, :]<jupyter_output>c:\users\20812018100700\appdata\local\programs\python\python39\lib\site-packages\pandas\core\indexing.py:1884: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self._setitem_single_column(loc, val, pi)<jupyter_text>replot the graph<jupyter_code>fig, plot = plt.subplots(figsize = (16,10))
plot = sns.lineplot(data=info_df, x="created_time", y="negative", color = "r", alpha = 0.5)
plot = sns.lineplot(data=info_df, x="created_time", y="positive", color = "g", alpha = 0.5)
plt.title('rate ber month')
plt.ylabel("value")<jupyter_output><empty_output><jupyter_text>check distripution of rate<jupyter_code>fig, plot = plt.subplots(figsize = (11,10))
plot = sns.distplot(x=info_df["positive"] )
plt.title('positive rate distripution')
fig, plot = plt.subplots(figsize = (11,10))
plot = sns.distplot(x=info_df["negative"])
info_df.is_human.unique()
plt.pie(ndf.is_human.value_counts(), labels = ndf.is_human.unique(),
        startangle = 90, counterclock = False, wedgeprops = {'width' : 0.4});
plt.pie(info_df.sentment.value_counts(), labels = info_df.sentment.unique(),
        startangle = 90, counterclock = False, wedgeprops = {'width' : 0.4});
colors = sns.color_palette()
plot = plt.subplots(figsize = (13,8))
plot = sns.barplot(data = info_df, x ="date",  y="positive", color = colors[-1], alpha = 0.5)
plot = sns.barplot(data = info_df, x ="date",  y="negative", color = colors[1], alpha = 0.5)
plt.title("positive/negative rate comparison")
plt.ylabel("Rate")
plt.xlabel("Date")
_ = plt.xticks(rotation=45)
plot = plt.subplots(figsize = (16,10))
plot = sns.countplot(data = info_df, x ="date",hue = "sentment", palette = 'Paired')
plt.title("rate comparison per month")
plt.xlabel("Data")
plt.ylabel("Count")
_ = plt.xticks(rotation=45)
plot = plt.subplots(figsize = (16,10))
plot = sns.countplot(data = ndf,y = "date", color = colors[0])
plt.title("Page interaction per month")
_ = plt.yticks(rotation=45)
ndf.query('sentment == "mixed"').sample()
ndf.query('sentment == "neutral"').sample()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1668236402166235, 'fraction_numerical': 0.027313397692488816, 'mean_word_length': 4.874827109266943, 'pattern_counts': {'":': 1, '<': 69, '<?xml version=': 0, '>': 64, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '13160', 'license': 'MIT', 'n_tokens_mistral': 3249, 'n_tokens_neox': 3044, 'n_words': 611, 'path': 'Almentor/explore_new2.ipynb', 'repo_name': 'HossamAbdelraof/Almentor-Data-Analysis', 'size': '8494'}