<jupyter_start><jupyter_text>-: Project 3    || <NAME>:- Comcast Telecom Consumer Complaints. Analysis Task :  To perform these tasks, you can use any of the different Python libraries such as NumPy, SciPy, Pandas, scikit-learn, matplotlib, and BeautifulSoup. - Import data into Python environment.<jupyter_code>import numpy as np 
import pandas as pd 
import tensorflow as tf
import matplotlib.pyplot as plt
%matplotlib inline
from patsy import dmatrices
import sklearn
import seaborn as sns
import nltk
from wordcloud import WordCloud, STOPWORDS
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import string
import gensim
from gensim import corpora
from gensim.models import LdaModel
import pyLDAvis.gensim
df_ctcc = pd.read_csv('Comcast_telecom_complaints_data.csv',index_col=0)
df_ctcc.head(6)
df_ctcc[df_ctcc.isnull()].count()
df_ctcc.describe(include='all')
df_ctcc.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 2224 entries, 250635 to 363614
Data columns (total 10 columns):
 #   Column                       Non-Null Count  Dtype 
---  ------                       --------------  ----- 
 0   Customer Complaint           2224 non-null   object
 1   Date                         2224 non-null   object
 2   Date_month_year              2224 non-null   object
 3   Time                         2224 non-null   object
 4   Received Via                 2224 non-null   object
 5   City                         2224 non-null   object
 6   State                        2224 non-null   object
 7   Zip code                     2224 non-null   int64 
 8   Status                       2224 non-null   object
 9   Filing on Behalf of Someone  2224 non-null   object
dtypes: int64(1), object(9)
memory usage: 191.1+ KB<jupyter_text>Analysis Task: - Provide the trend chart for the number of complaints at monthly and daily granularity levels.<jupyter_code>df_ctcc['Date_month_year'] = pd.to_datetime(df_ctcc['Date_month_year'])
df_ctcc['Created_Month'] =  df_ctcc['Date_month_year'].apply(lambda x: x.month)
df_ctcc['Created_Day'] = df_ctcc['Date_month_year'].apply(lambda x: x.day)
df_ctcc['Created_Day of Week'] = df_ctcc['Date_month_year'].apply(lambda x: x.dayofweek)
dmap = {0:'Mon',1:'Tue',2:'Wed',3:'Thur',4:'Fri',5:'Sat',6:'Sun'}
df_ctcc['Created_Day of Week']=df_ctcc['Created_Day of Week'].map(dmap)
df_ctcc.head(7)
#number of complaints monthly
plt.figure(figsize=(15,7))
plt.title('Number of Complaints Monthly\n')
bymonth = df_ctcc.groupby('Created_Month').count().reset_index()
lp = sns.lineplot(x='Created_Month', y= 'Customer Complaint', data = bymonth, sort=False,markers = "o")
ax = lp.axes
ax.set_xlim(0,13)
ax.annotate('Max complaints in Jun', color='red', xy=(6, 1060), xycoords='data', xytext=(0.8, 0.85), textcoords='axes fraction',
            arrowprops=dict(facecolor='black', shrink=0.1), horizontalalignment='right', verticalalignment='top')
#number of complaints Daily
plt.figure(figsize=(20,7))
plt.title('Number of Complaints Daily\n')
byday = df_ctcc.groupby('Created_Day').count().reset_index()
lp = sns.lineplot(x='Created_Day', y= 'Customer Complaint', data = byday, sort=False, color = 'red',markers = "o", )
ax = lp.axes
ax.set_xlim(0,32)
#number of complaints based on created day of the week
sns.countplot(x='Created_Day of Week', data = df_ctcc, order=df_ctcc['Created_Day of Week'].value_counts().index, palette ="Reds_d")<jupyter_output><empty_output><jupyter_text>More number of complaints on Tuesday and wednesday  Analysis Task: - Provide a table with the frequency of complaint types.<jupyter_code>df_ctcc['Customer Complaint'] = df_ctcc['Customer Complaint'].str.title() 
CT_freq = df_ctcc['Customer Complaint'].value_counts()
CT_freq
common_complaints = df_ctcc['Customer Complaint'].dropna().tolist()
common_complaints =''.join(common_complaints).lower()

list_stops = ('Comcast','Now','Company','Day','Someone','Thing','Also','Got','Way','Call','Called','One','Said','Tell')

for word in list_stops:
    STOPWORDS.add(word)
    wordcloud = WordCloud(stopwords=STOPWORDS, background_color='white', width=1200,height=1000).generate(common_complaints)
plt.figure( figsize=(15,15) )
plt.imshow(wordcloud)
plt.title('Frequent words for customer complaints\n')
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>Internet complaints are Maximum<jupyter_code>stop = set(stopwords.words('english'))
exclude = set(string.punctuation)
lemma = WordNetLemmatizer()
nltk.download('wordnet')
def clean(doc):
    stop_free = " ".join([i for i in doc.lower().split() if i not in stop])
    punc_free = "".join([ch for ch in stop_free if ch not in exclude])
    normalized = " ".join(lemma.lemmatize(word) for word in punc_free.split())
    return normalized
doc_complete = df_ctcc['Customer Complaint'].tolist()
doc_clean = [clean(doc).split() for doc in doc_complete]
dictionary = corpora.Dictionary(doc_clean)
dictionary
doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]
doc_term_matrix
num_topic = 9
ldamodel = LdaModel(doc_term_matrix,num_topics=num_topic,id2word = dictionary,passes=10)
topics = ldamodel.show_topics()
for topic in topics:
    print(topic)
    print()
word_dict = {}
for i in range(num_topic):
    words = ldamodel.show_topic(i,topn = 20)
    word_dict['Topic '+"{}".format(i)]=[i[0] for i in words]
pd.DataFrame(word_dict)
Lda_display = pyLDAvis.gensim.prepare(ldamodel,doc_term_matrix,dictionary,sort_topics=False)
pyLDAvis.display(Lda_display)<jupyter_output><empty_output><jupyter_text>Analysis Task: - Create a new categorical variable with value as Open and Closed. Open & Pending is to be categorized as Open and Closed & Solved is to be categorized as Closed.<jupyter_code>df_ctcc['Highlevel_Status'] = ["Open" if Status=="Open" or Status=="Pending" else "Closed" for Status in df_ctcc["Status"]]
df_ctcc['Highlevel_Status'].unique()<jupyter_output><empty_output><jupyter_text>Analysis Task: - Provide state wise status of complaints in a stacked bar chart. Use the categorized variable from Q3.<jupyter_code>df_ctcc['State'] = df_ctcc['State'].str.title() 
st_cmp = df_ctcc.groupby(['State','Highlevel_Status']).size().unstack().fillna(0)
st_cmp
st_cmp.sort_values('Closed',axis = 0,ascending=True).plot(kind="barh", figsize=(20,15), stacked=True)<jupyter_output><empty_output><jupyter_text>Analysis Task: - Which state has the maximum complaints<jupyter_code>df_ctcc.groupby(["State"]).size().sort_values(ascending=False).to_frame().rename({0: "Complaint count"}, axis=1)[:1]
CT = df_ctcc.groupby(["State","Highlevel_Status"]).size().unstack().fillna(0)
CT.sort_values('Closed',axis = 0,ascending=False)[:1]<jupyter_output><empty_output><jupyter_text>Georgia has highest complaints  Analysis Task: - Which state has the highest percentage of unresolved complaints<jupyter_code>#highest percentage of unresolved complaints
CT['Resolved_cmp_prct'] = CT['Closed']/CT['Closed'].sum()*100
CT['Unresolved_cmp_prct'] = CT['Open']/CT['Open'].sum()*100
CT.sort_values('Unresolved_cmp_prct',axis = 0,ascending=False)[:1]<jupyter_output><empty_output><jupyter_text>Georgia state has highest Unresolved complaints when compared to other states  Analysis Task: - Provide the percentage of complaints resolved till date, which were received through the Internet and customer care calls.<jupyter_code>cr = df_ctcc.groupby(['Received Via','Highlevel_Status']).size().unstack().fillna(0)
cr['resolved'] = cr['Closed']/cr['Closed'].sum()*100
cr['resolved']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13322696449940166, 'fraction_numerical': 0.02100784470150246, 'mean_word_length': 5.36917866215072, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '409369', 'license': 'Unlicense', 'n_tokens_mistral': 2661, 'n_tokens_neox': 2474, 'n_words': 584, 'path': 'Project 3 - Comcast Telecom Consumer Complaints.ipynb', 'repo_name': 'Nimlu011/Data-Science-with-Python', 'size': '7536'}