analyze\_ahd\_data(df)

### EDA- Univariate/ Bivariate analysis on `requests\_cleaned\_joined` variable 

import pandas as pd
df\_cleaned\_20smpl=pd.read\_csv("cleaned\_df\_20.csv", encoding='ISO-8859-1')
df\_cleaned\_20smpl.tail(3)

df\_cleaned\_20smpl.info()

df\_cleaned\_20smpl.columns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word\_tokenize
from nltk.probability import FreqDist
from wordcloud import WordCloud

def analyze\_ahd\_data(request\_data):
 # Load the text data set as a Pandas DataFrame
 #df = pd.read\_csv("cleaned\_df\_20.csv", encoding='ISO-8859-1')
 df = pd.DataFrame(request\_data, columns=['Category', 'requests\_cleaned\_joined'])
 
 
 # 1. Descriptive statistics
 print(df.describe(include='object'))

 print("\n")
 print("=======================================================")
 print("\n")

 df['text\_data']=df['requests\_cleaned\_joined']

 # 2. Word count, word length, character count, and length

 print('Description variable: Word count, word length, character count, and length')
 print("\n")
 df['word\_count'] = df['text\_data'].apply(lambda x: len(str(x).split()))
 df['word\_length'] = df['text\_data'].apply(lambda x: np.mean([len(word) for word in x.split()]))
 df['char\_count'] = df['text\_data'].apply(lambda x: len(str(x)))
 #df['length'] = df['text\_data'].apply(lambda x: len(str(x).split()))
 print(df[['word\_count', 'word\_length', 'char\_count']].describe())

 print("\n\n")
 print("=======================================================")
 print("\n")

 # 3. Part of speech tagging and counts

 print('Part of speech tagging and counts for Description variable')
 print("\n")
 nltk.download('averaged\_perceptron\_tagger')
 df['pos\_tags'] = df['text\_data'].apply(lambda x: nltk.pos\_tag(word\_tokenize(x)))
 tag\_counts = {}
 for row in df['pos\_tags']:
 for word, tag in row:
 if tag not in tag\_counts:
 tag\_counts[tag] = 1
 else:
 tag\_counts[tag] += 1
 #print(tag\_counts)

 # sort tag\_counts in descending order
 sorted\_tag\_counts = dict(sorted(tag\_counts.items(), key=lambda item: item[1], reverse=True))
 print(sorted\_tag\_counts)

 # plot POS tag counts
 plt.figure(figsize=(12, 6))
 plt.bar(sorted\_tag\_counts.keys(), sorted\_tag\_counts.values(), color='green')
 plt.title('Part of Speech Tagging and Counts')
 plt.xlabel('POS Tag')
 plt.ylabel('Count')
 plt.xticks(rotation=45, ha='right')
 plt.show()

 print("\n")
 print("=======================================================")
 print("\n")

 # 4. 20 most common words and plot

 stop\_words = set(stopwords.words('english'))
 df['text\_data'] = df['text\_data'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in stop\_words]))
 fdist = FreqDist(df['text\_data'].str.cat(sep=' ').split())
 print('20 most common words for Description variable ')
 print("\n")
 print(fdist.most\_common(20))
 print("\n")
 print('20 most common Words distribution plot for Description variable ')

 # plot 20 most common words
 plt.figure(figsize=(12, 6))
 plt.bar([x[0] for x in fdist.most\_common(20)], [x[1] for x in fdist.most\_common(20)], color='green')
 plt.title('20 Most Common Words Distribution')
 plt.xlabel('Words')
 plt.ylabel('Count')
 plt.xticks(rotation=45)
 plt.show()

 print("\n")
 print("=======================================================")

 # 5. Rare words distribution and plot
 rare\_words = fdist.hapaxes()
 print('count of rare word: {}'.format(len(rare\_words)))
 print("\n")
 print('Rare words distribution plot for Description variable ')
 fdist\_rare = FreqDist(rare\_words)

 # plot rare words distribution
 plt.figure(figsize=(12, 6))
 plt.bar([x[0] for x in fdist\_rare.most\_common(20)], [x[1] for x in fdist\_rare.most\_common(20)], color='green')
 plt.xticks(rotation=90)
 plt.title("Rare words distribution plot for Description variable")
 plt.xlabel("Words")
 plt.ylabel("Frequency")
 plt.show()

 print("\n")
 print("=======================================================")

 # 6. Word and character counts by category
 print('Word and character counts by category for Description variable ')
 print("\n")
 category\_counts = df.groupby('Category').agg({'word\_count': 'sum', 'char\_count': 'sum'})

 # plot word and character counts by category
 category\_counts.plot(kind='bar', rot=0, color=['green', 'orange'])
 plt.title("Word and Character Counts by Category for Description variable")
 plt.xlabel("Category")
 plt.ylabel("Count")
 plt.xticks(rotation=45, ha='right')
 plt.show()

 print("\n")
 print("=======================================================")
 print("\n")

 # 7. Word cloud for each category
 print('Category wise Wordcloud ')
 print("\n")

 categories = df['Category'].unique()

 # create subplots for wordclouds
 #fig, axs = plt.subplots(nrows=1, ncols=len(categories), figsize=(20, 8))

 # for i, category in enumerate(categories):
 # text = ' '.join(df[df['Category'] == category]['text\_data'].tolist())
 # wordcloud = WordCloud(width = 800, height = 800,
 # background\_color ='white',
 # stopwords = stop\_words,
 # min\_font\_size = 10).generate(text)
 # axs[i].imshow(wordcloud) 
 # axs[i].axis("off") 
 # axs[i].set\_title(category)
 # plt.tight\_layout(pad=0)
 # plt.show()

 for i in range(0, len(categories), 2):
 plt.figure(figsize=(10, 8))
 for j, category in enumerate(categories[i:i+2]):
 plt.subplot(1, 2, j+1)
 text = ' '.join(df[df['Category'] == category]['text\_data'].tolist())
 wordcloud = WordCloud(width=1000, height=600, background\_color='white', stopwords=stop\_words, min\_font\_size=10).generate(text)
 plt.imshow(wordcloud)
 plt.axis('off')
 plt.tight\_layout(pad=2.25)
 plt.title(category)

 plt.show()
analyze\_ahd\_data(df\_cleaned\_20smpl)
NALYZE\_AHD\_DATA(DF)### EDA- `requests\_cleaned\_joined` 변수에 대한 단변량/이변량 분석 팬더를 PD로 가져오기df\_cleaned\_20smpl=pd.read\_csv("cleaned\_df\_20.csv", encoding='ISO-8859-1')df\_cleaned\_20smpl.tail(3)df\_cleaned\_20smpl.info()df\_cleaned\_20smpl.columns팬더를 pd로 가져 오기numpy를 np로 가져옵니다.matplotlib.pyplot을 plt로 가져옵니다.seaborn을 sns로 가져옵니다.import nltknltk.corpus에서 스톱워드 가져오기nltk.tokenize에서 단어\_tokenize를 가져옵니다.nltk.probability에서 FreqDist를 가져옵니다.워드클라우드에서 워드클라우드 가져오기def analyze\_ahd\_data(request\_data): # 텍스트 데이터 세트를 판다스 데이터 프레임으로 로드합니다. #df = pd.read\_csv("cleaned\_df\_20.csv", encoding='ISO-8859-1') df = pd.DataFrame(request\_data, columns=['Category', 'requests\_cleaned\_joined'])   # 1. 설명 통계 print(df.describe(include='object')) print("\n") print("=======================================================") print("\n") df['text\_data']=df['requests\_cleaned\_joined'] # 2. 단어 수, 단어 길이, 문자 수 및 길이 print('설명 변수: 단어 수, 단어 길이, 문자 수 및 길이') print("\n") df['word\_count'] = df['text\_data'].apply(lambda x: len(str(x).split())) df['word\_length'] = df['text\_data'].apply(lambda x: np.mean([len(word) for word in x.split()]))) df['char\_count'] = df['text\_data'].apply(lambda x: len(str(x))) #df['length'] = df['text\_data'].apply(lambda x: len(str(x).split())) print(df[['word\_count', 'word\_length', 'char\_count']].describe()) print("\n\n") print("=======================================================") print("\n") # 3. 품사 태깅 및 개수 print('설명 변수에 대한 품사 태깅 및 개수') print("\n") nltk.download('평균\_퍼셉트론\_tagger') df['pos\_tags'] = df['text\_data'].apply(lambda x: nltk.pos\_tag(word\_tokenize(x))) tag\_counts = {} for row in df['pos\_tags']: 단어의 경우 행의 태그: tag\_counts에 없는 경우 tag\_counts[tag] = 1 else: tag\_counts[tag] += 1 #인쇄(태그\_counts) # 태그\_counts를 내림차순으로 정렬합니다. sorted\_tag\_counts = dict(sorted(tag\_counts.items(), key=lambda item: item[1], reverse=True)) print(sorted\_tag\_counts) # POS 태그 수 플롯 plt.figure(figsize=(12, 6)) plt.bar(sorted\_tag\_counts.keys(), sorted\_tag\_counts.values(), color='green') plt.title('품사 태깅 및 개수') plt.xlabel('POS 태그') plt.ylabel('Count') plt.xticks(rotation=45, ha='right') plt.show() print("\n") print("=======================================================") print("\n") # 4. 가장 일반적인 20개의 단어와 플롯 stop\_words = set(stopwords.words('english')) df['text\_data'] = df['text\_data'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in stop\_words])) fdist = FreqDist(df['text\_data'].str.cat(sep=' ').split())) print('설명 변수에 대한 가장 일반적인 단어 20개') print("\n") print(fdist.most\_common(20)) print("\n") print('Description 변수에 대한 가장 일반적인 단어 20개 분포도') # 가장 일반적인 단어 20개 플롯 plt.figure(figsize=(12, 6)) plt.bar([x[0] for x in fdist.most\_common(20)], [x[1] for x in fdist.most\_common(20)], color='green') plt.title('20개의 가장 일반적인 단어 분포') plt.xlabel('단어') plt.ylabel('Count') plt.xticks(회전=45) plt.show() print("\n") print("=======================================================") # 5. 희귀 단어 분포 및 플롯 rare\_words = fdist.hapaxes() print('희귀 단어 수: {}'.format(len(rare\_words))) print("\n") print('설명 변수에 대한 희귀 단어 분포도') fdist\_rare = FreqDist(rare\_words) # 희귀 단어 분포 플롯 plt.figure(figsize=(12, 6)) plt.bar([x[0] for x in fdist\_rare.most\_common(20)], [x[1] for x in fdist\_rare.most\_common(20)], color='green') plt.xticks(회전=90) plt.title("설명 변수에 대한 희귀 단어 분포도") plt.xlabel("단어") plt.ylabel("빈도") plt.show() print("\n") print("=======================================================") # 6. 카테고리별 단어 및 문자 수 print('Description 변수에 대한 카테고리별 단어 및 문자 수 ') print("\n") category\_counts = df.groupby('Category').agg({'word\_count': 'sum', 'char\_count': 'sum'}) # 카테고리별 단어 및 문자 수 플롯 category\_counts.plot(kind='bar', rot=0, color=['green', 'orange']) plt.title("설명 변수에 대한 카테고리별 단어 및 문자 수") plt.xlabel("카테고리") plt.ylabel("Count") plt.xticks(회전=45, ha='오른쪽') plt.show() print("\n") print("=======================================================") print("\n") # 7. 각 카테고리에 대한 워드 클라우드 print('카테고리별 워드클라우드') print("\n") categories = df['Category'].unique() # 워드클라우드에 대한 하위 플롯 만들기 #fig, axs = plt.subplots(nrows=1, ncols=len(categories), figsize=(20, 8)) # for i, category in enumerate(categories): # text = ' '.join(df[df['Category']) 