<jupyter_start><jupyter_text>POSTS<jupyter_code># remover os posts removidos
dataPosts = dataPosts[dataPosts.selftext != '[removed]']
# remover os posts deletados
dataPosts = dataPosts[dataPosts.selftext != '[deleted]']

# remover os posts dos autores removidos
dataPosts = dataPosts[dataPosts.author != '[removed]']
# remover os posts dos autores  deletados
dataPosts = dataPosts[dataPosts.author != '[deleted]']
# tirar o "\n" do texto selftext e body
dataPosts['selftext'].replace(regex=True,inplace=True,to_replace=r'\n',value=r'')<jupyter_output><empty_output><jupyter_text>Comments<jupyter_code># remover os comentarios removidos
dataComments = dataComments[dataComments.body != '[removed]']
# remover os comentrios deletados
dataComments = dataComments[dataComments.body != '[deleted]']

# remover os coments dos autores removidos
dataComments = dataComments[dataComments.author != '[removed]']
# remover os comments dos autores  deletados
dataComments = dataComments[dataComments.author != '[deleted]']
dataComments['body'].replace(regex=True,inplace=True,to_replace=r'\n',value=r'')
dataComments['link_id'].replace(regex=True,inplace=True,to_replace='t3_',value='')
dataComments["type"] = "comments"
dataPosts["type"] = "posts"<jupyter_output><empty_output><jupyter_text>Remove stopwords "body" and "selfText"<jupyter_code>import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = list(stopwords.words('english')) 
type(stop_words)
stop_words.append("im")
stop_words
# remove pontuação
dataComments["body"] = dataComments["body"].str.replace('[^\w\s]',' ')
dataPosts["selftext"] = dataPosts["selftext"].str.replace('[^\w\s]',' ')
#remover stopwords
dataComments['body_semStopWords'] = dataComments['body'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop_words)]))
dataPosts["selftext_semStopWords"] = dataPosts["selftext"].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop_words)]))
# remover execesso de espaços
dataComments['body_semStopWords'] = dataComments['body_semStopWords'].str.strip()<jupyter_output><empty_output><jupyter_text>Eliminar palavras que deram problema no RMN<jupyter_code>#NAN
dataComments["body_semStopWords"] = dataComments["body_semStopWords"].str.lower().str.replace(r"(\snan\s)|(^nan\s)|(\snan$)|(^nan$)", ' ')
len(dataComments[dataComments["body_semStopWords"].str.lower().str.contains(r"(\snan\s)|(^nan\s)|(\snan$)|(^nan$)")])

#NA
dataComments["body_semStopWords"] = dataComments["body_semStopWords"].str.lower().str.replace(r"(\sna\s)|(^na\s)|(\sna$)|(^na$)", ' ')
len(dataComments[dataComments["body_semStopWords"].str.lower().str.contains(r"(\sna\s)|(^na\s)|(\sna$)|(^na$)")])

#null
dataComments["body_semStopWords"] = dataComments["body_semStopWords"].str.lower().str.replace(r"(\snull\s)|(^null\s)|(\snull$)|(^null$)", ' ')
len(dataComments[dataComments["body_semStopWords"].str.lower().str.contains(r"(\snull\s)|(^null\s)|(\snull$)|(^null$)")])
dataComments.isnull().values.any()
#NAN
dataPosts["selftext_semStopWords"] = dataPosts["selftext_semStopWords"].str.lower().str.replace(r"(\snan\s)|(^nan\s)|(\snan$)|(^nan$)", ' ')
len(dataPosts[dataPosts["selftext_semStopWords"].str.lower().str.contains(r"(\snan\s)|(^nan\s)|(\snan$)|(^nan$)")])

#NA
dataPosts["selftext_semStopWords"] = dataPosts["selftext_semStopWords"].str.lower().str.replace(r"(\sna\s)|(^na\s)|(\sna$)|(^na$)", ' ')
len(dataPosts[dataPosts["selftext_semStopWords"].str.lower().str.contains(r"(\sna\s)|(^na\s)|(\sna$)|(^na$)")])

#Null
dataPosts["selftext_semStopWords"]= dataPosts["selftext_semStopWords"].str.lower().str.replace(r"(\snull\s)|(^null\s)|(\snull$)|(^null$)", ' ')
len(dataPosts[dataPosts["selftext_semStopWords"].str.lower().str.contains(r"(\snull\s)|(^null\s)|(\snull$)|(^null$)")])
dataPosts.isnull().values.any()<jupyter_output><empty_output><jupyter_text>Tirar textos != de ingles<jupyter_code># não achei nenhum algorimo bom e acredito q não vai fazer muito diferença para os descritores, uma vez que os textos são a maioria<jupyter_output><empty_output><jupyter_text>Remover acentos<jupyter_code>import unicodedata
# remover do comentarios
dataComments["body_semStopWords"] = dataComments["body_semStopWords"].str.normalize('NFKD').str.encode('ascii', errors='ignore').str.decode('utf-8')
# remover dos posts
dataPosts["selftext_semStopWords"] = dataPosts["selftext_semStopWords"].str.normalize('NFKD').str.encode('ascii', errors='ignore').str.decode('utf-8')<jupyter_output><empty_output><jupyter_text>Posts e Comentários por usuário    Acredito que isso não é necessário.<jupyter_code># autores únicos das postagens
#authorPosts = dataPosts.author.unique()
# autores únicos dos comentarios
#authorComments = dataComments.author.unique()
#authorsIntesec = set.intersection(set(authorPosts),set(authorComments))
#for a in authorsIntesec:
    #dataPosts[(dataPosts.author == a)].to_csv("depression_2017.txt", ";", columns=headerPost, mode='a', header=False)
    #dataComments[(dataComments.author == a)].to_csv("depression_2017.txt", ";", columns=headerCom, mode='a', header=False)<jupyter_output><empty_output><jupyter_text>Gerar Arquivos<jupyter_code># remover linhas vazias
dataPosts = dataPosts.drop(dataPosts[(dataPosts.selftext_semStopWords.str.strip() == '')].index)
dataComments = dataComments.drop(dataComments[(dataComments.body_semStopWords.str.strip() == '')].index)
nameFile = subreddit + "_2017.txt"
file = open(nameFile,"w") 
file.write("subs;user;spans;masks")
file.write("\n")
file.close()
headerPost = ['subreddit', 'author','selftext_semStopWords','type']
headerCom =  ['subreddit', 'author', 'body_semStopWords','type']
dataPosts.to_csv(nameFile, ";", columns=headerPost, mode='a', header=False, index=False)
dataComments.to_csv(nameFile, ";", columns=headerCom, mode='a', header=False, index=False)
file.close()
len(dataComments)
len(dataPosts)
#fileSpans = "spans_2017"
#file = open(fileSpans,"w") 
#dataPosts.to_csv(fileSpans, "\n", columns=["selftext_semStopWords"], mode='a', header=False, index=False)
#dataComments.to_csv(fileSpans , "\n", columns=["body_semStopWords"], mode='a', header=False, index=False)
#file.close()<jupyter_output><empty_output><jupyter_text>métricas<jupyter_code>dataPosts.head()
dicioComments = dict(dataComments.groupby("author")['body_semStopWords'].count())
dicioPosts  = dict(dataPosts.groupby("author")['selftext_semStopWords'].count())
# comentarios
contComment = 0
for item in sorted(dicioComments, key = dicioComments.get,  reverse=True):
    #print (dicioComments[item])
    if(dicioComments[item] >= 50):
        contComment = contComment + 1
print("Qtde Comments >= 50 = ", contComment)
# posts
contPosts = 0
for item in sorted(dicioPosts, key = dicioPosts.get):
    #print (dicioPosts[item])
    if(dicioPosts[item] >= 50):
        contPosts = contPosts + 1
print("Qtde Posts >= 50 = ", contPosts)
# funciona, só nao faz filter ainda..
# commentsGroup = dataComments.groupby("author")['body_semStopWords'].count()
# postsGroup = dataPosts.groupby("author")['selftext_semStopWords'].count()
#postsGroup = dataPosts.groupby("author")['selftext_semStopWords'].count()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1878391540281063, 'fraction_numerical': 0.0043133435369417, 'mean_word_length': 8.941908713692946, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '107635', 'license': 'MIT', 'n_tokens_mistral': 2711, 'n_tokens_neox': 2528, 'n_words': 357, 'path': 'src/1-Tratar Dados/Tratamento_Dados_Reddit.ipynb', 'repo_name': 'barbarasilveiraf/rmn-id-post', 'size': '7187'}