<commit_before># file dedicated to making the tables
# for the main document and the Supporting Information
# of the artigcle about text differentiation:
# https://github.com/ttm/artigoTextoNasRedes/ AND
# http://arxiv.org/abs/1412.7309

# tentar fazer as tabelas focadas nas medidas?

# fazer td com 100 mensagens somente de cada lista

# fazer tabela geral só com CPP
# incluir mensagens/ano na tabela
# tirar as datas para a tabela no corpo do artigo

# tentar paralelizar

# fazer tabela de caracteres soh com LAD
# para o corpo do artigo.
# renderizar 4 listas
# avaliar fazer para mais listas para tirar médias e desvios

import gmane as g, os, pickle, time as T, numpy as n
ENV=os.environ["PATH"]
import  importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.pca)
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
importlib.reload(g.utils)
dreload(g,exclude="pytz")
os.environ["PATH"]=ENV

TT=T.time()
#print("{0:.2f}".format(T.time()-TT)); TT=T.time()
def pDump(tobject,tfilename):
    with open(tfilename,"wb") as f:
        pickle.dump(tobject,f,-1)
def pRead(tfilename):
    with open(tfilename,"rb") as f:
        tobject=pickle.load(f)
    return tobject

#dl=g.DownloadGmaneData('~/.gmane3/')
#dl.downloadedStats()
PDIR="pickledir/"
#pDump(dl,"{}dl.pickle".format(PDIR))
dl=pRead("{}dl.pickle".format(PDIR))

###### DATA STRUCTURES
TOTAL_M=400
dss=[]; iNs=[]; nms=[]; tss=[]; nps=[]
PDIR="pickledir/"
ES=[]
for lid in dl.lists[14:18]:
    lid=lid[0]
    es=g.EmailStructures(lid,TOTAL_M)
    ES.append(es)
#pDump(ES,"{}ES{}.pickle".format(PDIR,lid))

measures=[]; count=0
for lid in dl.lists[14:18]:
    lid=lid[0]
    es=ES[count].structs; count+=1
    ds=es[1]; np=es[-1]
    measures.append(g.generalMeasures(ds,np))
# faz uma tabela para o corpo do artigo
# tabela com 4 na sequência para o SI

# rotina para a tabela seguinte:
# conta 
foo=g.makeText(ds)



#    date1=ds.messages[ds.message_ids[0]][2].isoformat().split("T")[0]
#    date2=ds.messages[ds.message_ids[-1]][2].isoformat().split("T")[0]
#    N=ds.n_authors
#    Ns=[len(i) for i in np.sectorialized_agents__]
#    Ns_=[100*len(i)/N for i in np.sectorialized_agents__]
#    M_=TOTAL_M-ds.n_messages
#    Mh=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[2]])
#    Mi=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[1]])
#    Mp=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[0]])
#    M=[Mh,Mi,Mp][::-1]
#    M2=[100*i/ds.n_messages for i in M]
#    MN=M_/N<commit_msg>registrando receitas para acessar as mensagens antes de limpar<commit_after># file dedicated to making the tables
# for the main document and the Supporting Information
# of the artigcle about text differentiation:
# https://github.com/ttm/artigoTextoNasRedes/ AND
# http://arxiv.org/abs/1412.7309

# tentar fazer as tabelas focadas nas medidas?

# fazer td com 100 mensagens somente de cada lista

# fazer tabela geral só com CPP
# incluir mensagens/ano na tabela
# tirar as datas para a tabela no corpo do artigo

# tentar paralelizar

# fazer tabela de caracteres soh com LAD
# para o corpo do artigo.
# renderizar 4 listas
# avaliar fazer para mais listas para tirar médias e desvios

import gmane as g, os, pickle, time as T, numpy as n
ENV=os.environ["PATH"]
import  importlib
from IPython.lib.deepreload import reload as dreload
importlib.reload(g.pca)
importlib.reload(g.loadMessages)
importlib.reload(g.listDataStructures)
importlib.reload(g.utils)
dreload(g,exclude="pytz")
os.environ["PATH"]=ENV

TT=T.time()
#print("{0:.2f}".format(T.time()-TT)); TT=T.time()
def pDump(tobject,tfilename):
    with open(tfilename,"wb") as f:
        pickle.dump(tobject,f,-1)
def pRead(tfilename):
    with open(tfilename,"rb") as f:
        tobject=pickle.load(f)
    return tobject

#dl=g.DownloadGmaneData('~/.gmane3/')
#dl.downloadedStats()
PDIR="pickledir/"
#pDump(dl,"{}dl.pickle".format(PDIR))
dl=pRead("{}dl.pickle".format(PDIR))

###### DATA STRUCTURES
TOTAL_M=100
dss=[]; iNs=[]; nms=[]; tss=[]; nps=[]
PDIR="pickledir/"
ES=[]
for lid in dl.lists[14:16]:
    lid=lid[0]
    es=g.EmailStructures(lid,TOTAL_M)
    ES.append(es)
#pDump(ES,"{}ES{}.pickle".format(PDIR,lid))

measures=[]; count=0
for lid in dl.lists[14:16]:
    lid=lid[0]
    es=ES[count].structs; count+=1
    ds=es[1]; np=es[-1]
    measures.append(g.generalMeasures(ds,np))
# faz uma tabela para o corpo do artigo
# tabela com 4 na sequência para o SI

# rotina para a tabela seguinte:
# conta 
foo=g.makeText(ds)

def getcharsets(msg):
    charsets = set({})
    for c in msg.get_charsets():
        if c is not None:
            charsets.update([c])
    return charsets

def handleerror(errmsg, emailmsg,cs,body):
    print()
    print(errmsg)
    print("This error occurred while decoding with ",cs," charset.")
    print("These charsets were found in the one email.",getcharsets(emailmsg))
    print("This is the subject:",emailmsg['subject'])
    print("This is the sender:",emailmsg['From'])
    print("This is the body:",body)

def getbodyfromemail(msg):
    body = None
    #Walk through the parts of the email to find the text body.    
    if msg.is_multipart():
        for part in msg.walk():
            # If part is multipart, walk through the subparts.            
            if part.is_multipart():
                for subpart in part.walk():
                    if subpart.get_content_type() == 'text/plain':
                        body = subpart.get_payload(decode=True)
                        continue
            elif part.get_content_type() == 'text/plain':
                body = part.get_payload(decode=True)
    elif msg.get_content_type() == 'text/plain':
        body = msg.get_payload(decode=True) 
    for charset in getcharsets(msg):
        try:
            body = body.decode(charset)
        except UnicodeDecodeError:
            handleerror("UnicodeDecodeError: encountered.",msg,charset,body)
        except AttributeError:
            handleerror("AttributeError: encountered" ,    msg,charset,body)
    return body 


def handleerror2(errmsg, t):
    print()
    print(errmsg)
#    print("This error occurred while decoding with ",cs," charset.")
#    print("These charsets were found in the one email.",getcharsets(emailmsg))
#    print("This is the subject:",emailmsg['subject'])
#    print("This is the sender:",emailmsg['From'])
    print("This is the body:",t)


def getBody(msg):
    while msg.is_multipart():
        msg=msg.get_payload()[0]
    t=msg.get_payload(decode=True)
    for charset in getcharsets(msg):
        try:
            t=t.decode(charset)
        except LookupError:
            handleerror2("UnicodeDecodeError: encountered.",t)
            t=t.decode()
    return t

    #    print("multipart:", msg.is_multipart())
    #print("content type:", msg.get_content_type())
    #while type(t) not in (type("astring"),type(b"abytes")):
    #    print("multipart:", msg.is_multipart())
    #    print("content type:", msg.get_content_type())
    #    t=t[0].get_payload(decode=True)

#def getBodyBACK(msg):
#    print("multipart:", msg.is_multipart())
#    print("content type:", msg.get_content_type())
#    t=msg.get_payload(decode=True)
#
#    while type(t) not in (type("astring"),type(b"abytes")):
#        print("multipart:", msg.is_multipart())
#        print("content type:", msg.get_content_type())
#        t=t[0].get_payload(decode=True)
#    for charset in getcharsets(msg):
#        try:
#            t=t.decode(charset)
#        except LookupError:
#            handleerror("UnicodeDecodeError: encountered.",msg,charset,body)
#    return t

# m.get_payload()[0].get_payload()
# m.get_payload()[2].get_payload()[0].get_payload()[0].get_payload()


#    date1=ds.messages[ds.message_ids[0]][2].isoformat().split("T")[0]
#    date2=ds.messages[ds.message_ids[-1]][2].isoformat().split("T")[0]
#    N=ds.n_authors
#    Ns=[len(i) for i in np.sectorialized_agents__]
#    Ns_=[100*len(i)/N for i in np.sectorialized_agents__]
#    M_=TOTAL_M-ds.n_messages
#    Mh=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[2]])
#    Mi=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[1]])
#    Mp=sum([len(ds.author_messages[author]) for author in np.sectorialized_agents__[0]])
#    M=[Mh,Mi,Mp][::-1]
#    M2=[100*i/ds.n_messages for i in M]
#    MN=M_/N
{'fraction_non_alphanumeric': 0.14949614700652045, 'fraction_numerical': 0.013633669235328987, 'mean_word_length': 3.8399311531841653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066853', 'n_tokens_mistral': 3158, 'n_tokens_neox': 3052, 'n_words': 681}