<commit_before>#!/usr/bin/env python2
# -*- coding:utf8 -*-

rawletters="new/" # new raw messages to parse them
indexdir="echo/"
messagesdir="msg/"

echo="mailbox.15"
addr="mscript, 1"
tags="ii/ok"

from email.utils import parsedate,parseaddr
from email.Header import decode_header
import email.parser as pars
import os
import time
import base64
import hashlib
import html2text

def hsh(str):
	return base64.urlsafe_b64encode( hashlib.sha256(str).digest() ).replace('-','A').replace('_','z')[:20]

myparser=pars.Parser()
h = html2text.HTML2Text()
h.ignore_links = True

for filename in os.listdir(rawletters):
	letterpath=os.path.join(rawletters, filename)
	
	plainmessage=open(letterpath).read()
	message=myparser.parsestr(plainmessage)
	
	msgdate=0
	sender="nobody"
	to="All"
	subj="..."
	msg="no message"
	
	if(message['date']):
		parsdate=parsedate(message['date'])
		msgdate=int(time.mktime(parsdate))

	if(message['from']):
		sender=parseaddr(message['from'])[1]
	
	if(message['to']):
		to=parseaddr(message['to'])[1]
	
	if(message['subject']):
		decodefrag = decode_header(message['subject'])
		subj_fragments = []
		for s , enc in decodefrag:
			if enc:
				s = unicode(s , enc).encode('utf8','replace')
			subj_fragments.append(s)
		subj = ''.join(subj_fragments)
	
	if(message.is_multipart()):
		for part in message.walk():
			charset=message.get_content_charset()
			if(not charset):
				charset='ascii'
			payload=part.get_payload(decode=True)
			if(not payload):
				continue
			msg+="\n"+h.handle(payload.decode("utf8")).strip().replace("\n\n","\n")
	else:
		# charset=message.get_content_charset() or "ascii"
		msg=h.handle(message.get_payload(decode=True).decode("utf8")).strip().replace("\n\n", "\n")
		
	iimsg=tags+"\n"+echo+"\n"+str(msgdate)+"\n"+sender+"\n"+addr+"\n"+to+"\n"+subj+"\n"
	iimsg+="\n"+msg.encode("utf8")

	msgid=hsh(iimsg)
	open(os.path.join(messagesdir, msgid), "w").write(iimsg)
	open(os.path.join(indexdir, echo), "a").write(msgid+"\n")
	os.remove(letterpath)<commit_msg>charset problems<commit_after>#!/usr/bin/env python2
# -*- coding:utf8 -*-

rawletters="new/" # new raw messages to parse them
indexdir="echo/"
messagesdir="msg/"

echo="mailbox.15"
addr="mscript, 1"
tags="ii/ok"

from email.utils import parsedate,parseaddr
from email.Header import decode_header
import email.parser as pars
import os
import time
import base64
import hashlib
import html2text

def hsh(str):
	return base64.urlsafe_b64encode( hashlib.sha256(str).digest() ).replace('-','A').replace('_','z')[:20]

myparser=pars.Parser()
h = html2text.HTML2Text()
h.ignore_links = True

for filename in os.listdir(rawletters):
	letterpath=os.path.join(rawletters, filename)
	
	plainmessage=open(letterpath).read()
	message=myparser.parsestr(plainmessage)
	
	msgdate=0
	sender="nobody"
	to="All"
	subj="..."
	msg="no message"
	
	if(message['date']):
		parsdate=parsedate(message['date'])
		msgdate=int(time.mktime(parsdate))

	if(message['from']):
		sender=parseaddr(message['from'])[1]
	
	if(message['to']):
		to=parseaddr(message['to'])[1]
	
	if(message['subject']):
		decodefrag = decode_header(message['subject'])
		subj_fragments = []
		for s , enc in decodefrag:
			if enc:
				s = unicode(s , enc).encode('utf8','replace')
			subj_fragments.append(s)
		subj = ''.join(subj_fragments)
	
	if(message.is_multipart()):
		for part in message.walk():
			charset=message.get_content_charset()
			if(not charset):
				charset='ascii'
			payload=part.get_payload(decode=True)
			if(not payload):
				continue
			try:
				payload=payload.decode(charset)
				msg+="\n"+h.handle(payload).strip().replace("\n\n","\n")
			except:
				msg+="can't decode"
				continue
	else:
		charset=message.get_content_charset() or "ascii"
		msg=h.handle(message.get_payload(decode=True).decode(charset)).strip().replace("\n\n", "\n")
		
	iimsg=tags+"\n"+echo+"\n"+str(msgdate)+"\n"+sender+"\n"+addr+"\n"+to+"\n"+subj+"\n"
	iimsg+="\n"+msg.encode("utf8")

	msgid=hsh(iimsg)
	open(os.path.join(messagesdir, msgid), "w").write(iimsg)
	open(os.path.join(indexdir, echo), "a").write(msgid+"\n")
	os.remove(letterpath)
{'fraction_non_alphanumeric': 0.191650390625, 'fraction_numerical': 0.01220703125, 'mean_word_length': 5.4825949367088604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7605795', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1519, 'n_words': 240}