<commit_before>import email
import re
import requests 
import urlparse
import time
import random
import string
import whois
import dns.resolver
from email.header import decode_header
from selenium import webdriver

# classes

class Sender():
    Name = ""
    Email = ""

    def __init__(self,pRaw):
        result = email.utils.parseaddr(pRaw)
        self.Name = result[0]
        self.Email = result[1]
    def __str__(self):
        name = ["-",self.Name][self.Name != ""]
        return "{0} <{1}>".format(name,self.Email)

class File(object):
    Name = ""
    Blob = None

    def __str__(self):
        return "{0}: ".format(self.Name)

class Mail(object):
    Subject=""
    Receiver=[],
    Sender=None
    Attachments=[]
    Body=""
    Mailer=""
    MessageID=""
    Server=[]
    Links=[]
    trbwrk=""

    def __str__(self):
        response = "{0} => {1}: \"{2}\"".format(self.Sender,self.Receiver,self.Subject) 
        return response

class Hyperlink(object):
    Url=""
    Status=404
    Whois={}
    Headers= {}
    Addresses=[]
    Screenshot=""

    def __str__(self):
        
        if  "Server" not in self.Headers:
            self.Headers["Server"] = None

        if (self.Whois.name_servers == None):
            return "{0} (HTTP {1} {4}, {2}, {3})".format(self.Url,self.Status,self.Headers["Server"],self.Addresses,self.Screenshot);
        else:
            return "{0} (HTTP {1}, {7}, {8}, {9}, Location: {2}, {3} (*{4}), {5} ({6})".format(self.Url,self.Status,self.Whois.city,self.Whois.country,self.Whois.creation_date,self.Whois.name,self.Whois.emails,self.Headers["Server"],self.Addresses,self.Screenshot);

    __repr__ = __str__





class MailBD(object):
    """
        Mail Breakdown class, downloads, parses and analyses emails
    """
    def getMail(self,raw):
        """ generate a Mail object from the source code and start analysis """
        msg = email.message_from_string(raw)
        got = Mail()
        got.Subject = self.getHeader(msg,"Subject")
        got.Sender = Sender(self.getHeader(msg,"From"))
        got.Receiver = self.getHeader(msg,"To")
        got.Body = self.getBody(msg)
        got.Mailer = self.getHeader(msg,"X-Mailer")
        got.MessageID = self.getHeader(msg,"Message-Id")
        got.Attachments = self.getAttachments(msg)
        got.Server = self.getServer(msg)
        return got

    def getHeader(self,message,needle):
        """ extracts an header from the list and returns it """
        response = ""
        try:        
            decode = decode_header(message[needle])[0]
            response = unicode(decode[0])
        except(UnicodeDecodeError):
            response = message[needle] # in case of error -> return raw header instead of decoded them
        return response  

    def getBody(self,message):
        """ extracts the email body from the list and returns it """
        body = "";
        if (message.is_multipart()):
            for part in message.walk():
                mime = part.get_content_type()	
                if mime.find("text") != -1:	
                    body = part.get_payload()
        else:
            body = message.get_payload() 
        return body.replace("\n","")

    def getAttachments(self,message):
        payload = []
        if (message.is_multipart()):
            for part in message.walk():
                mime = part.get_content_type()
                        
                if part.get('Content-Disposition') is None:		
                    continue
                if mime.find("text") == -1:	
                    content = part.get_payload(decode=True)	
                    name = part.get_filename()
                    if (name != None):
                        att = File()
                        att.Name = name
                        att.Blob = content
                        payload.append(att)
        else:
            body = message.get_payload(decode=True) 
        return payload

    
    def getServer(self,message):
        results = []
        headers = message.get_all("Received")
        if (headers == None or len(headers) == 0):
            return "";
        
        for header in headers:
            try:
                if (header.startswith("from ")):
                    # is a from-header
                    index = header.index("by")
                    relevant = header[0:index]
                    # todo hostname detection
                    regex = re.compile(r"(?P<word>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))")
                    for matches in re.findall(regex, relevant):
                        for m in matches:
                            if results.count(m) == -0:
                                results.append(m)
            except ValueError:<commit_msg>+ inject trbwrk instance for better option reading<commit_after>import email
import re
import requests 
import urlparse
import time
import random
import string
import whois
import dns.resolver
from email.header import decode_header
from selenium import webdriver

# classes

class Sender():
    Name = ""
    Email = ""

    def __init__(self,pRaw):
        result = email.utils.parseaddr(pRaw)
        self.Name = result[0]
        self.Email = result[1]
    def __str__(self):
        name = ["-",self.Name][self.Name != ""]
        return "{0} <{1}>".format(name,self.Email)

class File(object):
    Name = ""
    Blob = None

    def __str__(self):
        return "{0}: ".format(self.Name)

class Mail(object):
    Subject=""
    Receiver=[],
    Sender=None
    Attachments=[]
    Body=""
    Mailer=""
    MessageID=""
    Server=[]
    Links=[]
    trbwrk=""

    def __str__(self):
        response = "{0} => {1}: \"{2}\"".format(self.Sender,self.Receiver,self.Subject) 
        return response

class Hyperlink(object):
    Url=""
    Status=404
    Whois={}
    Headers= {}
    Addresses=[]
    Screenshot=""

    def __str__(self):
        
        if  "Server" not in self.Headers:
            self.Headers["Server"] = None

        if (self.Whois.name_servers == None):
            return "{0} (HTTP {1} {4}, {2}, {3})".format(self.Url,self.Status,self.Headers["Server"],self.Addresses,self.Screenshot);
        else:
            return "{0} (HTTP {1}, {7}, {8}, {9}, Location: {2}, {3} (*{4}), {5} ({6})".format(self.Url,self.Status,self.Whois.city,self.Whois.country,self.Whois.creation_date,self.Whois.name,self.Whois.emails,self.Headers["Server"],self.Addresses,self.Screenshot);

    __repr__ = __str__





class MailBD(object):
    """
        Mail Breakdown class, downloads, parses and analyses emails
    """

    trbwrkInstance = None

    def __init__(self,trbwrk):
        self.trbwrkInstance = trbwrk

    def getMail(self,raw):
        """ generate a Mail object from the source code and start analysis """
        msg = email.message_from_string(raw)
        got = Mail()
        got.Subject = self.getHeader(msg,"Subject")
        got.Sender = Sender(self.getHeader(msg,"From"))
        got.Receiver = self.getHeader(msg,"To")
        got.Body = self.getBody(msg)
        got.Mailer = self.getHeader(msg,"X-Mailer")
        got.MessageID = self.getHeader(msg,"Message-Id")
        got.Attachments = self.getAttachments(msg)
        got.Server = self.getServer(msg)
        return got

    def getHeader(self,message,needle):
        """ extracts an header from the list and returns it """
        response = ""
        try:        
            decode = decode_header(message[needle])[0]
            response = unicode(decode[0])
        except(UnicodeDecodeError):
            response = message[needle] # in case of error -> return raw header instead of decoded them
        return response  

    def getBody(self,message):
        """ extracts the email body from the list and returns it """
        body = "";
        if (message.is_multipart()):
            for part in message.walk():
                mime = part.get_content_type()	
                if mime.find("text") != -1:	
                    body = part.get_payload()
        else:
            body = message.get_payload() 
        
        body = body.replace("\n","")
        return body

    def getAttachments(self,message,addBlob=False):
        payload = []
        if (message.is_multipart()):
            for part in message.walk():
                mime = part.get_content_type()
                        
                if part.get('Content-Disposition') is None:		
                    continue
                if mime.find("text") == -1:	
                    content = part.get_payload(decode=True)	
                    name = part.get_filename()
                    if (name != None):
                        att = File()
                        att.Name = name
                        if addBlob:
                            att.Blob = content
                        payload.append(att)
        else:
            body = message.get_payload(decode=True) 
        return payload

    
    def getServer(self,message):
        results = []
        headers = message.get_all("Received")
        if (headers == None or len(headers) == 0):
            return "";
        
        for header in headers:
            try:
                if (header.startswith("from ")):
                    # is a from-header
                    index = header.index("by")
                    relevant = header[0:index]
                    # todo hostname detection
                    regex = re.compile(r"(?P<word>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))")
                    for matches in re.findall(regex, relevant):
                        for m in matches:
                            if results.count(m) == -0:
                                results.append(m)
            except ValueError:
{'fraction_non_alphanumeric': 0.12999281240373756, 'fraction_numerical': 0.008419755621727077, 'mean_word_length': 1.7924311926605505, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693625', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2759, 'n_words': 640}