<commit_before>#!/usr/bin/env python

import os
import email
import mimetypes
from common.dbconnect import mongo_connect
from common.entities import Artifact, EmailAttachment
from canari.maltego.message import UIMessage
from canari.framework import configure

__author__ = 'catalyst256'
__copyright__ = 'Copyright 2014, sniffmypacketsv2 Project'
__credits__ = []

__license__ = 'GPL'
__version__ = '0.1'
__maintainer__ = 'catalyst256'
__email__ = '<EMAIL>'
__status__ = 'Development'

__all__ = [
    'dotransform'
]


@configure(
    label='Extract Email Attachments',
    description='Extract Email attachments from an artifact',
    uuids=['sniffMyPacketsv2.v2.artifact_2_email_attachment'],
    inputs=[('[SmP] - Email', Artifact)],
    debug=True
)
def dotransform(request, response):

    f = request.value
    d = mongo_connect()
    c = d['ARTIFACTS']
    folder = []
    # Check the pcap file doesn't exist in the database already (based on MD5 hash)
    try:
        s = d.ARTIFACTS.find({"File Name": f}).count()
        if s > 0:
            r = d.ARTIFACTS.find({"File Name": f}, {"Path": 1, "_id": 0})
            for i in r:
                folder = i['Path']
                # print folder
        else:
            return response + UIMessage('File not found!!')
    except Exception as e:
        return response + UIMessage(str(e))
    # print folder
    msgdata = []
    lookfor = 'DATA'
    file = '%s/%s' % (folder, f)

    # split the original file into two parts, message and header and save as lists
    with open(file, mode='r') as msgfile:
        reader = msgfile.read()
        for i, part in enumerate(reader.split(lookfor)):
            if i == 1:
                msgdata.append(part.strip())

    for item in msgdata:
        newfolder = '%s/email' % folder
        if not os.path.exists(newfolder):
            os.makedirs(newfolder)
            filename = folder + '/' + 'msgdata.msg'
            fb = open(filename, 'w')
            fb.write('%s\n' % item)
            fb.close()
<commit_msg>fixed a bug with extracted emails and dir clash<commit_after>#!/usr/bin/env python

import os
import email
import mimetypes
from common.dbconnect import mongo_connect
from common.entities import Artifact, EmailAttachment
from canari.maltego.message import UIMessage
from canari.framework import configure

__author__ = 'catalyst256'
__copyright__ = 'Copyright 2014, sniffmypacketsv2 Project'
__credits__ = []

__license__ = 'GPL'
__version__ = '0.1'
__maintainer__ = 'catalyst256'
__email__ = '<EMAIL>'
__status__ = 'Development'

__all__ = [
    'dotransform'
]


@configure(
    label='Extract Email Attachments',
    description='Extract Email attachments from an artifact',
    uuids=['sniffMyPacketsv2.v2.artifact_2_email_attachment'],
    inputs=[('[SmP] - Email', Artifact)],
    debug=True
)
def dotransform(request, response):

    f = request.value
    d = mongo_connect()
    c = d['ARTIFACTS']
    folder = []
    # Check the pcap file doesn't exist in the database already (based on MD5 hash)
    try:
        s = d.ARTIFACTS.find({"File Name": f}).count()
        if s > 0:
            r = d.ARTIFACTS.find({"File Name": f}, {"Path": 1, "_id": 0})
            for i in r:
                folder = i['Path']
                # print folder
        else:
            return response + UIMessage('File not found!!')
    except Exception as e:
        return response + UIMessage(str(e))
    # print folder
    msgdata = []
    lookfor = 'DATA'
    file = '%s/%s' % (folder, f)

    # split the original file into two parts, message and header and save as lists
    with open(file, mode='r') as msgfile:
        reader = msgfile.read()
        for i, part in enumerate(reader.split(lookfor)):
            if i == 1:
                msgdata.append(part.strip())

    for item in msgdata:
        newfolder = '%s/emails' % folder
        if not os.path.exists(newfolder):
            os.makedirs(newfolder)
            filename = folder + '/' + 'msgdata.msg'
            fb = open(filename, 'w')
            fb.write('%s\n' % item)
            fb.close()

{'fraction_non_alphanumeric': 0.11934458302763512, 'fraction_numerical': 0.010271460014673514, 'mean_word_length': 2.590869183494293, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4435742', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1269, 'n_words': 368}