<commit_before># -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime, re, base64
import pymisp
from stix2 import *
from misp2stix2_mapping import *
from collections import defaultdict

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability', 'attachment']

misp_hash_types = ["authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh"]

class StixBuilder():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.SDOs = []
        self.object_refs = []
        self.external_refs = []

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        self.misp_event.load_file(filename)
        self.filename = filename

    def buildEvent(self):
        self.__set_identity()
        self.read_attributes()
        report = self.eventReport()
        self.SDOs.insert(1, report)
        self.stix_package = self.generate_package()

    def eventReport(self):
        report_args = {'type': 'report', 'id': 'report--{}'.format(self.misp_event.uuid),
                       'created_by_ref': self.identity_id, 'name': self.misp_event.info,
                       'published': self.misp_event.publish_timestamp,
                       'object_refs': self.object_refs}
        labels = []
        if self.misp_event.Tag:
            for tag in self.misp_event.Tag:
                labels.append(tag.name)
        if labels:
            report_args['labels'] = labels
        else:
            report_args['labels'] = ['Threat-Report']
        report_args['labels'].append('misp:tool="misp2stix2"')
        if self.external_refs:
            report_args['external_references'] = external_refs
        return Report(**report_args)

    def generate_package(self):
        bundle_args = {"type": "bundle", "spec_version": "2.0", "objects": self.SDOs,
                       "id": "bundle--{}".format(self.misp_event.uuid)}
        return Bundle(**bundle_args)

    def saveFile(self):
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(json.dumps(self.stix_package, cls=base.STIXJSONEncoder))

    def __set_identity(self):
        org = self.misp_event.Orgc
        identity_id = 'identity--{}'.format(org['uuid'])
        identity = Identity(type="identity", id=identity_id,
                            name=org["name"], identity_class="organization")
        self.SDOs.append(identity)
        self.identity_id = identity_id

    def misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        self.categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']

    def read_attributes(self):
        self.misp_types()
        if hasattr(self.misp_event, 'attributes') and self.misp_event.attributes:
            for attribute in self.misp_event.attributes:
                attribute_type = attribute.type
                if attribute_type in non_indicator_attributes:
                    self.handle_non_indicator(attribute, attribute_type)
                else:
                    if attribute_type in self.categories_mapping['Person']:
                        self.handle_person(attribute)
                    elif attribute_type in mispTypesMapping:
                        self.handle_usual_type(attribute)
                    else:
                        self.add_custom(attribute)
        if hasattr(self.misp_event, 'objects') and self.misp_event.objects:
            self.load_objects_mapping()
            for misp_object in self.misp_event.objects:
                object_attributes = misp_object.attributes
                to_ids = self.fetch_ids_flag(object_attributes)
                object_name = misp_object.name
                if object_name == "vulnerability":
                    self.add_object_vulnerability(misp_object, to_ids)
                elif object_name == "course-of-action":
                    self.add_course_of_action(misp_object, from_object=True)
                elif object_name in objectsMapping:
                    try:
                        if to_ids:
                            self.add_object_indicator(misp_object, to_ids)
                        else:
                            self.add_object_observable(misp_object, to_ids)
                    except:
                        self.add_object_custom(misp_object, to_ids)
                else:
                    self.add_object_custom(misp_object, to_ids)
        if hasattr(self.misp_event, 'Galaxy') and self.misp_event.Galaxy:
            for galaxy in self.misp_event.Galaxy:
                galaxy_type = galaxy.get('type')
                if 'attack-pattern' in galaxy_type:
                    self.add_attack_pattern(galaxy)
                elif 'course' in galaxy_type:
                    self.add_course_of_action(galaxy)
                elif 'intrusion' in galaxy_type:
                    self.add_intrusion_set(galaxy)<commit_msg>fix: [stix2 export] Added forgotten processes related function call<commit_after># -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime, re, base64
import pymisp
from stix2 import *
from misp2stix2_mapping import *
from collections import defaultdict

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability', 'attachment']

misp_hash_types = ["authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh"]

class StixBuilder():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.SDOs = []
        self.object_refs = []
        self.external_refs = []

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        self.misp_event.load_file(filename)
        self.filename = filename

    def buildEvent(self):
        self.__set_identity()
        self.read_attributes()
        report = self.eventReport()
        self.SDOs.insert(1, report)
        self.stix_package = self.generate_package()

    def eventReport(self):
        report_args = {'type': 'report', 'id': 'report--{}'.format(self.misp_event.uuid),
                       'created_by_ref': self.identity_id, 'name': self.misp_event.info,
                       'published': self.misp_event.publish_timestamp,
                       'object_refs': self.object_refs}
        labels = []
        if self.misp_event.Tag:
            for tag in self.misp_event.Tag:
                labels.append(tag.name)
        if labels:
            report_args['labels'] = labels
        else:
            report_args['labels'] = ['Threat-Report']
        report_args['labels'].append('misp:tool="misp2stix2"')
        if self.external_refs:
            report_args['external_references'] = external_refs
        return Report(**report_args)

    def generate_package(self):
        bundle_args = {"type": "bundle", "spec_version": "2.0", "objects": self.SDOs,
                       "id": "bundle--{}".format(self.misp_event.uuid)}
        return Bundle(**bundle_args)

    def saveFile(self):
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(json.dumps(self.stix_package, cls=base.STIXJSONEncoder))

    def __set_identity(self):
        org = self.misp_event.Orgc
        identity_id = 'identity--{}'.format(org['uuid'])
        identity = Identity(type="identity", id=identity_id,
                            name=org["name"], identity_class="organization")
        self.SDOs.append(identity)
        self.identity_id = identity_id

    def misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        self.categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']

    def read_attributes(self):
        self.misp_types()
        if hasattr(self.misp_event, 'attributes') and self.misp_event.attributes:
            for attribute in self.misp_event.attributes:
                attribute_type = attribute.type
                if attribute_type in non_indicator_attributes:
                    self.handle_non_indicator(attribute, attribute_type)
                else:
                    if attribute_type in self.categories_mapping['Person']:
                        self.handle_person(attribute)
                    elif attribute_type in mispTypesMapping:
                        self.handle_usual_type(attribute)
                    else:
                        self.add_custom(attribute)
        if hasattr(self.misp_event, 'objects') and self.misp_event.objects:
            self.load_objects_mapping()
            misp_objects = self.misp_event.objects
            self.object_references, self.processes = self.fetch_object_references(misp_objects)
            for misp_object in misp_objects:
                object_attributes = misp_object.attributes
                to_ids = self.fetch_ids_flag(object_attributes)
                object_name = misp_object.name
                if object_name == "vulnerability":
                    self.add_object_vulnerability(misp_object, to_ids)
                elif object_name == "course-of-action":
                    self.add_course_of_action(misp_object, from_object=True)
                elif object_name in objectsMapping:
                    try:
                        if to_ids:
                            self.add_object_indicator(misp_object, to_ids)
                        else:
                            self.add_object_observable(misp_object, to_ids)
                    except:
                        self.add_object_custom(misp_object, to_ids)
                else:
                    self.add_object_custom(misp_object, to_ids)
        if hasattr(self.misp_event, 'Galaxy') and self.misp_event.Galaxy:
            for galaxy in self.misp_event.Galaxy:
                galaxy_type = galaxy.get('type')
                if 'attack-pattern' in galaxy_type:
                    self.add_attack_pattern(galaxy)
                elif 'course' in galaxy_type:
                    self.add_course_of_action(galaxy)
                elif 'intrusion' in galaxy_type:
                    self.add_intrusion_set(galaxy)
{'fraction_non_alphanumeric': 0.10085044929396662, 'fraction_numerical': 0.009226572528883184, 'mean_word_length': 2.1855353948377205, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2632510', 'n_tokens_mistral': 3654, 'n_tokens_neox': 3455, 'n_words': 774}