<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime
import pymisp
from stix2 import *
from base64 import b64encode
from misp2stix2_mapping import *
from collections import defaultdict
from copy import deepcopy

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability']
misp_hash_types = ["authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh"]
attack_pattern_galaxies_list = ['mitre-attack-pattern', 'mitre-enterprise-attack-attack-pattern',
                                'mitre-mobile-attack-attack-pattern', 'mitre-pre-attack-attack-pattern']
course_of_action_galaxies_list = ['mitre-course-of-action', 'mitre-enterprise-attack-course-of-action',
                                  'mitre-mobile-attack-course-of-action']
intrusion_set_galaxies_list = ['mitre-enterprise-attack-intrusion-set', 'mitre-mobile-attack-intrusion-set',
                               'mitre-pre-attack-intrusion-set', 'mitre-intrusion-set']
malware_galaxies_list = ['android', 'banker', 'stealer', 'backdoor', 'ransomware', 'mitre-malware',
                         'mitre-enterprise-attack-malware', 'mitre-mobile-attack-malware']
threat_actor_galaxies_list = ['threat-actor', 'microsoft-activity-group']
tool_galaxies_list = ['botnet', 'rat', 'exploit-kit', 'tds', 'tool', 'mitre-tool',
                      'mitre-enterprise-attack-tool', 'mitre-mobile-attack-tool']

class StixBuilder():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.SDOs = []
        self.object_refs = []
        self.external_refs = []
        self.galaxies = []
        self.relationships = defaultdict(list)

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        self.misp_event.load_file(filename)
        self.filename = filename
        self.report_id = "report--{}".format(self.misp_event.uuid)

    def buildEvent(self):
        self.__set_identity()
        self.read_attributes()
        report = self.eventReport()
        self.SDOs.insert(1, report)

    def eventReport(self):
        report_args = {'type': 'report', 'id': self.report_id, 'name': self.misp_event.info,
                       'created_by_ref': self.identity_id, 'published': self.misp_event.publish_timestamp,
                       'created': self.misp_event.date, 'object_refs': self.object_refs}
        if self.misp_event.Tag:
            labels = []
            for tag in self.misp_event.Tag:
                labels.append(tag.name)
            report_args['labels'] = labels
            if 'misp:tool="misp2stix2"' not in labels:
                report_args['labels'].append('misp:tool="misp2stix2"')
        else:
            report_args['labels'] = ['Threat-Report']
            report_args['labels'].append('misp:tool="misp2stix2"')
        if self.external_refs:
            report_args['external_references'] = self.external_refs
        return Report(**report_args)

    def saveFile(self):
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(json.dumps(self.SDOs, cls=base.STIXJSONEncoder))

    def __set_identity(self):
        org = self.misp_event.Orgc
        identity_id = 'identity--{}'.format(org['uuid'])
        identity = Identity(type="identity", id=identity_id,
                            name=org["name"], identity_class="organization")
        self.SDOs.append(identity)
        self.identity_id = identity_id

    def misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        self.categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']

    def read_attributes(self):
        self.misp_types()
        if hasattr(self.misp_event, 'attributes') and self.misp_event.attributes:
            for attribute in self.misp_event.attributes:
                attribute_type = attribute.type
                if attribute_type in non_indicator_attributes:
                    self.handle_non_indicator(attribute, attribute_type)
                else:
                    if attribute_type in self.categories_mapping['Person']:
                        self.handle_person(attribute)
                    elif attribute_type in mispTypesMapping:
                        self.handle_usual_type(attribute)
                    else:
                        self.add_custom(attribute)
        if hasattr(self.misp_event, 'objects') and self.misp_event.objects:
            self.load_objects_mapping()
            objects_to_parse = defaultdict(dict)
            misp_objects = self.misp_event.objects
            self.object_references, self.processes = self.fetch_object_references(misp_objects)
            for misp_object in misp_objects:
                to_ids = self.fetch_ids_flag(misp_object.attributes)
                name = misp_object.name
                if name == "vulnerability":
                    self.add_object_vulnerability(misp_object, to_ids)
                elif name == "course-of-action":
                    self.add_course_of_action(misp_object, from_object=True)
                elif name in ('pe', 'pe-section'):
                    objects_to_parse[name][misp_object.uuid] = to_ids, misp_object
                elif name in objectsMapping:
                    if  name == 'file' and misp_object.references:
                        to_parse = False
                        for reference in misp_object.references:
                            if reference.relationship_type == 'included-in' and reference.Object['name'] == "pe":
                                objects_to_parse[name][misp_object.uuid] = to_ids, misp_object
                                to_parse = True
                                break
                        if to_parse:
                            continue
                    try:
                        if to_ids or name == "stix2-pattern":
                            self.add_object_indicator(misp_object)
                        else:
                            self.add_object_observable(misp_object)
                    except:
                        self.add_object_custom(misp_object, to_ids)
                else:
                    self.add_object_custom(misp_object, to_ids)
            if objects_to_parse: self.resolve_objects2parse(objects_to_parse)
        if hasattr(self.misp_event, 'Galaxy') and self.misp_event.Galaxy:
            for galaxy in self.misp_event.Galaxy:
                self.parse_galaxy(galaxy, self.report_id)
        for source, targets in self.relationships.items():
            if source.startswith('report'): continue
            source_type,_ = source.split('--')
            for target in targets:
                target_type,_ = target.split('--')
                try:
                    relation = relationshipsSpecifications[source_type][target_type]
                except KeyError:
                    # custom relationship (suggested by iglocska)
                    relation = "has"
                relationship = Relationship(source_ref=source, target_ref=target, relationship_type=relation)
                self.append_object(relationship, relationship.id)

    def load_objects_mapping(self):
        self.objects_mapping = {
            'asn': {'observable': self.resolve_asn_observable,
                    'pattern': self.resolve_asn_pattern},
            'domain-ip': {'observable': self.resolve_domain_ip_observable,
                          'pattern': self.resolve_domain_ip_pattern},
            'email': {'observable': self.resolve_email_object_observable,
                      'pattern': self.resolve_email_object_pattern},
            'file': {'observable': self.resolve_file_observable,
                     'pattern': self.resolve_file_pattern},
            'ip-port': {'observable': self.resolve_ip_port_observable,
                        'pattern': self.resolve_ip_port_pattern},
            'network-socket': {'observable': self.resolve_network_socket_observable,
                               'pattern': self.resolve_network_socket_pattern},
            'process': {'observable': self.resolve_process_observable,
                        'pattern': self.resolve_process_pattern},
            'registry-key': {'observable': self.resolve_regkey_observable,
                             'pattern': self.resolve_regkey_pattern},
            'stix2': {'pattern': self.resolve_stix2_pattern},
            'url': {'observable': self.resolve_url_observable,
                    'pattern': self.resolve_url_pattern},
            'x509': {'observable': self.resolve_x509_observable,
                     'pattern': self.resolve_x509_pattern}
        }
        self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', self.add_vulnerability]}
        self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', self.add_attack_pattern]))
        self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', self.add_course_of_action]))
        self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', self.add_intrusion_set]))
        self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', self.add_malware]))
        self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', self.add_threat_actor]))
        self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', self.add_tool]))

    def fetch_object_references(self, misp_objects):
        object_references, processes = {}, {}
        for misp_object in misp_objects:
            attributes = misp_object.attributes
            if misp_object.name == "process":
                self.get_process_attributes(processes, misp_object.attributes)
            if misp_object.references and not self.fetch_ids_flag(attributes):
                for reference in misp_object.references:
                    try:
                        referenced_object = reference.Attribute
                    except:
                        referenced_object = self.misp_event.get_object_by_uuid(reference.referenced_uuid)
                    object_references[reference.referenced_uuid] = referenced_object
        return object_references, processes

    @staticmethod
    def get_process_attributes(processes, attributes):
        pid, process = None, {}
        for attribute in attributes:
            relation = attribute.object_relation
            attribute_value = attribute.value
            if relation == 'pid':
                if attribute_value in processes:
                    return
                pid = attribute_value
                process[relation] = attribute_value
            elif relation in ('name', 'creation-time'):
                process[relation] = attribute_value
        if pid is not None:
            process['type'] = 'process'
            processes[pid] = process

    def handle_non_indicator(self, attribute, attribute_type):
        if attribute_type == "link":
            self.handle_link(attribute)
        elif attribute_type in ('text', 'comment', 'other') or attribute_type not in mispTypesMapping:
            self.add_custom(attribute)
        else:
            try:
                self.handle_non_indicator_attribute(attribute, attribute_type)
            except:
                self.add_custom(attribute)

    def handle_non_indicator_attribute(self, attribute, attribute_type):
        if attribute_type == "vulnerability":
            self.add_vulnerability(attribute, from_galaxy=False)
        else:
            self.add_observed_data(attribute)

    def handle_person(self, attribute):
        if attribute.category == "Person":
            self.add_identity(attribute)
        else:
            self.add_custom(attribute)

    def handle_usual_type(self, attribute):
        try:
            if attribute.to_ids:
                self.add_indicator(attribute)
            else:
                self.add_observed_data(attribute)
        except:
            self.add_custom(attribute)

    def handle_link(self, attribute):
        url = attribute.value
        source = "url"
        try:
            if hasattr(attribute, 'comment') and attribute.comment:
                source += " - {}".format(attribute.comment)
        except AttributeError:
            pass
        link = {'source_name': source, 'url': url}
        self.external_refs.append(link)

    def resolve_objects2parse(self, objects2parse):
        for uuid, misp_object in objects2parse['file'].items():
            to_ids_file, file_object = misp_object
            file_id = "file--{}".format(file_object.uuid)
            to_ids_list = [to_ids_file]
            object2create = defaultdict(list)
            for reference in file_object.references:
                if reference.relationship_type == "included-in" and reference.Object['name'] == "pe":
                    pe_uuid = reference.referenced_uuid
                    break
            to_ids_pe, pe_object = objects2parse['pe'][pe_uuid]
            to_ids_list.append(to_ids_pe)
            sections = []
            for reference in pe_object.references:
                if reference.Object['name'] == "pe-section":
                    to_ids_section, section_object = objects2parse['pe-section'][reference.referenced_uuid]
                    to_ids_list.append(to_ids_section)
                    sections.append(section_object)
            if True in to_ids_list:
                pattern = self.resolve_file_pattern(file_object.attributes, file_id)[1:-1]
                pattern += " AND {}".format(self.parse_pe_extensions_pattern(pe_object, sections))
                self.add_object_indicator(file_object, pattern_arg="[{}]".format(pattern))
            else:
                observable = self.resolve_file_observable(file_object.attributes, file_id)
                observable['1']['extensions'] = self.parse_pe_extensions_observable(pe_object, sections)
                self.add_object_observable(file_object, observable_arg=observable)

    def parse_pe_extensions_observable(self, pe_object, sections):
        extension = defaultdict(list)
        for attribute in pe_object.attributes:
            try:
                extension[peMapping[attribute.object_relation]] = attribute.value
            except KeyError:
                extension["x_misp_{}_{}".format(attribute.type, attribute.object_relation.replace('-', '_'))] = attribute.value
        for section in sections:
            d_section = defaultdict(dict)
            for attribute in section.attributes:
                relation = attribute.object_relation
                if relation in misp_hash_types:
                    d_section['hashes'][relation] = attribute.value
                else:
                    try:
                        d_section[peSectionMapping[relation]] = attribute.value
                    except KeyError:
                        continue
            extension['sections'].append(WindowsPESection(**d_section))
        return {"windows-pebinary-ext": extension}

    def parse_pe_extensions_pattern(self, pe_object, sections):
        pattern = ""
        mapping = objectsMapping['file']['pattern']
        pe_mapping = "extensions.'windows-pebinary-ext'"
        for attribute in pe_object.attributes:
            try:
                stix_type = "{}.{}".format(pe_mapping, peMapping[attribute.object_relation])
            except KeyError:
                stix_type = "{}.{}'".format(pe_mapping[:-1], "x_misp_{}_{}".format(attribute.type, attribute.object_relation.replace('-', '_')))
            pattern += mapping.format(stix_type, attribute.value)
        n_section = 0
        for section in sections:
            section_mapping = "{}.sections[{}]".format(pe_mapping, str(n_section))
            for attribute in section.attributes:
                relation = attribute.object_relation
                if relation in misp_hash_types:
                    stix_type = "{}.hashes.'{}'".format(section_mapping, relation)
                    pattern += mapping.format(stix_type, attribute.value)
                else:
                    try:
                        stix_type = "{}.{}".format(section_mapping, peSectionMapping[relation])
                        pattern += mapping.format(stix_type, attribute.value)
                    except KeyError:
                        continue
            n_section += 1
        return pattern[:-5]

    def parse_galaxies(self, galaxies, source_id):
        for galaxy in galaxies:
            self.parse_galaxy(galaxy, source_id)

    def parse_galaxy(self, galaxy, source_id):
        galaxy_type = galaxy.get('type')
        galaxy_uuid = galaxy['GalaxyCluster'][0]['uuid']
        try:
            stix_type, to_call = self.galaxies_mapping[galaxy_type]
        except:
            return
        if galaxy_uuid not in self.galaxies:
            to_call(galaxy)
            self.galaxies.append(galaxy_uuid)
        self.relationships[source_id].append("{}--{}".format(stix_type, galaxy_uuid))

    @staticmethod
    def generate_galaxy_args(galaxy, b_killchain, b_alias, sdo_type):
        galaxy_type = galaxy.get('type')
        name = galaxy.get('name')
        cluster = galaxy['GalaxyCluster'][0]
        sdo_id = "{}--{}".format(sdo_type, cluster.get('uuid'))
        description = "{} | {}".format(galaxy.get('description'), cluster.get('description'))
        labels = ['misp:type=\"{}\"'.format(galaxy_type)]
        sdo_args = {'id': sdo_id, 'type': sdo_type, 'name': name, 'description': description}
        if b_killchain:
            killchain = [{'kill_chain_name': 'misp-category',
                          'phase_name': galaxy_type}]
            sdo_args['kill_chain_phases'] = killchain
        if cluster['tag_name']:
            labels.append(cluster.get('tag_name'))
        meta = cluster.get('meta')
        if 'synonyms' in meta and b_alias:
            aliases = []
            for a in meta['synonyms']:
                aliases.append(a)
            sdo_args['aliases'] = aliases
        sdo_args['labels'] = labels
        return sdo_args, sdo_id

    def add_attack_pattern(self, galaxy):
        a_p_args, a_p_id = self.generate_galaxy_args(galaxy, True, False, 'attack-pattern')
        a_p_args['created_by_ref'] = self.identity_id
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern, a_p_id)

    def add_course_of_action(self, misp_object, from_object=False):
        if from_object:
            coa_id = 'course-of-action--{}'.format(misp_object.uuid)
            coa_args = {'id': coa_id, 'type': 'course-of-action'}
            for attribute in misp_object.attributes:
                self.parse_galaxies(attribute.Galaxy, coa_id)
                relation = attribute.object_relation
                if relation == 'name':
                    coa_args['name'] = attribute.value
                elif relation == 'description':
                    coa_args['description'] = attribute.value
            if not 'name' in coa_args:
                return
        else:
            coa_args, coa_id = self.generate_galaxy_args(misp_object, False, False, 'course-of-action')
        coa_args['created_by_ref'] = self.identity_id
        course_of_action = CourseOfAction(**coa_args)
        self.append_object(course_of_action, coa_id)

    def add_custom(self, attribute):
        custom_object_id = "x-misp-object--{}".format(attribute.uuid)
        custom_object_type = "x-misp-object-{}".format(attribute.type.replace('|', '-').lower())
        labels = self.create_labels(attribute)
        custom_object_args = {'id': custom_object_id, 'x_misp_timestamp': attribute.timestamp, 'labels': labels,
                               'x_misp_value': attribute.value, 'created_by_ref': self.identity_id,
                               'x_misp_category': attribute.category}
        if hasattr(attribute, 'comment') and attribute.comment:
            custom_object_args['x_misp_comment'] = attribute.comment
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                          ('x_misp_timestamp', properties.StringProperty(required=True)),
                                          ('labels', properties.ListProperty(labels, required=True)),
                                          ('x_misp_value', properties.StringProperty(required=True)),
                                          ('created_by_ref', properties.StringProperty(required=True)),
                                          ('x_misp_comment', properties.StringProperty()),
                                          ('x_misp_category', properties.StringProperty())
                                         ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object, custom_object_id)

    def add_identity(self, attribute):
        identity_id = "identity--{}".format(attribute.uuid)
        name = attribute.value
        labels = self.create_labels(attribute)
        identity_args = {'id': identity_id,  'type': identity, 'name': name, 'labels': labels,
                          'identity_class': 'individual', 'created_by_ref': self.identity_id}
        if hasattr(attribute, 'comment') and attribute.comment:
            identity_args['description'] = attribute.comment
        identity = Identity(**identity_args)
        self.append_object(identity, identity_id)

    def add_indicator(self, attribute):
        attribute_type = attribute.type
        indicator_id = "indicator--{}".format(attribute.uuid)
        self.parse_galaxies(attribute.Galaxy, indicator_id)
        category = attribute.category
        killchain = self.create_killchain(category)
        labels = self.create_labels(attribute)
        attribute_value = attribute.value if attribute_type != "AS" else self.define_attribute_value(attribute.value, attribute.comment)
        pattern = mispTypesMapping[attribute_type]['pattern'](attribute_type, attribute_value, b64encode(attribute.data.getbuffer()).decode()[1:-1]) if ('data' in attribute and attribute.data) else self.define_pattern(attribute_type, attribute_value)
        indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, 'kill_chain_phases': killchain,
                           'valid_from': self.misp_event.date, 'created_by_ref': self.identity_id, 'pattern': pattern}
        if hasattr(attribute, 'Sighting'):
            for sighting in attribute.Sighting:
                if sighting['Organisation']['name'] == self.misp_event.Orgc.name and sighting['type'] == "2":
                    indicator_args['valid_until'] = datetime.datetime.fromtimestamp(int(sighting['date_sighting'])).isoformat()
                    break
        if hasattr(attribute, 'comment') and attribute.comment:
            indicator_args['description'] = attribute.comment
        indicator = Indicator(**indicator_args)
        self.append_object(indicator, indicator_id)

    def add_intrusion_set(self, galaxy):
        i_s_args, i_s_id = self.generate_galaxy_args(galaxy, False, True, 'intrusion-set')
        i_s_args['created_by_ref'] = self.identity_id
        intrusion_set = IntrusionSet(**i_s_args)
        self.append_object(intrusion_set, i_s_id)

    def add_malware(self, galaxy):
        malware_args, malware_id = self.generate_galaxy_args(galaxy, True, False, 'malware')
        malware_args['created_by_ref'] = self.identity_id
        malware = Malware(**malware_args)
        self.append_object(malware, malware_id)

    def add_observed_data(self, attribute):
        attribute_type = attribute.type
        observed_data_id = "observed-data--{}".format(attribute.uuid)
        self.parse_galaxies(attribute.Galaxy, observed_data_id)
        timestamp = attribute.timestamp
        labels = self.create_labels(attribute)
        attribute_value = attribute.value if attribute_type != "AS" else self.define_attribute_value(attribute.value, attribute.comment)
        observable = mispTypesMapping[attribute_type]['observable'](attribute_type, attribute_value, b64encode(attribute.data.getbuffer())) if 'data' in attribute else self.define_observable(attribute_type, attribute_value)
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'number_observed': 1,
                              'first_observed': timestamp, 'last_observed': timestamp, 'labels': labels,<commit_msg>fix: [stix2 export] Fixed timestamp to datetime conversion for 'date_sighting', using utc format<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime
import pymisp
from stix2 import *
from base64 import b64encode
from misp2stix2_mapping import *
from collections import defaultdict
from copy import deepcopy

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability']
misp_hash_types = ["authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh"]
attack_pattern_galaxies_list = ['mitre-attack-pattern', 'mitre-enterprise-attack-attack-pattern',
                                'mitre-mobile-attack-attack-pattern', 'mitre-pre-attack-attack-pattern']
course_of_action_galaxies_list = ['mitre-course-of-action', 'mitre-enterprise-attack-course-of-action',
                                  'mitre-mobile-attack-course-of-action']
intrusion_set_galaxies_list = ['mitre-enterprise-attack-intrusion-set', 'mitre-mobile-attack-intrusion-set',
                               'mitre-pre-attack-intrusion-set', 'mitre-intrusion-set']
malware_galaxies_list = ['android', 'banker', 'stealer', 'backdoor', 'ransomware', 'mitre-malware',
                         'mitre-enterprise-attack-malware', 'mitre-mobile-attack-malware']
threat_actor_galaxies_list = ['threat-actor', 'microsoft-activity-group']
tool_galaxies_list = ['botnet', 'rat', 'exploit-kit', 'tds', 'tool', 'mitre-tool',
                      'mitre-enterprise-attack-tool', 'mitre-mobile-attack-tool']

class StixBuilder():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.SDOs = []
        self.object_refs = []
        self.external_refs = []
        self.galaxies = []
        self.relationships = defaultdict(list)

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        self.misp_event.load_file(filename)
        self.filename = filename
        self.report_id = "report--{}".format(self.misp_event.uuid)

    def buildEvent(self):
        self.__set_identity()
        self.read_attributes()
        report = self.eventReport()
        self.SDOs.insert(1, report)

    def eventReport(self):
        report_args = {'type': 'report', 'id': self.report_id, 'name': self.misp_event.info,
                       'created_by_ref': self.identity_id, 'published': self.misp_event.publish_timestamp,
                       'created': self.misp_event.date, 'object_refs': self.object_refs}
        if self.misp_event.Tag:
            labels = []
            for tag in self.misp_event.Tag:
                labels.append(tag.name)
            report_args['labels'] = labels
            if 'misp:tool="misp2stix2"' not in labels:
                report_args['labels'].append('misp:tool="misp2stix2"')
        else:
            report_args['labels'] = ['Threat-Report']
            report_args['labels'].append('misp:tool="misp2stix2"')
        if self.external_refs:
            report_args['external_references'] = self.external_refs
        return Report(**report_args)

    def saveFile(self):
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(json.dumps(self.SDOs, cls=base.STIXJSONEncoder))

    def __set_identity(self):
        org = self.misp_event.Orgc
        identity_id = 'identity--{}'.format(org['uuid'])
        identity = Identity(type="identity", id=identity_id,
                            name=org["name"], identity_class="organization")
        self.SDOs.append(identity)
        self.identity_id = identity_id

    def misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        self.categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']

    def read_attributes(self):
        self.misp_types()
        if hasattr(self.misp_event, 'attributes') and self.misp_event.attributes:
            for attribute in self.misp_event.attributes:
                attribute_type = attribute.type
                if attribute_type in non_indicator_attributes:
                    self.handle_non_indicator(attribute, attribute_type)
                else:
                    if attribute_type in self.categories_mapping['Person']:
                        self.handle_person(attribute)
                    elif attribute_type in mispTypesMapping:
                        self.handle_usual_type(attribute)
                    else:
                        self.add_custom(attribute)
        if hasattr(self.misp_event, 'objects') and self.misp_event.objects:
            self.load_objects_mapping()
            objects_to_parse = defaultdict(dict)
            misp_objects = self.misp_event.objects
            self.object_references, self.processes = self.fetch_object_references(misp_objects)
            for misp_object in misp_objects:
                to_ids = self.fetch_ids_flag(misp_object.attributes)
                name = misp_object.name
                if name == "vulnerability":
                    self.add_object_vulnerability(misp_object, to_ids)
                elif name == "course-of-action":
                    self.add_course_of_action(misp_object, from_object=True)
                elif name in ('pe', 'pe-section'):
                    objects_to_parse[name][misp_object.uuid] = to_ids, misp_object
                elif name in objectsMapping:
                    if  name == 'file' and misp_object.references:
                        to_parse = False
                        for reference in misp_object.references:
                            if reference.relationship_type == 'included-in' and reference.Object['name'] == "pe":
                                objects_to_parse[name][misp_object.uuid] = to_ids, misp_object
                                to_parse = True
                                break
                        if to_parse:
                            continue
                    try:
                        if to_ids or name == "stix2-pattern":
                            self.add_object_indicator(misp_object)
                        else:
                            self.add_object_observable(misp_object)
                    except:
                        self.add_object_custom(misp_object, to_ids)
                else:
                    self.add_object_custom(misp_object, to_ids)
            if objects_to_parse: self.resolve_objects2parse(objects_to_parse)
        if hasattr(self.misp_event, 'Galaxy') and self.misp_event.Galaxy:
            for galaxy in self.misp_event.Galaxy:
                self.parse_galaxy(galaxy, self.report_id)
        for source, targets in self.relationships.items():
            if source.startswith('report'): continue
            source_type,_ = source.split('--')
            for target in targets:
                target_type,_ = target.split('--')
                try:
                    relation = relationshipsSpecifications[source_type][target_type]
                except KeyError:
                    # custom relationship (suggested by iglocska)
                    relation = "has"
                relationship = Relationship(source_ref=source, target_ref=target, relationship_type=relation)
                self.append_object(relationship, relationship.id)

    def load_objects_mapping(self):
        self.objects_mapping = {
            'asn': {'observable': self.resolve_asn_observable,
                    'pattern': self.resolve_asn_pattern},
            'domain-ip': {'observable': self.resolve_domain_ip_observable,
                          'pattern': self.resolve_domain_ip_pattern},
            'email': {'observable': self.resolve_email_object_observable,
                      'pattern': self.resolve_email_object_pattern},
            'file': {'observable': self.resolve_file_observable,
                     'pattern': self.resolve_file_pattern},
            'ip-port': {'observable': self.resolve_ip_port_observable,
                        'pattern': self.resolve_ip_port_pattern},
            'network-socket': {'observable': self.resolve_network_socket_observable,
                               'pattern': self.resolve_network_socket_pattern},
            'process': {'observable': self.resolve_process_observable,
                        'pattern': self.resolve_process_pattern},
            'registry-key': {'observable': self.resolve_regkey_observable,
                             'pattern': self.resolve_regkey_pattern},
            'stix2': {'pattern': self.resolve_stix2_pattern},
            'url': {'observable': self.resolve_url_observable,
                    'pattern': self.resolve_url_pattern},
            'x509': {'observable': self.resolve_x509_observable,
                     'pattern': self.resolve_x509_pattern}
        }
        self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', self.add_vulnerability]}
        self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', self.add_attack_pattern]))
        self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', self.add_course_of_action]))
        self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', self.add_intrusion_set]))
        self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', self.add_malware]))
        self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', self.add_threat_actor]))
        self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', self.add_tool]))

    def fetch_object_references(self, misp_objects):
        object_references, processes = {}, {}
        for misp_object in misp_objects:
            attributes = misp_object.attributes
            if misp_object.name == "process":
                self.get_process_attributes(processes, misp_object.attributes)
            if misp_object.references and not self.fetch_ids_flag(attributes):
                for reference in misp_object.references:
                    try:
                        referenced_object = reference.Attribute
                    except:
                        referenced_object = self.misp_event.get_object_by_uuid(reference.referenced_uuid)
                    object_references[reference.referenced_uuid] = referenced_object
        return object_references, processes

    @staticmethod
    def get_process_attributes(processes, attributes):
        pid, process = None, {}
        for attribute in attributes:
            relation = attribute.object_relation
            attribute_value = attribute.value
            if relation == 'pid':
                if attribute_value in processes:
                    return
                pid = attribute_value
                process[relation] = attribute_value
            elif relation in ('name', 'creation-time'):
                process[relation] = attribute_value
        if pid is not None:
            process['type'] = 'process'
            processes[pid] = process

    def handle_non_indicator(self, attribute, attribute_type):
        if attribute_type == "link":
            self.handle_link(attribute)
        elif attribute_type in ('text', 'comment', 'other') or attribute_type not in mispTypesMapping:
            self.add_custom(attribute)
        else:
            try:
                self.handle_non_indicator_attribute(attribute, attribute_type)
            except:
                self.add_custom(attribute)

    def handle_non_indicator_attribute(self, attribute, attribute_type):
        if attribute_type == "vulnerability":
            self.add_vulnerability(attribute, from_galaxy=False)
        else:
            self.add_observed_data(attribute)

    def handle_person(self, attribute):
        if attribute.category == "Person":
            self.add_identity(attribute)
        else:
            self.add_custom(attribute)

    def handle_usual_type(self, attribute):
        try:
            if attribute.to_ids:
                self.add_indicator(attribute)
            else:
                self.add_observed_data(attribute)
        except:
            self.add_custom(attribute)

    def handle_link(self, attribute):
        url = attribute.value
        source = "url"
        try:
            if hasattr(attribute, 'comment') and attribute.comment:
                source += " - {}".format(attribute.comment)
        except AttributeError:
            pass
        link = {'source_name': source, 'url': url}
        self.external_refs.append(link)

    def resolve_objects2parse(self, objects2parse):
        for uuid, misp_object in objects2parse['file'].items():
            to_ids_file, file_object = misp_object
            file_id = "file--{}".format(file_object.uuid)
            to_ids_list = [to_ids_file]
            object2create = defaultdict(list)
            for reference in file_object.references:
                if reference.relationship_type == "included-in" and reference.Object['name'] == "pe":
                    pe_uuid = reference.referenced_uuid
                    break
            to_ids_pe, pe_object = objects2parse['pe'][pe_uuid]
            to_ids_list.append(to_ids_pe)
            sections = []
            for reference in pe_object.references:
                if reference.Object['name'] == "pe-section":
                    to_ids_section, section_object = objects2parse['pe-section'][reference.referenced_uuid]
                    to_ids_list.append(to_ids_section)
                    sections.append(section_object)
            if True in to_ids_list:
                pattern = self.resolve_file_pattern(file_object.attributes, file_id)[1:-1]
                pattern += " AND {}".format(self.parse_pe_extensions_pattern(pe_object, sections))
                self.add_object_indicator(file_object, pattern_arg="[{}]".format(pattern))
            else:
                observable = self.resolve_file_observable(file_object.attributes, file_id)
                observable['1']['extensions'] = self.parse_pe_extensions_observable(pe_object, sections)
                self.add_object_observable(file_object, observable_arg=observable)

    def parse_pe_extensions_observable(self, pe_object, sections):
        extension = defaultdict(list)
        for attribute in pe_object.attributes:
            try:
                extension[peMapping[attribute.object_relation]] = attribute.value
            except KeyError:
                extension["x_misp_{}_{}".format(attribute.type, attribute.object_relation.replace('-', '_'))] = attribute.value
        for section in sections:
            d_section = defaultdict(dict)
            for attribute in section.attributes:
                relation = attribute.object_relation
                if relation in misp_hash_types:
                    d_section['hashes'][relation] = attribute.value
                else:
                    try:
                        d_section[peSectionMapping[relation]] = attribute.value
                    except KeyError:
                        continue
            extension['sections'].append(WindowsPESection(**d_section))
        return {"windows-pebinary-ext": extension}

    def parse_pe_extensions_pattern(self, pe_object, sections):
        pattern = ""
        mapping = objectsMapping['file']['pattern']
        pe_mapping = "extensions.'windows-pebinary-ext'"
        for attribute in pe_object.attributes:
            try:
                stix_type = "{}.{}".format(pe_mapping, peMapping[attribute.object_relation])
            except KeyError:
                stix_type = "{}.{}'".format(pe_mapping[:-1], "x_misp_{}_{}".format(attribute.type, attribute.object_relation.replace('-', '_')))
            pattern += mapping.format(stix_type, attribute.value)
        n_section = 0
        for section in sections:
            section_mapping = "{}.sections[{}]".format(pe_mapping, str(n_section))
            for attribute in section.attributes:
                relation = attribute.object_relation
                if relation in misp_hash_types:
                    stix_type = "{}.hashes.'{}'".format(section_mapping, relation)
                    pattern += mapping.format(stix_type, attribute.value)
                else:
                    try:
                        stix_type = "{}.{}".format(section_mapping, peSectionMapping[relation])
                        pattern += mapping.format(stix_type, attribute.value)
                    except KeyError:
                        continue
            n_section += 1
        return pattern[:-5]

    def parse_galaxies(self, galaxies, source_id):
        for galaxy in galaxies:
            self.parse_galaxy(galaxy, source_id)

    def parse_galaxy(self, galaxy, source_id):
        galaxy_type = galaxy.get('type')
        galaxy_uuid = galaxy['GalaxyCluster'][0]['uuid']
        try:
            stix_type, to_call = self.galaxies_mapping[galaxy_type]
        except:
            return
        if galaxy_uuid not in self.galaxies:
            to_call(galaxy)
            self.galaxies.append(galaxy_uuid)
        self.relationships[source_id].append("{}--{}".format(stix_type, galaxy_uuid))

    @staticmethod
    def generate_galaxy_args(galaxy, b_killchain, b_alias, sdo_type):
        galaxy_type = galaxy.get('type')
        name = galaxy.get('name')
        cluster = galaxy['GalaxyCluster'][0]
        sdo_id = "{}--{}".format(sdo_type, cluster.get('uuid'))
        description = "{} | {}".format(galaxy.get('description'), cluster.get('description'))
        labels = ['misp:type=\"{}\"'.format(galaxy_type)]
        sdo_args = {'id': sdo_id, 'type': sdo_type, 'name': name, 'description': description}
        if b_killchain:
            killchain = [{'kill_chain_name': 'misp-category',
                          'phase_name': galaxy_type}]
            sdo_args['kill_chain_phases'] = killchain
        if cluster['tag_name']:
            labels.append(cluster.get('tag_name'))
        meta = cluster.get('meta')
        if 'synonyms' in meta and b_alias:
            aliases = []
            for a in meta['synonyms']:
                aliases.append(a)
            sdo_args['aliases'] = aliases
        sdo_args['labels'] = labels
        return sdo_args, sdo_id

    def add_attack_pattern(self, galaxy):
        a_p_args, a_p_id = self.generate_galaxy_args(galaxy, True, False, 'attack-pattern')
        a_p_args['created_by_ref'] = self.identity_id
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern, a_p_id)

    def add_course_of_action(self, misp_object, from_object=False):
        if from_object:
            coa_id = 'course-of-action--{}'.format(misp_object.uuid)
            coa_args = {'id': coa_id, 'type': 'course-of-action'}
            for attribute in misp_object.attributes:
                self.parse_galaxies(attribute.Galaxy, coa_id)
                relation = attribute.object_relation
                if relation == 'name':
                    coa_args['name'] = attribute.value
                elif relation == 'description':
                    coa_args['description'] = attribute.value
            if not 'name' in coa_args:
                return
        else:
            coa_args, coa_id = self.generate_galaxy_args(misp_object, False, False, 'course-of-action')
        coa_args['created_by_ref'] = self.identity_id
        course_of_action = CourseOfAction(**coa_args)
        self.append_object(course_of_action, coa_id)

    def add_custom(self, attribute):
        custom_object_id = "x-misp-object--{}".format(attribute.uuid)
        custom_object_type = "x-misp-object-{}".format(attribute.type.replace('|', '-').lower())
        labels = self.create_labels(attribute)
        custom_object_args = {'id': custom_object_id, 'x_misp_timestamp': attribute.timestamp, 'labels': labels,
                               'x_misp_value': attribute.value, 'created_by_ref': self.identity_id,
                               'x_misp_category': attribute.category}
        if hasattr(attribute, 'comment') and attribute.comment:
            custom_object_args['x_misp_comment'] = attribute.comment
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                          ('x_misp_timestamp', properties.StringProperty(required=True)),
                                          ('labels', properties.ListProperty(labels, required=True)),
                                          ('x_misp_value', properties.StringProperty(required=True)),
                                          ('created_by_ref', properties.StringProperty(required=True)),
                                          ('x_misp_comment', properties.StringProperty()),
                                          ('x_misp_category', properties.StringProperty())
                                         ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object, custom_object_id)

    def add_identity(self, attribute):
        identity_id = "identity--{}".format(attribute.uuid)
        name = attribute.value
        labels = self.create_labels(attribute)
        identity_args = {'id': identity_id,  'type': identity, 'name': name, 'labels': labels,
                          'identity_class': 'individual', 'created_by_ref': self.identity_id}
        if hasattr(attribute, 'comment') and attribute.comment:
            identity_args['description'] = attribute.comment
        identity = Identity(**identity_args)
        self.append_object(identity, identity_id)

    def add_indicator(self, attribute):
        attribute_type = attribute.type
        indicator_id = "indicator--{}".format(attribute.uuid)
        self.parse_galaxies(attribute.Galaxy, indicator_id)
        category = attribute.category
        killchain = self.create_killchain(category)
        labels = self.create_labels(attribute)
        attribute_value = attribute.value if attribute_type != "AS" else self.define_attribute_value(attribute.value, attribute.comment)
        pattern = mispTypesMapping[attribute_type]['pattern'](attribute_type, attribute_value, b64encode(attribute.data.getbuffer()).decode()[1:-1]) if ('data' in attribute and attribute.data) else self.define_pattern(attribute_type, attribute_value)
        indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, 'kill_chain_phases': killchain,
                           'valid_from': self.misp_event.date, 'created_by_ref': self.identity_id, 'pattern': pattern}
        if hasattr(attribute, 'Sighting'):
            for sighting in attribute.Sighting:
                if sighting['Organisation']['name'] == self.misp_event.Orgc.name and sighting['type'] == "2":
                    indicator_args['valid_until'] = datetime.datetime.fromtimestamp(int(sighting['date_sighting']), datetime.timezone.utc).isoformat()
                    break
        if hasattr(attribute, 'comment') and attribute.comment:
            indicator_args['description'] = attribute.comment
        indicator = Indicator(**indicator_args)
        self.append_object(indicator, indicator_id)

    def add_intrusion_set(self, galaxy):
        i_s_args, i_s_id = self.generate_galaxy_args(galaxy, False, True, 'intrusion-set')
        i_s_args['created_by_ref'] = self.identity_id
        intrusion_set = IntrusionSet(**i_s_args)
        self.append_object(intrusion_set, i_s_id)

    def add_malware(self, galaxy):
        malware_args, malware_id = self.generate_galaxy_args(galaxy, True, False, 'malware')
        malware_args['created_by_ref'] = self.identity_id
        malware = Malware(**malware_args)
        self.append_object(malware, malware_id)

    def add_observed_data(self, attribute):
        attribute_type = attribute.type
        observed_data_id = "observed-data--{}".format(attribute.uuid)
        self.parse_galaxies(attribute.Galaxy, observed_data_id)
        timestamp = attribute.timestamp
        labels = self.create_labels(attribute)
        attribute_value = attribute.value if attribute_type != "AS" else self.define_attribute_value(attribute.value, attribute.comment)
        observable = mispTypesMapping[attribute_type]['observable'](attribute_type, attribute_value, b64encode(attribute.data.getbuffer())) if 'data' in attribute else self.define_observable(attribute_type, attribute_value)
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'number_observed': 1,
                              'first_observed': timestamp, 'last_observed': timestamp, 'labels': labels,
{'fraction_non_alphanumeric': 0.10583659093548013, 'fraction_numerical': 0.003812362116344777, 'mean_word_length': 2.3287168255604227, 'pattern_counts': {'":': 24, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2540231', 'n_tokens_mistral': 14552, 'n_tokens_neox': 13995, 'n_words': 2616}