<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime
import pymisp
import re
import uuid
from stix2 import *
from misp2stix2_mapping import *
from collections import defaultdict
from copy import deepcopy

misp_hash_types = ("authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh")
attack_pattern_galaxies_list = ('mitre-attack-pattern', 'mitre-enterprise-attack-attack-pattern',
                                'mitre-mobile-attack-attack-pattern', 'mitre-pre-attack-attack-pattern')
course_of_action_galaxies_list = ('mitre-course-of-action', 'mitre-enterprise-attack-course-of-action',
                                  'mitre-mobile-attack-course-of-action')
intrusion_set_galaxies_list = ('mitre-enterprise-attack-intrusion-set', 'mitre-mobile-attack-intrusion-set',
                               'mitre-pre-attack-intrusion-set', 'mitre-intrusion-set')
malware_galaxies_list = ('android', 'banker', 'stealer', 'backdoor', 'ransomware', 'mitre-malware',
                         'mitre-enterprise-attack-malware', 'mitre-mobile-attack-malware')
threat_actor_galaxies_list = ('threat-actor', 'microsoft-activity-group')
tool_galaxies_list = ('botnet', 'rat', 'exploit-kit', 'tds', 'tool', 'mitre-tool',
                      'mitre-enterprise-attack-tool', 'mitre-mobile-attack-tool')
_MISP_event_tags = ['Threat-Report', 'misp:tool="misp2stix2"']

class StixBuilder():
    def __init__(self):
        self.orgs = []
        self.galaxies = []

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        with open(filename, 'rt', encoding='utf-8') as f:
            self.json_event = json.loads(f.read())
        self.filename = filename
        self.load_objects_mapping()
        self.load_galaxy_mapping()

    def buildEvent(self):
        self.initialize_misp_types()
        stix_packages = [sdo for event in self.json_event['response'] for sdo in self.handler(event['Event'])] if self.json_event.get('response') else self.handler(self.json_event['Event'])
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'wt', encoding='utf-8') as f:
            f.write(json.dumps(stix_packages, cls=base.STIXJSONEncoder))
        print(json.dumps({'success': 1}))

    def eventReport(self):
        if not self.object_refs and self.links:
            self.add_custom(self.links.pop(0))
        external_refs = [self.__parse_link(link) for link in self.links]
        report_args = {'type': 'report', 'id': self.report_id, 'name': self.misp_event['info'],
                       'created_by_ref': self.identity_id, 'created': self.misp_event['date'],
                       'published': self.get_datetime_from_timestamp(self.misp_event['publish_timestamp']),
                       'interoperability': True}
        labels = [tag for tag in _MISP_event_tags]
        if self.misp_event.get('Tag'):
            markings = []
            for tag in self.misp_event['Tag']:
                name = tag['name']
                markings.append(name) if name.startswith('tlp:') else labels.append(name)
            if markings:
                report_args['object_marking_refs'] = self.handle_tags(markings)
        report_args['labels'] = labels
        if external_refs:
            report_args['external_references'] = external_refs
        self.add_all_markings()
        self.add_all_relationships()
        report_args['object_refs'] = self.object_refs
        return Report(**report_args)

    @staticmethod
    def __parse_link(link):
        url = link['value']
        source = "url"
        if link.get('comment'):
            source += " - {}".format(link['comment'])
        return {'source_name': source, 'url': url}

    def add_all_markings(self):
        for marking_args in self.markings.values():
            marking = MarkingDefinition(**marking_args)
            self.append_object(marking)

    def add_all_relationships(self):
        for source, targets in self.relationships.items():
            if source.startswith('report'):
                continue
            source_type,_ = source.split('--')
            for target in targets:
                target_type,_ = target.split('--')
                try:
                    relation = relationshipsSpecifications[source_type][target_type]
                except KeyError:
                    # custom relationship (suggested by iglocska)
                    relation = "has"
                relationship = Relationship(source_ref=source, target_ref=target,
                                            relationship_type=relation, interoperability=True)
                self.append_object(relationship, relationship.id)

    def __set_identity(self):
        org = self.misp_event['Orgc']
        org_uuid = org['uuid']
        identity_id = 'identity--{}'.format(org_uuid)
        self.identity_id = identity_id
        if org_uuid not in self.orgs:
            identity = Identity(type="identity", id=identity_id, name=org["name"],
                                identity_class="organization", interoperability=True)
            self.SDOs.append(identity)
            self.orgs.append(org_uuid)
            return 1
        return 0

    def initialize_misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']
        for category in categories_mapping:
            mispTypesMapping[category] = {'to_call': 'handle_person'}

    def handler(self, event):
        self.misp_event = event
        self.report_id = "report--{}".format(self.misp_event['uuid'])
        self.SDOs = []
        self.object_refs = []
        self.links = []
        self.markings = {}
        self.relationships = defaultdict(list)
        i = self.__set_identity()
        if self.misp_event.get('Attribute'):
            for attribute in self.misp_event['Attribute']:
                try:
                    getattr(self, mispTypesMapping[attribute['type']]['to_call'])(attribute)
                except KeyError:
                    self.add_custom(attribute)
        if self.misp_event.get('Object'):
            self.objects_to_parse = defaultdict(dict)
            misp_objects = self.misp_event['Object']
            for misp_object in misp_objects:
                name = misp_object['name']
                if name == 'original-imported-file':
                    continue
                to_ids = self.fetch_ids_flag(misp_object['Attribute'])
                try:
                    getattr(self, objectsMapping[name]['to_call'])(misp_object, to_ids)
                except KeyError:
                    self.add_object_custom(misp_object, to_ids)
            if self.objects_to_parse:
                self.resolve_objects2parse()
        if self.misp_event.get('Galaxy'):
            for galaxy in self.misp_event['Galaxy']:
                self.parse_galaxy(galaxy, self.report_id)
        report = self.eventReport()
        self.SDOs.insert(i, report)
        return self.SDOs

    def load_objects_mapping(self):
        self.objects_mapping = {
            'asn': {'observable': self.resolve_asn_observable,
                    'pattern': self.resolve_asn_pattern},
            'credential': {'observable': self.resolve_credential_observable,
                           'pattern': self.resolve_credential_pattern},
            'domain-ip': {'observable': self.resolve_domain_ip_observable,
                          'pattern': self.resolve_domain_ip_pattern},
            'email': {'observable': self.resolve_email_object_observable,
                      'pattern': self.resolve_email_object_pattern},
            'file': {'observable': self.resolve_file_observable,
                     'pattern': self.resolve_file_pattern},
            'ip-port': {'observable': self.resolve_ip_port_observable,
                        'pattern': self.resolve_ip_port_pattern},
            'network-connection': {'observable': self.resolve_network_connection_observable,
                                   'pattern': self.resolve_network_connection_pattern},
            'network-socket': {'observable': self.resolve_network_socket_observable,
                               'pattern': self.resolve_network_socket_pattern},
            'process': {'observable': self.resolve_process_observable,
                        'pattern': self.resolve_process_pattern},
            'registry-key': {'observable': self.resolve_regkey_observable,
                             'pattern': self.resolve_regkey_pattern},
            'stix2': {'pattern': self.resolve_stix2_pattern},
            'url': {'observable': self.resolve_url_observable,
                    'pattern': self.resolve_url_pattern},
            'user-account': {'observable': self.resolve_user_account_observable,
                             'pattern': self.resolve_user_account_pattern},
            'x509': {'observable': self.resolve_x509_observable,
                     'pattern': self.resolve_x509_pattern}
        }

    def load_galaxy_mapping(self):
        self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', self.add_vulnerability_from_galaxy]}
        self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', self.add_attack_pattern]))
        self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', self.add_course_of_action]))
        self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', self.add_intrusion_set]))
        self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', self.add_malware]))
        self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', self.add_threat_actor]))
        self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', self.add_tool]))

    def get_object_by_uuid(self, uuid):
        for _object in self.misp_event['Object']:
            if _object.get('uuid') and _object['uuid'] == uuid:
                return _object
        raise Exception('Object with uuid {} does not exist in this event.'.format(uuid))

    def handle_person(self, attribute):
        if attribute['category'] == "Person":
            self.add_identity(attribute)
        else:
            self.add_custom(attribute)

    def handle_usual_type(self, attribute):
        try:
            if attribute['to_ids']:
                self.add_indicator(attribute)
            else:
                self.add_observed_data(attribute)
        except Exception:
            self.add_custom(attribute)

    def handle_usual_object_name(self, misp_object, to_ids):
        name = misp_object['name']
        if  name == 'file' and misp_object.get('ObjectReference'):
            for reference in misp_object['ObjectReference']:
                if reference['relationship_type'] in ('includes',  'included-in') and reference['Object']['name'] == "pe":
                    self.objects_to_parse[name][misp_object['uuid']] = to_ids, misp_object
                    return
        try:
            if to_ids or name == "stix2-pattern":
                self.add_object_indicator(misp_object)
            else:
                self.add_object_observable(misp_object)
        except Exception:
            self.add_object_custom(misp_object, to_ids)

    def handle_link(self, attribute):
        self.links.append(attribute)

    def populate_objects_to_parse(self, misp_object, to_ids):
        self.objects_to_parse[misp_object['name']][misp_object['uuid']] = to_ids, misp_object

    def resolve_objects2parse(self):
        for uuid, misp_object in self.objects_to_parse['file'].items():
            to_ids_file, file_object = misp_object
            file_id = "file--{}".format(file_object['uuid'])
            to_ids_list = [to_ids_file]
            for reference in file_object['ObjectReference']:
                if reference['relationship_type'] in ("includes", "included-in") and reference['Object']['name'] == "pe":
                    pe_uuid = reference['referenced_uuid']
                    break
            to_ids_pe, pe_object = self.objects_to_parse['pe'][pe_uuid]
            to_ids_list.append(to_ids_pe)
            sections = []
            for reference in pe_object['ObjectReference']:
                if reference['Object']['name'] == "pe-section" and reference['referenced_uuid'] in self.objects_to_parse['pe-section']:
                    to_ids_section, section_object = self.objects_to_parse['pe-section'][reference['referenced_uuid']]
                    to_ids_list.append(to_ids_section)
                    sections.append(section_object)
            if True in to_ids_list:
                pattern = self.resolve_file_pattern(file_object['Attribute'], file_id)[1:-1]
                pattern += " AND {}".format(self.parse_pe_extensions_pattern(pe_object, sections))
                self.add_object_indicator(file_object, pattern_arg="[{}]".format(pattern))
            else:
                observable = self.resolve_file_observable(file_object['Attribute'], file_id)
                pe_type = self._get_pe_type_from_filename(observable['0'])
                observable['0']['extensions'] = self.parse_pe_extensions_observable(pe_object, sections, pe_type)
                self.add_object_observable(file_object, observable_arg=observable)

    @staticmethod
    def _create_pe_type_test(observable, extension):
        return [
            ('name' in observable and observable['name'].endswith('.%s' % extension)),
            ('mime_type' in observable and re.compile(".* .+{0}.+ .*|.* {0} .*".format(extension)).match(observable['mime_type'].lower()))]

    def _get_pe_type_from_filename(self, observable):
        for extension in ('exe', 'dll'):
            if any(self._create_pe_type_test(observable, extension)):
                return extension
        return 'sys'

    def parse_pe_extensions_observable(self, pe_object, sections, pe_type):
        extension = defaultdict(list)
        extension['pe_type'] = pe_type
        for attribute in pe_object['Attribute']:
            try:
                extension[peMapping[attribute['object_relation']]] = attribute['value']
            except KeyError:
                extension["x_misp_{}_{}".format(attribute['type'], attribute['object_relation'].replace('-', '_'))] = attribute['value']
        for section in sections:
            d_section = defaultdict(dict)
            for attribute in section['Attribute']:
                relation = attribute['object_relation']
                if relation in misp_hash_types:
                    d_section['hashes'][relation] = attribute['value']
                else:
                    try:
                        d_section[peSectionMapping[relation]] = attribute['value']
                    except KeyError:
                        continue
            if 'name' not in d_section:
                d_section['name'] = 'Section {}'.format(sections.index(section))
            extension['sections'].append(WindowsPESection(**d_section))
        if len(sections) != int(extension['number_of_sections']):
            extension['number_of_sections'] = str(len(sections))
        return {"windows-pebinary-ext": extension}

    def parse_pe_extensions_pattern(self, pe_object, sections):
        pattern = ""
        mapping = objectsMapping['file']['pattern']
        pe_mapping = "extensions.'windows-pebinary-ext'"
        for attribute in pe_object['Attribute']:
            try:
                stix_type = "{}.{}".format(pe_mapping, peMapping[attribute['object_relation']])
            except KeyError:
                stix_type = "{}.{}'".format(pe_mapping[:-1], "x_misp_{}_{}".format(attribute['type'], attribute['object_relation'].replace('-', '_')))
            pattern += mapping.format(stix_type, attribute['value'])
        n_section = 0
        for section in sections:
            section_mapping = "{}.sections[{}]".format(pe_mapping, str(n_section))
            for attribute in section['Attribute']:
                relation = attribute['object_relation']
                if relation in misp_hash_types:
                    stix_type = "{}.hashes.'{}'".format(section_mapping, relation)
                    pattern += mapping.format(stix_type, attribute['value'])
                else:
                    try:
                        stix_type = "{}.{}".format(section_mapping, peSectionMapping[relation])
                        pattern += mapping.format(stix_type, attribute['value'])
                    except KeyError:
                        continue
            n_section += 1
        return pattern[:-5]

    def parse_galaxies(self, galaxies, source_id):
        for galaxy in galaxies:
            self.parse_galaxy(galaxy, source_id)

    def parse_galaxy(self, galaxy, source_id):
        galaxy_type = galaxy.get('type')
        galaxy_uuid = galaxy['GalaxyCluster'][0]['collection_uuid']
        try:
            stix_type, to_call = self.galaxies_mapping[galaxy_type]
        except Exception:
            return
        if galaxy_uuid not in self.galaxies:
            to_call(galaxy)
            self.galaxies.append(galaxy_uuid)
        self.relationships[source_id].append("{}--{}".format(stix_type, galaxy_uuid))

    @staticmethod
    def generate_galaxy_args(galaxy, b_killchain, b_alias, sdo_type):
        cluster = galaxy['GalaxyCluster'][0]
        try:
            cluster_uuid = cluster['collection_uuid']
        except KeyError:
            cluster_uuid = cluster['uuid']
        sdo_id = "{}--{}".format(sdo_type, cluster_uuid)
        description = "{} | {}".format(galaxy['description'], cluster['description'])
        labels = ['misp:name=\"{}\"'.format(galaxy['name'])]
        sdo_args = {'id': sdo_id, 'type': sdo_type, 'name': cluster['value'],
                    'description': description, 'interoperability': True}
        if b_killchain:
            killchain = [{'kill_chain_name': 'misp-category',
                          'phase_name': galaxy['type']}]
            sdo_args['kill_chain_phases'] = killchain
        if cluster['tag_name']:
            labels.append(cluster.get('tag_name'))
        meta = cluster.get('meta')
        if 'synonyms' in meta and b_alias:
            aliases = []
            for a in meta['synonyms']:
                aliases.append(a)
            sdo_args['aliases'] = aliases
        sdo_args['labels'] = labels
        return sdo_args

    def add_attack_pattern(self, galaxy):
        a_p_args = self.generate_galaxy_args(galaxy, True, False, 'attack-pattern')
        a_p_args['created_by_ref'] = self.identity_id
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern)

    def add_attack_pattern_object(self, misp_object, to_ids):
        a_p_id = 'attack-pattern--{}'.format(misp_object['uuid'])
        attributes_dict = {attribute['object_relation']: attribute['value'] for attribute in misp_object['Attribute']}
        a_p_args = {'id': a_p_id, 'type': 'attack-pattern', 'created_by_ref': self.identity_id}
        a_p_args['labels'] = self.create_object_labels(misp_object['name'], misp_object['meta-category'], to_ids)
        for relation, key in attackPatternObjectMapping.items():
            if relation in attributes_dict:
                a_p_args[key] = attributes_dict[relation]
        if 'id' in attributes_dict:
            capec_id = "CAPEC-{}".format(attributes_dict['id'])
            a_p_args['external_references'] = [{'source_name': 'capec', 'external_id': capec_id}]
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern)

    def add_course_of_action(self, misp_object):
        coa_args= self.generate_galaxy_args(misp_object, False, False, 'course-of-action')
        self.add_coa_stix_object(coa_args)

    def add_course_of_action_from_object(self, misp_object, _):
        coa_id = 'course-of-action--{}'.format(misp_object['uuid'])
        coa_args = {'id': coa_id, 'type': 'course-of-action'}
        for attribute in misp_object['Attribute']:
            self.parse_galaxies(attribute['Galaxy'], coa_id)
            relation = attribute['object_relation']
            if relation == 'name':
                coa_args['name'] = attribute['value']
            elif relation == 'description':
                coa_args['description'] = attribute['value']
        if not 'name' in coa_args:
            return
        self.add_coa_stix_object(coa_args)

    def add_coa_stix_object(self, coa_args):
        coa_args['created_by_ref'] = self.identity_id
        course_of_action = CourseOfAction(**coa_args)
        self.append_object(course_of_action)

    def add_custom(self, attribute):
        custom_object_id = "x-misp-object--{}".format(attribute['uuid'])
        custom_object_type = "x-misp-object-{}".format(attribute['type'].replace('|', '-').replace(' ', '-').lower())
        labels, markings = self.create_labels(attribute)
        custom_object_args = {'id': custom_object_id, 'x_misp_category': attribute['category'], 'labels': labels,
                              'x_misp_timestamp': self.get_datetime_from_timestamp(attribute['timestamp']),
                              'x_misp_value': attribute['value'], 'created_by_ref': self.identity_id}
        if attribute.get('comment'):
            custom_object_args['x_misp_comment'] = attribute['comment']
        if markings:
            markings = self.handle_tags(markings)
            custom_object_args['object_marking_refs'] = markings
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                          ('x_misp_timestamp', properties.StringProperty(required=True)),
                                          ('labels', properties.ListProperty(labels, required=True)),
                                          ('x_misp_value', properties.StringProperty(required=True)),
                                          ('created_by_ref', properties.StringProperty(required=True)),
                                          ('object_marking_refs', properties.ListProperty(markings)),
                                          ('x_misp_comment', properties.StringProperty()),
                                          ('x_misp_category', properties.StringProperty())
                                         ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object)

    def add_identity(self, attribute):
        identity_id = "identity--{}".format(attribute['uuid'])
        name = attribute['value']
        labels, markings = self.create_labels(attribute)
        identity_args = {'id': identity_id,  'type': identity, 'name': name, 'labels': labels,
                          'identity_class': 'individual', 'created_by_ref': self.identity_id,
                          'interoperability': True}
        if attribute.get('comment'):
            identity_args['description'] = attribute['comment']
        if markings:
            identity_args['object_marking_refs'] = self.handle_tags(markings)
        identity = Identity(**identity_args)
        self.append_object(identity)

    def add_indicator(self, attribute):
        attribute_type = attribute['type']
        indicator_id = "indicator--{}".format(attribute['uuid'])
        self.parse_galaxies(attribute['Galaxy'], indicator_id)
        category = attribute['category']
        killchain = self.create_killchain(category)
        labels, markings = self.create_labels(attribute)
        attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment'])
        pattern = mispTypesMapping[attribute_type]['pattern'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_pattern(attribute_type, attribute_value)
        indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, 'kill_chain_phases': killchain,
                           'valid_from': self.misp_event['date'], 'created_by_ref': self.identity_id,
                           'pattern': pattern, 'interoperability': True}
        if hasattr(attribute, 'Sighting'):
            for sighting in attribute['Sighting']:
                if sighting['Organisation']['name'] == self.misp_event['Orgc']['name'] and sighting['type'] == "2":
                    indicator_args['valid_until'] = self.get_datetime_from_timestamp(sighting['date_sighting'])
                    break
        if attribute.get('comment'):
            indicator_args['description'] = attribute['comment']
        if markings:
            indicator_args['object_marking_refs'] = self.handle_tags(markings)
        indicator = Indicator(**indicator_args)
        self.append_object(indicator)

    def add_intrusion_set(self, galaxy):
        i_s_args = self.generate_galaxy_args(galaxy, False, True, 'intrusion-set')
        i_s_args['created_by_ref'] = self.identity_id
        intrusion_set = IntrusionSet(**i_s_args)
        self.append_object(intrusion_set)

    def add_malware(self, galaxy):
        malware_args= self.generate_galaxy_args(galaxy, True, False, 'malware')
        malware_args['created_by_ref'] = self.identity_id
        malware = Malware(**malware_args)
        self.append_object(malware)

    def add_observed_data(self, attribute):
        attribute_type = attribute['type']
        observed_data_id = "observed-data--{}".format(attribute['uuid'])
        self.parse_galaxies(attribute['Galaxy'], observed_data_id)
        timestamp = self.get_datetime_from_timestamp(attribute['timestamp'])
        labels, markings = self.create_labels(attribute)
        attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment'])
        observable = mispTypesMapping[attribute_type]['observable'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_observable(attribute_type, attribute_value)
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'number_observed': 1,
                              'first_observed': timestamp, 'last_observed': timestamp, 'labels': labels,
                              'created_by_ref': self.identity_id, 'objects': observable, 'interoperability': True}
        if markings:
            observed_data_args['object_marking_refs'] = self.handle_tags(markings)
        observed_data = ObservedData(**observed_data_args)
        self.append_object(observed_data)

    def add_threat_actor(self, galaxy):
        t_a_args = self.generate_galaxy_args(galaxy, False, True, 'threat-actor')
        t_a_args['created_by_ref'] = self.identity_id
        threat_actor = ThreatActor(**t_a_args)
        self.append_object(threat_actor)

    def add_tool(self, galaxy):
        tool_args = self.generate_galaxy_args(galaxy, True, False, 'tool')
        tool_args['created_by_ref'] = self.identity_id
        tool = Tool(**tool_args)
        self.append_object(tool)

    def add_vulnerability(self, attribute):
        vulnerability_id = "vulnerability--{}".format(attribute['uuid'])
        name = attribute['value']
        vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](name)]
        labels, markings = self.create_labels(attribute)
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'external_references': vulnerability_data,
                              'created_by_ref': self.identity_id, 'labels': labels,
                              'interoperability': True}
        if markings:
            vulnerability_args['object_marking_refs'] = self.handle_tags(markings)
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def add_vulnerability_from_galaxy(self, attribute):
        vulnerability_id = "vulnerability--{}".format(attribute['uuid'])
        cluster = attribute['GalaxyCluster'][0]
        name = cluster['value']
        if cluster['meta'] and cluster['meta']['aliases']:
            vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](alias) for alias in cluster['meta']['aliases']]
        else:
            vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](name)]
        labels = ['misp:type=\"{}\"'.format(attribute.get('type'))]
        if cluster['tag_name']:
            labels.append(cluster['tag_name'])
        description = "{} | {}".format(attribute.get('description'), cluster.get('description'))
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'external_references': vulnerability_data,
                              'created_by_ref': self.identity_id, 'labels': labels,
                              'description': description, 'interoperability': True}
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def add_object_custom(self, misp_object, to_ids):
        custom_object_id = 'x-misp-object--{}'.format(misp_object['uuid'])
        name = misp_object['name']
        custom_object_type = 'x-misp-object-{}'.format(name)
        category = misp_object.get('meta-category')
        labels = self.create_object_labels(name, category, to_ids)
        values = self.fetch_custom_values(misp_object['Attribute'], custom_object_id)
        custom_object_args = {'id': custom_object_id, 'x_misp_values': values, 'labels': labels,
                              'x_misp_category': category, 'created_by_ref': self.identity_id,
                              'x_misp_timestamp': self.get_datetime_from_timestamp(misp_object['timestamp'])}
        if hasattr(misp_object, 'comment') and misp_object.get('comment'):
            custom_object_args['x_misp_comment'] = misp_object['comment']
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                           ('x_misp_timestamp', properties.StringProperty(required=True)),
                                           ('labels', properties.ListProperty(labels, required=True)),
                                           ('x_misp_values', properties.DictionaryProperty(required=True)),
                                           ('created_by_ref', properties.StringProperty(required=True)),
                                           ('x_misp_comment', properties.StringProperty()),
                                           ('x_misp_category', properties.StringProperty())
                                          ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object)

    def add_object_indicator(self, misp_object, pattern_arg=None):
        indicator_id = 'indicator--{}'.format(misp_object['uuid'])
        if pattern_arg:
            name = 'WindowsPEBinaryFile'
            pattern = pattern_arg
        else:
            name = misp_object['name']
            pattern = self.objects_mapping[name]['pattern'](misp_object['Attribute'], indicator_id)
        category = misp_object.get('meta-category')
        killchain = self.create_killchain(category)
        labels = self.create_object_labels(name, category, True)
        indicator_args = {'id': indicator_id, 'valid_from': self.misp_event['date'],
                          'type': 'indicator', 'labels': labels, 'pattern': pattern,
                          'description': misp_object['description'], 'allow_custom': True,
                          'kill_chain_phases': killchain, 'interoperability': True,
                          'created_by_ref': self.identity_id}
        indicator = Indicator(**indicator_args)
        self.append_object(indicator)

    def add_object_observable(self, misp_object, observable_arg=None):
        observed_data_id = 'observed-data--{}'.format(misp_object['uuid'])
        if observable_arg:
            name = 'WindowsPEBinaryFile'
            observable_objects = observable_arg
        else:
            name = misp_object['name']
            observable_objects = self.objects_mapping[name]['observable'](misp_object['Attribute'], observed_data_id)
        category = misp_object.get('meta-category')
        labels = self.create_object_labels(name, category, False)
        timestamp = self.get_datetime_from_timestamp(misp_object['timestamp'])
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'labels': labels,
                              'number_observed': 1, 'objects': observable_objects, 'allow_custom': True,
                              'first_observed': timestamp, 'last_observed': timestamp,
                              'created_by_ref': self.identity_id, 'interoperability': True}
        try:
            observed_data = ObservedData(**observed_data_args)
        except exceptions.InvalidValueError:
            observed_data = self.fix_enumeration_issues(name, observed_data_args)
        self.append_object(observed_data)

    @staticmethod
    def fix_enumeration_issues(name, args):
        enumeration_fails = {}
        if name == 'network-socket':
            ns_args = deepcopy(args)
            observable_object = ns_args['objects']
            n = sorted(observable_object.keys())[-1]
            current_dict = observable_object[n]['extensions']['socket-ext']
            for field in ('address_family', 'protocol_family'):
                enumeration_fails[field] = current_dict.pop(field)
                try:
                    return ObservedData(**ns_args)
                except (exceptions.InvalidValueError, exceptions.MissingPropertiesError):
                    current_dict[field] = enumeration_fails[field]
            for field in enumeration_fails:
                current_dict.pop(field)
            return ObservedData(**ns_args)
        return ObservedData(**args)

    def add_object_vulnerability(self, misp_object, to_ids):
        vulnerability_id = 'vulnerability--{}'.format(misp_object['uuid'])
        name = self.fetch_vulnerability_name(misp_object['Attribute'])
        labels = self.create_object_labels(name, misp_object.get('meta-category'), to_ids)
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'created_by_ref': self.identity_id,
                              'labels': labels, 'interoperability': True}
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def append_object(self, stix_object):
        self.SDOs.append(stix_object)
        self.object_refs.append(stix_object.id)

    @staticmethod
    def create_killchain(category):
        return [{'kill_chain_name': 'misp-category', 'phase_name': category}]

    @staticmethod
    def create_labels(attribute):
        labels = ['misp:type="{}"'.format(attribute['type']),
                  'misp:category="{}"'.format(attribute['category']),<commit_msg>new: [stix2 export] Parsing relationships between objects<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime
import pymisp
import re
import uuid
from stix2 import *
from misp2stix2_mapping import *
from collections import defaultdict
from copy import deepcopy

misp_hash_types = ("authentihash", "ssdeep", "imphash", "md5", "sha1", "sha224",
                   "sha256", "sha384", "sha512", "sha512/224","sha512/256","tlsh")
attack_pattern_galaxies_list = ('mitre-attack-pattern', 'mitre-enterprise-attack-attack-pattern',
                                'mitre-mobile-attack-attack-pattern', 'mitre-pre-attack-attack-pattern')
course_of_action_galaxies_list = ('mitre-course-of-action', 'mitre-enterprise-attack-course-of-action',
                                  'mitre-mobile-attack-course-of-action')
intrusion_set_galaxies_list = ('mitre-enterprise-attack-intrusion-set', 'mitre-mobile-attack-intrusion-set',
                               'mitre-pre-attack-intrusion-set', 'mitre-intrusion-set')
malware_galaxies_list = ('android', 'banker', 'stealer', 'backdoor', 'ransomware', 'mitre-malware',
                         'mitre-enterprise-attack-malware', 'mitre-mobile-attack-malware')
threat_actor_galaxies_list = ('threat-actor', 'microsoft-activity-group')
tool_galaxies_list = ('botnet', 'rat', 'exploit-kit', 'tds', 'tool', 'mitre-tool',
                      'mitre-enterprise-attack-tool', 'mitre-mobile-attack-tool')
_MISP_event_tags = ['Threat-Report', 'misp:tool="misp2stix2"']

class StixBuilder():
    def __init__(self):
        self.orgs = []
        self.galaxies = []
        self.ids = {}

    def loadEvent(self, args):
        pathname = os.path.dirname(args[0])
        filename = os.path.join(pathname, args[1])
        with open(filename, 'rt', encoding='utf-8') as f:
            self.json_event = json.loads(f.read())
        self.filename = filename
        self.load_objects_mapping()
        self.load_galaxy_mapping()

    def buildEvent(self):
        self.initialize_misp_types()
        stix_packages = [sdo for event in self.json_event['response'] for sdo in self.handler(event['Event'])] if self.json_event.get('response') else self.handler(self.json_event['Event'])
        outputfile = "{}.out".format(self.filename)
        with open(outputfile, 'wt', encoding='utf-8') as f:
            f.write(json.dumps(stix_packages, cls=base.STIXJSONEncoder))
        print(json.dumps({'success': 1}))

    def eventReport(self):
        if not self.object_refs and self.links:
            self.add_custom(self.links.pop(0))
        external_refs = [self.__parse_link(link) for link in self.links]
        report_args = {'type': 'report', 'id': self.report_id, 'name': self.misp_event['info'],
                       'created_by_ref': self.identity_id, 'created': self.misp_event['date'],
                       'published': self.get_datetime_from_timestamp(self.misp_event['publish_timestamp']),
                       'interoperability': True}
        labels = [tag for tag in _MISP_event_tags]
        if self.misp_event.get('Tag'):
            markings = []
            for tag in self.misp_event['Tag']:
                name = tag['name']
                markings.append(name) if name.startswith('tlp:') else labels.append(name)
            if markings:
                report_args['object_marking_refs'] = self.handle_tags(markings)
        report_args['labels'] = labels
        if external_refs:
            report_args['external_references'] = external_refs
        self.add_all_markings()
        self.add_all_relationships()
        report_args['object_refs'] = self.object_refs
        return Report(**report_args)

    @staticmethod
    def __parse_link(link):
        url = link['value']
        source = "url"
        if link.get('comment'):
            source += " - {}".format(link['comment'])
        return {'source_name': source, 'url': url}

    def add_all_markings(self):
        for marking_args in self.markings.values():
            marking = MarkingDefinition(**marking_args)
            self.append_object(marking)

    def add_all_relationships(self):
        for source, targets in self.relationships['defined'].items():
            if source.startswith('report'):
                continue
            source_type,_ = source.split('--')
            for target in targets:
                target_type,_ = target.split('--')
                try:
                    relation = relationshipsSpecifications[source_type][target_type]
                except KeyError:
                    # custom relationship (suggested by iglocska)
                    relation = "has"
                relationship = Relationship(source_ref=source, target_ref=target,
                                            relationship_type=relation, interoperability=True)
                self.append_object(relationship, id_mapping=False)
        for source_uuid, references in self.relationships['to_define'].items():
            for reference in references:
                target_uuid, relationship_type = reference
                try:
                    source = '{}--{}'.format(self.ids[source_uuid], source_uuid)
                    target = '{}--{}'.format(self.ids[target_uuid], target_uuid)
                except KeyError:
                    continue
                relationship = Relationship(source_ref=source, relationship_type=relationship_type,
                                            target_ref=target, interoperability=True)
                self.append_object(relationship, id_mapping=False)

    def __set_identity(self):
        org = self.misp_event['Orgc']
        org_uuid = org['uuid']
        identity_id = 'identity--{}'.format(org_uuid)
        self.identity_id = identity_id
        if org_uuid not in self.orgs:
            identity = Identity(type="identity", id=identity_id, name=org["name"],
                                identity_class="organization", interoperability=True)
            self.SDOs.append(identity)
            self.orgs.append(org_uuid)
            return 1
        return 0

    def initialize_misp_types(self):
        describe_types_filename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
        describe_types = open(describe_types_filename, 'r')
        categories_mapping = json.loads(describe_types.read())['result']['category_type_mappings']
        for category in categories_mapping:
            mispTypesMapping[category] = {'to_call': 'handle_person'}

    def handler(self, event):
        self.misp_event = event
        self.report_id = "report--{}".format(self.misp_event['uuid'])
        self.SDOs = []
        self.object_refs = []
        self.links = []
        self.markings = {}
        self.relationships = {'defined': defaultdict(list),
                              'to_define': {}}
        i = self.__set_identity()
        if self.misp_event.get('Attribute'):
            for attribute in self.misp_event['Attribute']:
                try:
                    getattr(self, mispTypesMapping[attribute['type']]['to_call'])(attribute)
                except KeyError:
                    self.add_custom(attribute)
        if self.misp_event.get('Object'):
            self.objects_to_parse = defaultdict(dict)
            for misp_object in self.misp_event['Object']:
                name = misp_object['name']
                if name == 'original-imported-file':
                    continue
                to_ids = self.fetch_ids_flag(misp_object['Attribute'])
                try:
                    getattr(self, objectsMapping[name]['to_call'])(misp_object, to_ids)
                except KeyError:
                    self.add_object_custom(misp_object, to_ids)
                if misp_object.get('ObjectReference'):
                    self.relationships['to_define'][misp_object['uuid']] = tuple((r['referenced_uuid'], r['relationship_type']) for r in misp_object['ObjectReference'])
            if self.objects_to_parse:
                self.resolve_objects2parse()
        if self.misp_event.get('Galaxy'):
            for galaxy in self.misp_event['Galaxy']:
                self.parse_galaxy(galaxy, self.report_id)
        report = self.eventReport()
        self.SDOs.insert(i, report)
        return self.SDOs

    def load_objects_mapping(self):
        self.objects_mapping = {
            'asn': {'observable': self.resolve_asn_observable,
                    'pattern': self.resolve_asn_pattern},
            'credential': {'observable': self.resolve_credential_observable,
                           'pattern': self.resolve_credential_pattern},
            'domain-ip': {'observable': self.resolve_domain_ip_observable,
                          'pattern': self.resolve_domain_ip_pattern},
            'email': {'observable': self.resolve_email_object_observable,
                      'pattern': self.resolve_email_object_pattern},
            'file': {'observable': self.resolve_file_observable,
                     'pattern': self.resolve_file_pattern},
            'ip-port': {'observable': self.resolve_ip_port_observable,
                        'pattern': self.resolve_ip_port_pattern},
            'network-connection': {'observable': self.resolve_network_connection_observable,
                                   'pattern': self.resolve_network_connection_pattern},
            'network-socket': {'observable': self.resolve_network_socket_observable,
                               'pattern': self.resolve_network_socket_pattern},
            'process': {'observable': self.resolve_process_observable,
                        'pattern': self.resolve_process_pattern},
            'registry-key': {'observable': self.resolve_regkey_observable,
                             'pattern': self.resolve_regkey_pattern},
            'stix2': {'pattern': self.resolve_stix2_pattern},
            'url': {'observable': self.resolve_url_observable,
                    'pattern': self.resolve_url_pattern},
            'user-account': {'observable': self.resolve_user_account_observable,
                             'pattern': self.resolve_user_account_pattern},
            'x509': {'observable': self.resolve_x509_observable,
                     'pattern': self.resolve_x509_pattern}
        }

    def load_galaxy_mapping(self):
        self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', self.add_vulnerability_from_galaxy]}
        self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', self.add_attack_pattern]))
        self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', self.add_course_of_action]))
        self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', self.add_intrusion_set]))
        self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', self.add_malware]))
        self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', self.add_threat_actor]))
        self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', self.add_tool]))

    def get_object_by_uuid(self, uuid):
        for _object in self.misp_event['Object']:
            if _object.get('uuid') and _object['uuid'] == uuid:
                return _object
        raise Exception('Object with uuid {} does not exist in this event.'.format(uuid))

    def handle_person(self, attribute):
        if attribute['category'] == "Person":
            self.add_identity(attribute)
        else:
            self.add_custom(attribute)

    def handle_usual_type(self, attribute):
        try:
            if attribute['to_ids']:
                self.add_indicator(attribute)
            else:
                self.add_observed_data(attribute)
        except Exception:
            self.add_custom(attribute)

    def handle_usual_object_name(self, misp_object, to_ids):
        name = misp_object['name']
        if  name == 'file' and misp_object.get('ObjectReference'):
            for reference in misp_object['ObjectReference']:
                if reference['relationship_type'] in ('includes',  'included-in') and reference['Object']['name'] == "pe":
                    self.objects_to_parse[name][misp_object['uuid']] = to_ids, misp_object
                    return
        try:
            if to_ids or name == "stix2-pattern":
                self.add_object_indicator(misp_object)
            else:
                self.add_object_observable(misp_object)
        except Exception:
            self.add_object_custom(misp_object, to_ids)

    def handle_link(self, attribute):
        self.links.append(attribute)

    def populate_objects_to_parse(self, misp_object, to_ids):
        self.objects_to_parse[misp_object['name']][misp_object['uuid']] = to_ids, misp_object

    def resolve_objects2parse(self):
        for uuid, misp_object in self.objects_to_parse['file'].items():
            to_ids_file, file_object = misp_object
            file_id = "file--{}".format(file_object['uuid'])
            to_ids_list = [to_ids_file]
            for reference in file_object['ObjectReference']:
                if reference['relationship_type'] in ("includes", "included-in") and reference['Object']['name'] == "pe":
                    pe_uuid = reference['referenced_uuid']
                    break
            to_ids_pe, pe_object = self.objects_to_parse['pe'][pe_uuid]
            to_ids_list.append(to_ids_pe)
            sections = []
            for reference in pe_object['ObjectReference']:
                if reference['Object']['name'] == "pe-section" and reference['referenced_uuid'] in self.objects_to_parse['pe-section']:
                    to_ids_section, section_object = self.objects_to_parse['pe-section'][reference['referenced_uuid']]
                    to_ids_list.append(to_ids_section)
                    sections.append(section_object)
            if True in to_ids_list:
                pattern = self.resolve_file_pattern(file_object['Attribute'], file_id)[1:-1]
                pattern += " AND {}".format(self.parse_pe_extensions_pattern(pe_object, sections))
                self.add_object_indicator(file_object, pattern_arg="[{}]".format(pattern))
            else:
                observable = self.resolve_file_observable(file_object['Attribute'], file_id)
                pe_type = self._get_pe_type_from_filename(observable['0'])
                observable['0']['extensions'] = self.parse_pe_extensions_observable(pe_object, sections, pe_type)
                self.add_object_observable(file_object, observable_arg=observable)

    @staticmethod
    def _create_pe_type_test(observable, extension):
        return [
            ('name' in observable and observable['name'].endswith('.%s' % extension)),
            ('mime_type' in observable and re.compile(".* .+{0}.+ .*|.* {0} .*".format(extension)).match(observable['mime_type'].lower()))]

    def _get_pe_type_from_filename(self, observable):
        for extension in ('exe', 'dll'):
            if any(self._create_pe_type_test(observable, extension)):
                return extension
        return 'sys'

    def parse_pe_extensions_observable(self, pe_object, sections, pe_type):
        extension = defaultdict(list)
        extension['pe_type'] = pe_type
        for attribute in pe_object['Attribute']:
            try:
                extension[peMapping[attribute['object_relation']]] = attribute['value']
            except KeyError:
                extension["x_misp_{}_{}".format(attribute['type'], attribute['object_relation'].replace('-', '_'))] = attribute['value']
        for section in sections:
            d_section = defaultdict(dict)
            for attribute in section['Attribute']:
                relation = attribute['object_relation']
                if relation in misp_hash_types:
                    d_section['hashes'][relation] = attribute['value']
                else:
                    try:
                        d_section[peSectionMapping[relation]] = attribute['value']
                    except KeyError:
                        continue
            if 'name' not in d_section:
                d_section['name'] = 'Section {}'.format(sections.index(section))
            extension['sections'].append(WindowsPESection(**d_section))
        if len(sections) != int(extension['number_of_sections']):
            extension['number_of_sections'] = str(len(sections))
        return {"windows-pebinary-ext": extension}

    def parse_pe_extensions_pattern(self, pe_object, sections):
        pattern = ""
        mapping = objectsMapping['file']['pattern']
        pe_mapping = "extensions.'windows-pebinary-ext'"
        for attribute in pe_object['Attribute']:
            try:
                stix_type = "{}.{}".format(pe_mapping, peMapping[attribute['object_relation']])
            except KeyError:
                stix_type = "{}.{}'".format(pe_mapping[:-1], "x_misp_{}_{}".format(attribute['type'], attribute['object_relation'].replace('-', '_')))
            pattern += mapping.format(stix_type, attribute['value'])
        n_section = 0
        for section in sections:
            section_mapping = "{}.sections[{}]".format(pe_mapping, str(n_section))
            for attribute in section['Attribute']:
                relation = attribute['object_relation']
                if relation in misp_hash_types:
                    stix_type = "{}.hashes.'{}'".format(section_mapping, relation)
                    pattern += mapping.format(stix_type, attribute['value'])
                else:
                    try:
                        stix_type = "{}.{}".format(section_mapping, peSectionMapping[relation])
                        pattern += mapping.format(stix_type, attribute['value'])
                    except KeyError:
                        continue
            n_section += 1
        return pattern[:-5]

    def parse_galaxies(self, galaxies, source_id):
        for galaxy in galaxies:
            self.parse_galaxy(galaxy, source_id)

    def parse_galaxy(self, galaxy, source_id):
        galaxy_type = galaxy.get('type')
        galaxy_uuid = galaxy['GalaxyCluster'][0]['collection_uuid']
        try:
            stix_type, to_call = self.galaxies_mapping[galaxy_type]
        except Exception:
            return
        if galaxy_uuid not in self.galaxies:
            to_call(galaxy)
            self.galaxies.append(galaxy_uuid)
        self.relationships['defined'][source_id].append("{}--{}".format(stix_type, galaxy_uuid))

    @staticmethod
    def generate_galaxy_args(galaxy, b_killchain, b_alias, sdo_type):
        cluster = galaxy['GalaxyCluster'][0]
        try:
            cluster_uuid = cluster['collection_uuid']
        except KeyError:
            cluster_uuid = cluster['uuid']
        sdo_id = "{}--{}".format(sdo_type, cluster_uuid)
        description = "{} | {}".format(galaxy['description'], cluster['description'])
        labels = ['misp:name=\"{}\"'.format(galaxy['name'])]
        sdo_args = {'id': sdo_id, 'type': sdo_type, 'name': cluster['value'],
                    'description': description, 'interoperability': True}
        if b_killchain:
            killchain = [{'kill_chain_name': 'misp-category',
                          'phase_name': galaxy['type']}]
            sdo_args['kill_chain_phases'] = killchain
        if cluster['tag_name']:
            labels.append(cluster.get('tag_name'))
        meta = cluster.get('meta')
        if 'synonyms' in meta and b_alias:
            aliases = []
            for a in meta['synonyms']:
                aliases.append(a)
            sdo_args['aliases'] = aliases
        sdo_args['labels'] = labels
        return sdo_args

    def add_attack_pattern(self, galaxy):
        a_p_args = self.generate_galaxy_args(galaxy, True, False, 'attack-pattern')
        a_p_args['created_by_ref'] = self.identity_id
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern)

    def add_attack_pattern_object(self, misp_object, to_ids):
        a_p_id = 'attack-pattern--{}'.format(misp_object['uuid'])
        attributes_dict = {attribute['object_relation']: attribute['value'] for attribute in misp_object['Attribute']}
        a_p_args = {'id': a_p_id, 'type': 'attack-pattern', 'created_by_ref': self.identity_id}
        a_p_args['labels'] = self.create_object_labels(misp_object['name'], misp_object['meta-category'], to_ids)
        for relation, key in attackPatternObjectMapping.items():
            if relation in attributes_dict:
                a_p_args[key] = attributes_dict[relation]
        if 'id' in attributes_dict:
            capec_id = "CAPEC-{}".format(attributes_dict['id'])
            a_p_args['external_references'] = [{'source_name': 'capec', 'external_id': capec_id}]
        attack_pattern = AttackPattern(**a_p_args)
        self.append_object(attack_pattern)

    def add_course_of_action(self, misp_object):
        coa_args= self.generate_galaxy_args(misp_object, False, False, 'course-of-action')
        self.add_coa_stix_object(coa_args)

    def add_course_of_action_from_object(self, misp_object, _):
        coa_id = 'course-of-action--{}'.format(misp_object['uuid'])
        coa_args = {'id': coa_id, 'type': 'course-of-action'}
        for attribute in misp_object['Attribute']:
            self.parse_galaxies(attribute['Galaxy'], coa_id)
            relation = attribute['object_relation']
            if relation == 'name':
                coa_args['name'] = attribute['value']
            elif relation == 'description':
                coa_args['description'] = attribute['value']
        if not 'name' in coa_args:
            return
        self.add_coa_stix_object(coa_args)

    def add_coa_stix_object(self, coa_args):
        coa_args['created_by_ref'] = self.identity_id
        course_of_action = CourseOfAction(**coa_args)
        self.append_object(course_of_action)

    def add_custom(self, attribute):
        custom_object_id = "x-misp-object--{}".format(attribute['uuid'])
        custom_object_type = "x-misp-object-{}".format(attribute['type'].replace('|', '-').replace(' ', '-').lower())
        labels, markings = self.create_labels(attribute)
        custom_object_args = {'id': custom_object_id, 'x_misp_category': attribute['category'], 'labels': labels,
                              'x_misp_timestamp': self.get_datetime_from_timestamp(attribute['timestamp']),
                              'x_misp_value': attribute['value'], 'created_by_ref': self.identity_id}
        if attribute.get('comment'):
            custom_object_args['x_misp_comment'] = attribute['comment']
        if markings:
            markings = self.handle_tags(markings)
            custom_object_args['object_marking_refs'] = markings
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                          ('x_misp_timestamp', properties.StringProperty(required=True)),
                                          ('labels', properties.ListProperty(labels, required=True)),
                                          ('x_misp_value', properties.StringProperty(required=True)),
                                          ('created_by_ref', properties.StringProperty(required=True)),
                                          ('object_marking_refs', properties.ListProperty(markings)),
                                          ('x_misp_comment', properties.StringProperty()),
                                          ('x_misp_category', properties.StringProperty())
                                         ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object)

    def add_identity(self, attribute):
        identity_id = "identity--{}".format(attribute['uuid'])
        name = attribute['value']
        labels, markings = self.create_labels(attribute)
        identity_args = {'id': identity_id,  'type': identity, 'name': name, 'labels': labels,
                          'identity_class': 'individual', 'created_by_ref': self.identity_id,
                          'interoperability': True}
        if attribute.get('comment'):
            identity_args['description'] = attribute['comment']
        if markings:
            identity_args['object_marking_refs'] = self.handle_tags(markings)
        identity = Identity(**identity_args)
        self.append_object(identity)

    def add_indicator(self, attribute):
        attribute_type = attribute['type']
        indicator_id = "indicator--{}".format(attribute['uuid'])
        self.parse_galaxies(attribute['Galaxy'], indicator_id)
        category = attribute['category']
        killchain = self.create_killchain(category)
        labels, markings = self.create_labels(attribute)
        attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment'])
        pattern = mispTypesMapping[attribute_type]['pattern'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_pattern(attribute_type, attribute_value)
        indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, 'kill_chain_phases': killchain,
                           'valid_from': self.misp_event['date'], 'created_by_ref': self.identity_id,
                           'pattern': pattern, 'interoperability': True}
        if hasattr(attribute, 'Sighting'):
            for sighting in attribute['Sighting']:
                if sighting['Organisation']['name'] == self.misp_event['Orgc']['name'] and sighting['type'] == "2":
                    indicator_args['valid_until'] = self.get_datetime_from_timestamp(sighting['date_sighting'])
                    break
        if attribute.get('comment'):
            indicator_args['description'] = attribute['comment']
        if markings:
            indicator_args['object_marking_refs'] = self.handle_tags(markings)
        indicator = Indicator(**indicator_args)
        self.append_object(indicator)

    def add_intrusion_set(self, galaxy):
        i_s_args = self.generate_galaxy_args(galaxy, False, True, 'intrusion-set')
        i_s_args['created_by_ref'] = self.identity_id
        intrusion_set = IntrusionSet(**i_s_args)
        self.append_object(intrusion_set)

    def add_malware(self, galaxy):
        malware_args= self.generate_galaxy_args(galaxy, True, False, 'malware')
        malware_args['created_by_ref'] = self.identity_id
        malware = Malware(**malware_args)
        self.append_object(malware)

    def add_observed_data(self, attribute):
        attribute_type = attribute['type']
        observed_data_id = "observed-data--{}".format(attribute['uuid'])
        self.parse_galaxies(attribute['Galaxy'], observed_data_id)
        timestamp = self.get_datetime_from_timestamp(attribute['timestamp'])
        labels, markings = self.create_labels(attribute)
        attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment'])
        observable = mispTypesMapping[attribute_type]['observable'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_observable(attribute_type, attribute_value)
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'number_observed': 1,
                              'first_observed': timestamp, 'last_observed': timestamp, 'labels': labels,
                              'created_by_ref': self.identity_id, 'objects': observable, 'interoperability': True}
        if markings:
            observed_data_args['object_marking_refs'] = self.handle_tags(markings)
        observed_data = ObservedData(**observed_data_args)
        self.append_object(observed_data)

    def add_threat_actor(self, galaxy):
        t_a_args = self.generate_galaxy_args(galaxy, False, True, 'threat-actor')
        t_a_args['created_by_ref'] = self.identity_id
        threat_actor = ThreatActor(**t_a_args)
        self.append_object(threat_actor)

    def add_tool(self, galaxy):
        tool_args = self.generate_galaxy_args(galaxy, True, False, 'tool')
        tool_args['created_by_ref'] = self.identity_id
        tool = Tool(**tool_args)
        self.append_object(tool)

    def add_vulnerability(self, attribute):
        vulnerability_id = "vulnerability--{}".format(attribute['uuid'])
        name = attribute['value']
        vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](name)]
        labels, markings = self.create_labels(attribute)
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'external_references': vulnerability_data,
                              'created_by_ref': self.identity_id, 'labels': labels,
                              'interoperability': True}
        if markings:
            vulnerability_args['object_marking_refs'] = self.handle_tags(markings)
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def add_vulnerability_from_galaxy(self, attribute):
        vulnerability_id = "vulnerability--{}".format(attribute['uuid'])
        cluster = attribute['GalaxyCluster'][0]
        name = cluster['value']
        if cluster['meta'] and cluster['meta']['aliases']:
            vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](alias) for alias in cluster['meta']['aliases']]
        else:
            vulnerability_data = [mispTypesMapping['vulnerability']['vulnerability_args'](name)]
        labels = ['misp:type=\"{}\"'.format(attribute.get('type'))]
        if cluster['tag_name']:
            labels.append(cluster['tag_name'])
        description = "{} | {}".format(attribute.get('description'), cluster.get('description'))
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'external_references': vulnerability_data,
                              'created_by_ref': self.identity_id, 'labels': labels,
                              'description': description, 'interoperability': True}
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def add_object_custom(self, misp_object, to_ids):
        custom_object_id = 'x-misp-object--{}'.format(misp_object['uuid'])
        name = misp_object['name']
        custom_object_type = 'x-misp-object-{}'.format(name)
        category = misp_object.get('meta-category')
        labels = self.create_object_labels(name, category, to_ids)
        values = self.fetch_custom_values(misp_object['Attribute'], custom_object_id)
        custom_object_args = {'id': custom_object_id, 'x_misp_values': values, 'labels': labels,
                              'x_misp_category': category, 'created_by_ref': self.identity_id,
                              'x_misp_timestamp': self.get_datetime_from_timestamp(misp_object['timestamp'])}
        if hasattr(misp_object, 'comment') and misp_object.get('comment'):
            custom_object_args['x_misp_comment'] = misp_object['comment']
        @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
                                           ('x_misp_timestamp', properties.StringProperty(required=True)),
                                           ('labels', properties.ListProperty(labels, required=True)),
                                           ('x_misp_values', properties.DictionaryProperty(required=True)),
                                           ('created_by_ref', properties.StringProperty(required=True)),
                                           ('x_misp_comment', properties.StringProperty()),
                                           ('x_misp_category', properties.StringProperty())
                                          ])
        class Custom(object):
            def __init__(self, **kwargs):
                return
        custom_object = Custom(**custom_object_args)
        self.append_object(custom_object)

    def add_object_indicator(self, misp_object, pattern_arg=None):
        indicator_id = 'indicator--{}'.format(misp_object['uuid'])
        if pattern_arg:
            name = 'WindowsPEBinaryFile'
            pattern = pattern_arg
        else:
            name = misp_object['name']
            pattern = self.objects_mapping[name]['pattern'](misp_object['Attribute'], indicator_id)
        category = misp_object.get('meta-category')
        killchain = self.create_killchain(category)
        labels = self.create_object_labels(name, category, True)
        indicator_args = {'id': indicator_id, 'valid_from': self.misp_event['date'],
                          'type': 'indicator', 'labels': labels, 'pattern': pattern,
                          'description': misp_object['description'], 'allow_custom': True,
                          'kill_chain_phases': killchain, 'interoperability': True,
                          'created_by_ref': self.identity_id}
        indicator = Indicator(**indicator_args)
        self.append_object(indicator)

    def add_object_observable(self, misp_object, observable_arg=None):
        observed_data_id = 'observed-data--{}'.format(misp_object['uuid'])
        if observable_arg:
            name = 'WindowsPEBinaryFile'
            observable_objects = observable_arg
        else:
            name = misp_object['name']
            observable_objects = self.objects_mapping[name]['observable'](misp_object['Attribute'], observed_data_id)
        category = misp_object.get('meta-category')
        labels = self.create_object_labels(name, category, False)
        timestamp = self.get_datetime_from_timestamp(misp_object['timestamp'])
        observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'labels': labels,
                              'number_observed': 1, 'objects': observable_objects, 'allow_custom': True,
                              'first_observed': timestamp, 'last_observed': timestamp,
                              'created_by_ref': self.identity_id, 'interoperability': True}
        try:
            observed_data = ObservedData(**observed_data_args)
        except exceptions.InvalidValueError:
            observed_data = self.fix_enumeration_issues(name, observed_data_args)
        self.append_object(observed_data)

    @staticmethod
    def fix_enumeration_issues(name, args):
        enumeration_fails = {}
        if name == 'network-socket':
            ns_args = deepcopy(args)
            observable_object = ns_args['objects']
            n = sorted(observable_object.keys())[-1]
            current_dict = observable_object[n]['extensions']['socket-ext']
            for field in ('address_family', 'protocol_family'):
                enumeration_fails[field] = current_dict.pop(field)
                try:
                    return ObservedData(**ns_args)
                except (exceptions.InvalidValueError, exceptions.MissingPropertiesError):
                    current_dict[field] = enumeration_fails[field]
            for field in enumeration_fails:
                current_dict.pop(field)
            return ObservedData(**ns_args)
        return ObservedData(**args)

    def add_object_vulnerability(self, misp_object, to_ids):
        vulnerability_id = 'vulnerability--{}'.format(misp_object['uuid'])
        name = self.fetch_vulnerability_name(misp_object['Attribute'])
        labels = self.create_object_labels(name, misp_object.get('meta-category'), to_ids)
        vulnerability_args = {'id': vulnerability_id, 'type': 'vulnerability',
                              'name': name, 'created_by_ref': self.identity_id,
                              'labels': labels, 'interoperability': True}
        vulnerability = Vulnerability(**vulnerability_args)
        self.append_object(vulnerability)

    def append_object(self, stix_object, id_mapping=True):
        self.SDOs.append(stix_object)
        self.object_refs.append(stix_object.id)
        if id_mapping:
            object_type, uuid = stix_object.id.split('--')
            self.ids[uuid] = object_type

    @staticmethod
    def create_killchain(category):
        return [{'kill_chain_name': 'misp-category', 'phase_name': category}]

    @staticmethod
    def create_labels(attribute):
        labels = ['misp:type="{}"'.format(attribute['type']),
                  'misp:category="{}"'.format(attribute['category']),
{'fraction_non_alphanumeric': 0.12020903564707634, 'fraction_numerical': 0.002450482650312064, 'mean_word_length': 2.5771223788076902, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1196700', 'n_tokens_mistral': 20987, 'n_tokens_neox': 20317, 'n_words': 3570}