<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import json
import os
import time
import io
import re
import stix2
import stix2misp_mapping
from collections import defaultdict
from copy import deepcopy
from pathlib import Path
from pymisp import MISPEvent, MISPObject, MISPAttribute, PyMISPInvalidFormat


class StixParser():
    _misp_dir = Path(os.path.realpath(__file__)).parents[4]
    _misp_objects_path = _misp_dir / 'app' / 'files' / 'misp-objects' / 'objects'
    _pymisp_dir = _misp_dir / 'PyMISP'
    with open(_pymisp_dir / 'pymisp' / 'data' / 'describeTypes.json', 'r') as f:
        _misp_types = json.loads(f.read())['result'].get('types')
    _galaxy_types = ('intrusion-set', 'malware', 'threat-actor', 'tool')
    _stix2misp_mapping = {'marking-definition': '_load_marking',
                             'relationship': '_load_relationship',
                             'report': '_load_report',
                             'indicator': '_parse_indicator',
                             'observed-data': '_parse_observable',
                             'identity': '_load_identity'}
    _stix2misp_mapping.update({special_type: '_parse_undefined' for special_type in ('attack-pattern', 'course-of-action', 'vulnerability')})
    _stix2misp_mapping.update({galaxy_type: '_load_galaxy' for galaxy_type in _galaxy_types})
    _special_mapping = {'attack-pattern': 'parse_attack_pattern',
                        'course-of-action': 'parse_course_of_action',
                        'vulnerability': 'parse_vulnerability'}
    _timeline_mapping = {'indicator': ('valid_from', 'valid_until'),
                         'observed-data': ('first_observed', 'last_observed')}

    def __init__(self):
        super().__init__()
        self.misp_event = MISPEvent()
        self.relationship = defaultdict(list)
        self.tags = set()
        self.galaxy = {}
        self.marking_definition = {}

    def handler(self, event, filename, args):
        self.filename = filename
        self.stix_version = 'STIX {}'.format(event.get('spec_version'))
        try:
            event_distribution = args[0]
            if not isinstance(event_distribution, int):
                event_distribution = int(event_distribution) if event_distribution.isdigit() else 0
        except IndexError:
            event_distribution = 0
        try:
            attribute_distribution = args[1]
            if attribute_distribution == 'event':
                attribute_distribution = 5
            if not isinstance(attribute_distribution, int):
                attribute_distribution = int(attribute_distribution) if attribute_distribution.isdigit() else 5
        except IndexError:
            attribute_distribution = 5
        self._synonyms_to_tag_names = args[2] if len(args) > 2 else '/var/www/MISP/app/files/scripts/synonymsToTagNames.json'
        self.parse_event(event.objects)

    def _load_galaxy(self, galaxy):
        self.galaxy[galaxy['id'].split('--')[1]] = {'tag_names': self.parse_galaxy(galaxy), 'used': False}

    def _load_identity(self, identity):
        try:
            self.identity[identity['id'].split('--')[1]] = identity['name']
        except AttributeError:
            self.identity = {identity['id'].split('--')[1]: identity['name']}

    def _load_marking(self, marking):
        tag = self.parse_marking(marking)
        self.marking_definition[marking['id'].split('--')[1]] = {'object': tag, 'used': False}

    def _load_relationship(self, relationship):
        target_uuid = relationship.target_ref.split('--')[1]
        reference = (target_uuid, relationship.relationship_type)
        source_uuid = relationship.source_ref.split('--')[1]
        self.relationship[source_uuid].append(reference)

    def _load_report(self, report):
        try:
            self.report[report['id'].split('--')[1]] = report
        except AttributeError:
            self.report = {report['id'].split('--')[1]: report}

    def _load_synonyms_to_tag_names(self):
        with open(self._synonyms_to_tag_names, 'rt', encoding='utf-8') as f:
            synonyms_to_tag_names = json.loads(f.read())
        self._synonyms_to_tag_names = synonyms_to_tag_names

    def save_file(self):
        event = self.misp_event.to_json()
        event = json.loads(event)
        # print(json.dumps(event, indent=4))

    ################################################################################
    ##                 PARSING FUNCTIONS USED BY BOTH SUBCLASSES.                 ##
    ################################################################################

    def create_attribute_with_tag(self, attribute_dict, marking_refs):
        attribute = MISPAttribute()
        attribute.from_dict(**attribute_dict)
        try:
            self.marking_refs[attribute.uuid] = (marking.split('--')[1] for marking in marking_refs)
        except AttributeError:
            self.marking_refs = {attribute.uuid: (marking.split('--')[1] for marking in marking_refs)}
        return attribute

    def parse_galaxies(self):
        for galaxy in self.galaxy.values():
            if not galaxy['used']:
                for tag_name in galaxy['tag_names']:
                    self.tags.add(tag_name)

    def parse_relationships(self):
        attribute_uuids = (attribute.uuid for attribute in self.misp_event.attributes)
        object_uuids = (object.uuid for object in self.misp_event.objects)
        for source, references in self.relationship.items():
            if source in self.galaxy:
                continue
            if source in object_uuids:
                source_object = self.misp_event.get_object_by_uuid(source)
                for reference in references:
                    target, reference = reference
                    if target in attribute_uuids or target in object_uuids:
                        source_object.add_reference(target, reference)
            elif source in attribute_uuids:
                for attribute in self.misp_event.atributes:
                    if attribute.uuid == source:
                        for reference in references:
                            target, reference = reference
                            if target in self.galaxy:
                                for tag_name in self.galaxy[target]['tag_names']:
                                    attribute.add_tag(tag_name)
                                self.galaxy[target]['used'] = True
                        break

    def parse_report(self):
        event_infos = set()
        for report in self.report.values():
            if hasattr(report, 'name') and report.name:
                event_infos.add(report.name)
            if hasattr(report, 'labels') and report.labels:
                for label in report.labels:
                    self.tags.add(label)
            if hasattr(report, 'object_marking_refs') and report.object_marking_refs:
                for marking_ref in report.object_marking_refs:
                    marking_ref = marking_ref.split('--')[1]
                    try:
                        self.tags.add(self.marking_definition[marking_ref]['object'])
                        self.marking_definition[marking_ref]['used'] = True
                    except KeyError:
                        continue
        if len(event_infos) == 1:
            self.misp_event.info = event_infos.pop()
        else:
            self.misp_event.info = f'Imported with MISP import script for {self.stix_version}'

    ################################################################################
    ##                             UTILITY FUNCTIONS.                             ##
    ################################################################################

    def _get_tag_names_from_synonym(self, name):
        try:
            return self._synonyms_to_tag_names[name]
        except TypeError:
            self._load_synonyms_to_tag_names()
            return self._synonyms_to_tag_names[name]

    @staticmethod
    def getTimestampfromDate(date):
        try:
            return int(date.timestamp())
        except AttributeError:
            return int(time.mktime(time.strptime(date.split('+')[0], "%Y-%m-%d %H:%M:%S")))

    @staticmethod
    def _handle_data(data):
        return io.BytesIO(data.encode())

    @staticmethod
    def parse_marking(marking):<commit_msg>wip: [stix2 import] Importing reports external references as links<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import json
import os
import time
import io
import re
import stix2
import stix2misp_mapping
from collections import defaultdict
from copy import deepcopy
from pathlib import Path
from pymisp import MISPEvent, MISPObject, MISPAttribute, PyMISPInvalidFormat


class StixParser():
    _misp_dir = Path(os.path.realpath(__file__)).parents[4]
    _misp_objects_path = _misp_dir / 'app' / 'files' / 'misp-objects' / 'objects'
    _pymisp_dir = _misp_dir / 'PyMISP'
    with open(_pymisp_dir / 'pymisp' / 'data' / 'describeTypes.json', 'r') as f:
        _misp_types = json.loads(f.read())['result'].get('types')
    _galaxy_types = ('intrusion-set', 'malware', 'threat-actor', 'tool')
    _stix2misp_mapping = {'marking-definition': '_load_marking',
                             'relationship': '_load_relationship',
                             'report': '_load_report',
                             'indicator': '_parse_indicator',
                             'observed-data': '_parse_observable',
                             'identity': '_load_identity'}
    _stix2misp_mapping.update({special_type: '_parse_undefined' for special_type in ('attack-pattern', 'course-of-action', 'vulnerability')})
    _stix2misp_mapping.update({galaxy_type: '_load_galaxy' for galaxy_type in _galaxy_types})
    _special_mapping = {'attack-pattern': 'parse_attack_pattern',
                        'course-of-action': 'parse_course_of_action',
                        'vulnerability': 'parse_vulnerability'}
    _timeline_mapping = {'indicator': ('valid_from', 'valid_until'),
                         'observed-data': ('first_observed', 'last_observed')}

    def __init__(self):
        super().__init__()
        self.misp_event = MISPEvent()
        self.relationship = defaultdict(list)
        self.tags = set()
        self.galaxy = {}
        self.marking_definition = {}

    def handler(self, event, filename, args):
        self.filename = filename
        self.stix_version = 'STIX {}'.format(event.get('spec_version'))
        try:
            event_distribution = args[0]
            if not isinstance(event_distribution, int):
                event_distribution = int(event_distribution) if event_distribution.isdigit() else 0
        except IndexError:
            event_distribution = 0
        try:
            attribute_distribution = args[1]
            if attribute_distribution == 'event':
                attribute_distribution = 5
            if not isinstance(attribute_distribution, int):
                attribute_distribution = int(attribute_distribution) if attribute_distribution.isdigit() else 5
        except IndexError:
            attribute_distribution = 5
        self._synonyms_to_tag_names = args[2] if len(args) > 2 else '/var/www/MISP/app/files/scripts/synonymsToTagNames.json'
        self.parse_event(event.objects)

    def _load_galaxy(self, galaxy):
        self.galaxy[galaxy['id'].split('--')[1]] = {'tag_names': self.parse_galaxy(galaxy), 'used': False}

    def _load_identity(self, identity):
        try:
            self.identity[identity['id'].split('--')[1]] = identity['name']
        except AttributeError:
            self.identity = {identity['id'].split('--')[1]: identity['name']}

    def _load_marking(self, marking):
        tag = self.parse_marking(marking)
        self.marking_definition[marking['id'].split('--')[1]] = {'object': tag, 'used': False}

    def _load_relationship(self, relationship):
        target_uuid = relationship.target_ref.split('--')[1]
        reference = (target_uuid, relationship.relationship_type)
        source_uuid = relationship.source_ref.split('--')[1]
        self.relationship[source_uuid].append(reference)

    def _load_report(self, report):
        try:
            self.report[report['id'].split('--')[1]] = report
        except AttributeError:
            self.report = {report['id'].split('--')[1]: report}

    def _load_synonyms_to_tag_names(self):
        with open(self._synonyms_to_tag_names, 'rt', encoding='utf-8') as f:
            synonyms_to_tag_names = json.loads(f.read())
        self._synonyms_to_tag_names = synonyms_to_tag_names

    def save_file(self):
        event = self.misp_event.to_json()
        event = json.loads(event)
        # print(json.dumps(event, indent=4))

    ################################################################################
    ##                 PARSING FUNCTIONS USED BY BOTH SUBCLASSES.                 ##
    ################################################################################

    def create_attribute_with_tag(self, attribute_dict, marking_refs):
        attribute = MISPAttribute()
        attribute.from_dict(**attribute_dict)
        try:
            self.marking_refs[attribute.uuid] = (marking.split('--')[1] for marking in marking_refs)
        except AttributeError:
            self.marking_refs = {attribute.uuid: (marking.split('--')[1] for marking in marking_refs)}
        return attribute

    def parse_galaxies(self):
        for galaxy in self.galaxy.values():
            if not galaxy['used']:
                for tag_name in galaxy['tag_names']:
                    self.tags.add(tag_name)

    def parse_relationships(self):
        attribute_uuids = (attribute.uuid for attribute in self.misp_event.attributes)
        object_uuids = (object.uuid for object in self.misp_event.objects)
        for source, references in self.relationship.items():
            if source in self.galaxy:
                continue
            if source in object_uuids:
                source_object = self.misp_event.get_object_by_uuid(source)
                for reference in references:
                    target, reference = reference
                    if target in attribute_uuids or target in object_uuids:
                        source_object.add_reference(target, reference)
            elif source in attribute_uuids:
                for attribute in self.misp_event.atributes:
                    if attribute.uuid == source:
                        for reference in references:
                            target, reference = reference
                            if target in self.galaxy:
                                for tag_name in self.galaxy[target]['tag_names']:
                                    attribute.add_tag(tag_name)
                                self.galaxy[target]['used'] = True
                        break

    def parse_report(self):
        event_infos = set()
        for report in self.report.values():
            if hasattr(report, 'name') and report.name:
                event_infos.add(report.name)
            if hasattr(report, 'labels') and report.labels:
                for label in report.labels:
                    self.tags.add(label)
            if hasattr(report, 'object_marking_refs') and report.object_marking_refs:
                for marking_ref in report.object_marking_refs:
                    marking_ref = marking_ref.split('--')[1]
                    try:
                        self.tags.add(self.marking_definition[marking_ref]['object'])
                        self.marking_definition[marking_ref]['used'] = True
                    except KeyError:
                        continue
            if hasattr(report, 'external_references'):
                for reference in report.external_references:
                    self.misp_event.add_attribute(**{'type': 'link', 'value': reference['url']})

        if len(event_infos) == 1:
            self.misp_event.info = event_infos.pop()
        else:
            self.misp_event.info = f'Imported with MISP import script for {self.stix_version}'

    ################################################################################
    ##                             UTILITY FUNCTIONS.                             ##
    ################################################################################

    def _get_tag_names_from_synonym(self, name):
        try:
            return self._synonyms_to_tag_names[name]
        except TypeError:
            self._load_synonyms_to_tag_names()
            return self._synonyms_to_tag_names[name]

    @staticmethod
    def getTimestampfromDate(date):
        try:
            return int(date.timestamp())
        except AttributeError:
            return int(time.mktime(time.strptime(date.split('+')[0], "%Y-%m-%d %H:%M:%S")))

    @staticmethod
    def _handle_data(data):
        return io.BytesIO(data.encode())

    @staticmethod
    def parse_marking(marking):
{'fraction_non_alphanumeric': 0.1350844277673546, 'fraction_numerical': 0.005306888233717502, 'mean_word_length': 2.3711601011926273, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5552775', 'n_tokens_mistral': 5319, 'n_tokens_neox': 5020, 'n_words': 1168}