<commit_before>#!/usr/bin/env python3
#    Copyright (C) 2017 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime, re, base64
import pymisp
from stix2 import *
from misp2stix2_dictionaries import *
from copy import deepcopy

namespace = ['https://github.com/MISP/MISP', 'MISP']

not_implemented_attributes = ['yara', 'pattern-in-traffic', 'pattern-in-memory']

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability', 'attachment']

def saveFile(args, package):
    filename = args[1] + '.out'
    with open(filename, 'w') as f:
        f.write(json.dumps(package, cls=base.STIXJSONEncoder))

# converts timestamp to the format used by STIX
def getDateFromTimestamp(timestamp):
    return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(timestamp))

def setIdentity(event, SDOs):
    org = event.Orgc
    identity_id = 'identity--{}'.format(org['uuid'])
    identity = Identity(type="identity", id=identity_id,
                        name=org["name"], identity_class="organization")
    SDOs.append(identity)
    return identity_id

def readAttributes(event, identity, object_refs, external_refs):
    attributes = []
    descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
    descFile = open(descFilename, 'r')
    types = json.loads(descFile.read())['result']
    for attribute in event.attributes:
        attr_type = attribute.type
        if attr_type in non_indicator_attributes:
            if attr_type == "link":
                handleLink(attribute, external_refs)
            elif attr_type in ('text', 'comment', 'other') or attr_type not in mispTypesMapping:
                addCustomObject(object_refs, attributes, attribute, identity)
            else:
                handleNonIndicatorAttribute(object_refs, attributes, attribute, identity)
        else:
            mapping = types['category_type_mappings']
            if attr_type in mapping['Person']:
                addIdentity(object_refs, attributes, attribute, identity, 'individual')
            elif attr_type in mispTypesMapping:
                if attribute.to_ids:
                    handleIndicatorAttribute(object_refs, attributes, attribute, identity)
                else:
                    addObservedData(object_refs, attributes, attribute, identity)
            else:
                addCustomObject(object_refs, attributes, attribute, identity)
    galaxy = False
    try:
        if event.Galaxy:
            galaxy = True
    except:
        pass
    if galaxy:<commit_msg>fix: fixed an abusive use of Identity SDO<commit_after>#!/usr/bin/env python3
#    Copyright (C) 2017 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, datetime, re, base64
import pymisp
from stix2 import *
from misp2stix2_dictionaries import *
from copy import deepcopy

namespace = ['https://github.com/MISP/MISP', 'MISP']

not_implemented_attributes = ['yara', 'pattern-in-traffic', 'pattern-in-memory']

non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', 'target-email',
                            'target-machine', 'target-org', 'target-location', 'target-external',
                            'vulnerability', 'attachment']

def saveFile(args, package):
    filename = args[1] + '.out'
    with open(filename, 'w') as f:
        f.write(json.dumps(package, cls=base.STIXJSONEncoder))

# converts timestamp to the format used by STIX
def getDateFromTimestamp(timestamp):
    return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(timestamp))

def setIdentity(event, SDOs):
    org = event.Orgc
    identity_id = 'identity--{}'.format(org['uuid'])
    identity = Identity(type="identity", id=identity_id,
                        name=org["name"], identity_class="organization")
    SDOs.append(identity)
    return identity_id

def selectSDO(object_refs, attributes, attribute, identity):
    if attribute.to_ids:
        handleIndicatorAttribute(object_refs, attributes, attribute, identity)
    else:
        addObservedData(object_refs, attributes, attribute, identity)

def readAttributes(event, identity, object_refs, external_refs):
    attributes = []
    descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
    descFile = open(descFilename, 'r')
    types = json.loads(descFile.read())['result']
    for attribute in event.attributes:
        attr_type = attribute.type
        if attr_type in non_indicator_attributes:
            if attr_type == "link":
                handleLink(attribute, external_refs)
            elif attr_type in ('text', 'comment', 'other') or attr_type not in mispTypesMapping:
                addCustomObject(object_refs, attributes, attribute, identity)
            else:
                handleNonIndicatorAttribute(object_refs, attributes, attribute, identity)
        else:
            mapping = types['category_type_mappings']
            if attr_type in mapping['Person']:
                if attribute.category == 'Person':
                    addIdentity(object_refs, attributes, attribute, identity, 'individual')
                else:
                    addCustomObject(object_refs, attributes, attribute, identity)
            elif attr_type in mispTypesMapping:
                selectSDO(object_refs, attributes, attribute, identity)
            else:
                addCustomObject(object_refs, attributes, attribute, identity)
    galaxy = False
    try:
        if event.Galaxy:
            galaxy = True
    except:
        pass
    if galaxy:
{'fraction_non_alphanumeric': 0.09733649421280156, 'fraction_numerical': 0.0064147259796402175, 'mean_word_length': 2.9977703455964324, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '350350', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1926, 'n_words': 609}