<commit_before># -*- coding: utf-8 -*-
from pymisp import MISPEvent, MISPObject
from pymisp import __path__ as pymisp_path
import csv
import io
import json
import os
import base64

misperrors = {'error': 'Error'}
moduleinfo = {'version': '0.2', 'author': '<NAME>',
              'description': 'Import Attributes from a csv file.',
              'module-type': ['import']}
moduleconfig = []
userConfig = {
    'header': {
        'type': 'String',
        'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
    'has_header': {
        'type': 'Boolean',
        'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'},
    'special_delimiter': {
        'type': 'String',
        'message': 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t" for tabs, etc).'
    }
}
mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'}

misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date',
                            'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category']
misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution',
                                  'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag']
misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields


class CsvParser():
    def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes):
        self.misp_event = MISPEvent()
        self.header = header
        self.has_header = has_header
        self.delimiter = delimiter
        self.data = data
        self.from_misp = from_misp
        self.MISPtypes = MISPtypes
        self.fields_number = len(self.header)
        self.__score_mapping = {0: self.__create_standard_misp,
                                1: self.__create_attribute_with_ids,
                                2: self.__create_attribute_with_tags,
                                3: self.__create_attribute_with_ids_and_tags}

    def parse_csv(self):
        if self.from_misp:
            if self.header == misp_standard_csv_header:
                self.__parse_misp_csv()
            else:
                attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:10]
                object_fields = ['object_id'] + misp_standard_csv_header[10:]
                attribute_indexes = []
                object_indexes = []
                for i in range(len(self.header)):
                    if self.header[i] in attribute_fields:
                        attribute_indexes.append(i)
                    elif self.header[i] in object_fields:
                        object_indexes.append(i)
                if object_indexes:
                    if not any(field in self.header for field in ('object_uuid', 'object_id')) or 'object_name' not in self.header:
                        for line in self.data:
                            for index in object_indexes:
                                if line[index].strip():
                                    return {'error': 'It is not possible to import MISP objects from your csv file if you do not specify any object identifier and object name to separate each object from each other.'}
                    if 'object_relation' not in self.header:
                        return {'error': 'In order to import MISP objects, an object relation for each attribute contained in an object is required.'}
                self.__build_misp_event(attribute_indexes, object_indexes)
        else:
            attribute_fields = attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:9]
            attribute_indexes = []
            types_indexes = []
            for i in range(len(self.header)):
                if self.header[i] in attribute_fields:
                    attribute_indexes.append(i)
                elif self.header[i] in self.MISPtypes:
                    types_indexes.append(i)
            self.__parse_external_csv(attribute_indexes, types_indexes)
        self.__finalize_results()
        return {'success': 1}

    ################################################################################
    ####                   Parsing csv data with MISP fields,                   ####
    ####                          but a custom header                           ####
    ################################################################################

    def __build_misp_event(self, attribute_indexes, object_indexes):
        score = self.__get_score()
        if object_indexes:
            objects = {}
            id_name = 'object_id' if 'object_id' in self.header else 'object_uuid'
            object_id_index = self.header.index(id_name)
            name_index = self.header.index('object_name')
            for line in self.data:
                attribute = self.__score_mapping[score](line, attribute_indexes)
                object_id = line[object_id_index]
                if object_id:
                    if object_id not in objects:
                        misp_object = MISPObject(line[name_index])
                        if id_name == 'object_uuid':
                            misp_object.uuid = object_id
                        objects[object_id] = misp_object
                    objects[object_id].add_attribute(**attribute)
                else:
                    self.event.add_attribute(**attribute)
            for misp_object in objects.values():
                self.misp_event.add_object(**misp_object)
        else:
            for line in self.data:
                attribute = self.__score_mapping[score](line, attribute_indexes)
                self.misp_event.add_attribute(**attribute)

    ################################################################################
    ####            Parsing csv data containing fields that are not             ####
    ####               MISP attributes or objects standard fields               ####
    ####                 (but should be MISP attribute types!!)                 ####
    ################################################################################

    def __parse_external_csv(self, attribute_indexes, types_indexes):
        score = self.__get_score()
        if attribute_indexes:
            for line in self.data:
                base_attribute = self.__score_mapping[score](line, attribute_indexes)
                for index in types_indexes:
                    attribute = {'type': self.header[index], 'value': line[index]}
                    attribute.update(base_attribute)
                    self.misp_event.add_attribute(**attribute)
        else:
            for line in self.data:
                for index in types_indexes:
                    self.misp_event.add_attribute(**{'type': self.header[index], 'value': line[index]})

    ################################################################################
    ####                    Parsing standard MISP csv format                    ####
    ################################################################################

    def __parse_misp_csv(self):
        objects = {}
        attribute_fields = self.header[:1] + self.header[2:8]
        for line in self.data:
            a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number]
            attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))}
            attribute['to_ids'] = True if to_ids == '1' else False
            if tag:
                attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')]
            if relation:<commit_msg>fix: Fixed unassigned variable name<commit_after># -*- coding: utf-8 -*-
from pymisp import MISPEvent, MISPObject
from pymisp import __path__ as pymisp_path
import csv
import io
import json
import os
import base64

misperrors = {'error': 'Error'}
moduleinfo = {'version': '0.2', 'author': '<NAME>',
              'description': 'Import Attributes from a csv file.',
              'module-type': ['import']}
moduleconfig = []
userConfig = {
    'header': {
        'type': 'String',
        'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
    'has_header': {
        'type': 'Boolean',
        'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'},
    'special_delimiter': {
        'type': 'String',
        'message': 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t" for tabs, etc).'
    }
}
mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'}

misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date',
                            'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category']
misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution',
                                  'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag']
misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields


class CsvParser():
    def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes):
        self.misp_event = MISPEvent()
        self.header = header
        self.has_header = has_header
        self.delimiter = delimiter
        self.data = data
        self.from_misp = from_misp
        self.MISPtypes = MISPtypes
        self.fields_number = len(self.header)
        self.__score_mapping = {0: self.__create_standard_misp,
                                1: self.__create_attribute_with_ids,
                                2: self.__create_attribute_with_tags,
                                3: self.__create_attribute_with_ids_and_tags}

    def parse_csv(self):
        if self.from_misp:
            if self.header == misp_standard_csv_header:
                self.__parse_misp_csv()
            else:
                attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:10]
                object_fields = ['object_id'] + misp_standard_csv_header[10:]
                attribute_indexes = []
                object_indexes = []
                for i in range(len(self.header)):
                    if self.header[i] in attribute_fields:
                        attribute_indexes.append(i)
                    elif self.header[i] in object_fields:
                        object_indexes.append(i)
                if object_indexes:
                    if not any(field in self.header for field in ('object_uuid', 'object_id')) or 'object_name' not in self.header:
                        for line in self.data:
                            for index in object_indexes:
                                if line[index].strip():
                                    return {'error': 'It is not possible to import MISP objects from your csv file if you do not specify any object identifier and object name to separate each object from each other.'}
                    if 'object_relation' not in self.header:
                        return {'error': 'In order to import MISP objects, an object relation for each attribute contained in an object is required.'}
                self.__build_misp_event(attribute_indexes, object_indexes)
        else:
            attribute_fields = attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:9]
            attribute_indexes = []
            types_indexes = []
            for i in range(len(self.header)):
                if self.header[i] in attribute_fields:
                    attribute_indexes.append(i)
                elif self.header[i] in self.MISPtypes:
                    types_indexes.append(i)
            self.__parse_external_csv(attribute_indexes, types_indexes)
        self.__finalize_results()
        return {'success': 1}

    ################################################################################
    ####                   Parsing csv data with MISP fields,                   ####
    ####                          but a custom header                           ####
    ################################################################################

    def __build_misp_event(self, attribute_indexes, object_indexes):
        score = self.__get_score()
        if object_indexes:
            objects = {}
            id_name = 'object_id' if 'object_id' in self.header else 'object_uuid'
            object_id_index = self.header.index(id_name)
            name_index = self.header.index('object_name')
            for line in self.data:
                attribute = self.__score_mapping[score](line, attribute_indexes)
                object_id = line[object_id_index]
                if object_id:
                    if object_id not in objects:
                        misp_object = MISPObject(line[name_index])
                        if id_name == 'object_uuid':
                            misp_object.uuid = object_id
                        objects[object_id] = misp_object
                    objects[object_id].add_attribute(**attribute)
                else:
                    self.event.add_attribute(**attribute)
            for misp_object in objects.values():
                self.misp_event.add_object(**misp_object)
        else:
            for line in self.data:
                attribute = self.__score_mapping[score](line, attribute_indexes)
                self.misp_event.add_attribute(**attribute)

    ################################################################################
    ####            Parsing csv data containing fields that are not             ####
    ####               MISP attributes or objects standard fields               ####
    ####                 (but should be MISP attribute types!!)                 ####
    ################################################################################

    def __parse_external_csv(self, attribute_indexes, types_indexes):
        score = self.__get_score()
        if attribute_indexes:
            for line in self.data:
                base_attribute = self.__score_mapping[score](line, attribute_indexes)
                for index in types_indexes:
                    attribute = {'type': self.header[index], 'value': line[index]}
                    attribute.update(base_attribute)
                    self.misp_event.add_attribute(**attribute)
        else:
            for line in self.data:
                for index in types_indexes:
                    self.misp_event.add_attribute(**{'type': self.header[index], 'value': line[index]})

    ################################################################################
    ####                    Parsing standard MISP csv format                    ####
    ################################################################################

    def __parse_misp_csv(self):
        objects = {}
        attribute_fields = self.header[:1] + self.header[2:8]
        for line in self.data:
            a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number]
            attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))}
            attribute['to_ids'] = True if attribute['to_ids'] == '1' else False
            if tag:
                attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')]
            if relation:
{'fraction_non_alphanumeric': 0.15086602607258706, 'fraction_numerical': 0.0028153497766081155, 'mean_word_length': 2.1636011616650532, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4545215', 'n_tokens_mistral': 4326, 'n_tokens_neox': 4076, 'n_words': 1093}