<commit_before># -*- coding: utf-8 -*-
import json, os, base64
import pymisp

misperrors = {'error': 'Error'}
mispattributes = {'inputSource': ['file'], 'output': ['MISP attributes']}
moduleinfo = {'version': '0.1', 'author': '<NAME>',
              'description': 'Import Attributes from a csv file.',
              'module-type': ['import']}
moduleconfig = ['header']

duplicatedFields = {'mispType': {'mispComment': 'comment'},
                    'attrField': {'eventComment': 'comment'}}

def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if request.get('data'):
        data = base64.b64decode(request['data']).decode('utf-8')
    else:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors
    if not request.get('config') and not request['config'].get('header'):
        misperrors['error'] = "Configuration error"
        return misperrors
    config = request['config'].get('header').split(',')
    config = [c.strip() for c in config]
    data = parse_data(data.split('\n'))
    # find which delimiter is used
    delimiter, length = findDelimiter(config, data)
    # build the attributes
    result = buildAttributes(config, data, delimiter, length)
    r = {'results': result}
    return r

def parse_data(data):
    return_data = []
    for line in data:
        l = line.split('#')[0].strip() if '#' in line else line.strip()
        if l:
            return_data.append(l)
    print(len(return_data))
    return return_data

def findDelimiter(header, data):
    n = len(header)
    if n > 1:
        tmpData = []
        for da in data:
            tmp = []
            for d in (';', '|', '/', ',', '\t', '    ',):
                if da.count(d) == (n-1):
                    tmp.append(d)
            if len(tmp) == 1 and tmp == tmpData:
                return tmpData[0], n
            else:
                tmpData = tmp
    else:
        return None, 1

def buildAttributes(header, dataValues, delimiter, length):
    attributes = []
    # if there is only 1 field of data
    if delimiter is None:
        mispType = header[0]
        for data in dataValues:<commit_msg>Removed print<commit_after># -*- coding: utf-8 -*-
import json, os, base64
import pymisp

misperrors = {'error': 'Error'}
mispattributes = {'inputSource': ['file'], 'output': ['MISP attributes']}
moduleinfo = {'version': '0.1', 'author': '<NAME>',
              'description': 'Import Attributes from a csv file.',
              'module-type': ['import']}
moduleconfig = ['header']

duplicatedFields = {'mispType': {'mispComment': 'comment'},
                    'attrField': {'eventComment': 'comment'}}

def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if request.get('data'):
        data = base64.b64decode(request['data']).decode('utf-8')
    else:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors
    if not request.get('config') and not request['config'].get('header'):
        misperrors['error'] = "Configuration error"
        return misperrors
    config = request['config'].get('header').split(',')
    config = [c.strip() for c in config]
    data = parse_data(data.split('\n'))
    # find which delimiter is used
    delimiter, length = findDelimiter(config, data)
    # build the attributes
    result = buildAttributes(config, data, delimiter, length)
    r = {'results': result}
    return r

def parse_data(data):
    return_data = []
    for line in data:
        l = line.split('#')[0].strip() if '#' in line else line.strip()
        if l:
            return_data.append(l)
    return return_data

def findDelimiter(header, data):
    n = len(header)
    if n > 1:
        tmpData = []
        for da in data:
            tmp = []
            for d in (';', '|', '/', ',', '\t', '    ',):
                if da.count(d) == (n-1):
                    tmp.append(d)
            if len(tmp) == 1 and tmp == tmpData:
                return tmpData[0], n
            else:
                tmpData = tmp
    else:
        return None, 1

def buildAttributes(header, dataValues, delimiter, length):
    attributes = []
    # if there is only 1 field of data
    if delimiter is None:
        mispType = header[0]
        for data in dataValues:
{'fraction_non_alphanumeric': 0.15088207985143917, 'fraction_numerical': 0.008356545961002786, 'mean_word_length': 2.358534684333593, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579564', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1292, 'n_words': 373}