<commit_before>var path     = require('path')
var debug    = require('debug')('app:' + path.basename(__filename).replace('.js', ''))
var request  = require('request')
var async    = require('async')
var op       = require('object-path')
var md       = require('marked')
var random   = require('randomstring')
var crypto   = require('crypto')
// var sanitize = require('sanitize-html')


LIMIT_PARALLEL = 3

function sign_data(data) {
    data = data || {}

    if (!APP_ENTU_USER || !APP_ENTU_KEY) return data

    var conditions = []
    for (k in data) {
        conditions.push({k: data[k]})
    }

    var expiration = new Date()
    expiration.setMinutes(expiration.getMinutes() + 10)

    data.user = APP_ENTU_USER
    var buff_str = JSON.stringify({expiration: expiration.toISOString(), conditions: conditions})
    data.policy = new Buffer(buff_str).toString('base64')
    data.signature = crypto.createHmac('sha1', APP_ENTU_KEY).update(data.policy).digest('base64')

    return data
}


//Get entity from Entu
function get_entity(id, auth_id, auth_token, callback) {
    var headers = {}
    var qs = {}
    if (auth_id && auth_token) {
        headers = {'X-Auth-UserId': auth_id, 'X-Auth-Token': auth_token}
    } else {
        qs = sign_data()
    }

    // debug('get_entity: ' + APP_ENTU_URL + '/entity-' + id)
    request.get({url: APP_ENTU_URL + '/entity-' + id, headers: headers, qs: qs, strictSSL: true, json: true}, function(error, response, body) {
        if (error) {
            return callback(error)
        }
        if (response.statusCode !== 200 || !body.result) return callback(new Error(op.get(body, 'error', body)))
        var properties = op.get(body, 'result.properties', {})
        var entity = {
            id: op.get(body, 'result.id', null),
            displayname: op.get(body, 'result.displayname', null),
            displayinfo: op.get(body, 'result.displayinfo', null),
            definition: op.get(body, 'result.definition.keyname', null),
            picture: APP_ENTU_URL + '/entity-' + op.get(body, 'result.id', null) + '/picture',
            right: op.get(body, 'result.right', null),
            properties: {}
        }
        for (var p in properties) {
            if (op.has(properties, [p, 'values'])) {
                for (var v in op.get(properties, [p, 'values'])) {
                    if (op.get(properties, [p, 'datatype']) === 'file') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            file: APP_ENTU_URL + '/file-' + op.get(properties, [p, 'values', v, 'db_value'])
                        })
                    } else if (op.get(properties, [p, 'datatype']) === 'text') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            md: md(op.get(properties, [p, 'values', v, 'db_value']))
                        })
                    } else if (op.get(properties, [p, 'datatype']) === 'reference') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            reference: op.get(properties, [p, 'values', v, 'db_value'])
                        })
                    } else {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                        })
                    }
                }
                if (op.get(properties, [p, 'multiplicity']) === 1) op.set(entity, ['properties', p], op.get(entity, ['properties', p, 0]))
            }
        }
        // debug(JSON.stringify(entity, null, '  '))
        callback(null, op(entity))
    })
}


//Get entities by definition
function get_entities(definition, limit, auth_id, auth_token, callback) {
    if (!definition) {
        callback(new Error('Missing "definition"'))
        return
    }

    var qs = {definition: definition}
    var headers = {}
    if (limit) {
        qs['limit'] = limit
    }
    if (auth_id && auth_token) {
        headers = {'X-Auth-UserId': auth_id, 'X-Auth-Token': auth_token}
    } else {
        qs = sign_data(qs)
    }<commit_msg>fix potential hidden issue with accessing alien properties<commit_after>var path     = require('path')
var debug    = require('debug')('app:' + path.basename(__filename).replace('.js', ''))
var request  = require('request')
var async    = require('async')
var op       = require('object-path')
var md       = require('marked')
var crypto   = require('crypto')
// var sanitize = require('sanitize-html')

LIMIT_PARALLEL = 3

function sign_data(data) {
    data = data || {}

    if (!APP_ENTU_USER || !APP_ENTU_KEY) { return data }

    var conditions = []
    for (k in data) {
        if (data.hasOwnProperty(k)) {
            conditions.push({k: data[k]})
        }
    }

    var expiration = new Date()
    expiration.setMinutes(expiration.getMinutes() + 10)

    data.user = APP_ENTU_USER
    var buff_str = JSON.stringify({expiration: expiration.toISOString(), conditions: conditions})
    data.policy = new Buffer(buff_str).toString('base64')
    data.signature = crypto.createHmac('sha1', APP_ENTU_KEY).update(data.policy).digest('base64')

    return data
}


//Get entity from Entu
function get_entity(id, auth_id, auth_token, callback) {
    var headers = {}
    var qs = {}
    if (auth_id && auth_token) {
        headers = {'X-Auth-UserId': auth_id, 'X-Auth-Token': auth_token}
    } else {
        qs = sign_data()
    }

    // debug('get_entity: ' + APP_ENTU_URL + '/entity-' + id)
    request.get({url: APP_ENTU_URL + '/entity-' + id, headers: headers, qs: qs, strictSSL: true, json: true}, function(error, response, body) {
        if (error) {
            return callback(error)
        }
        if (response.statusCode !== 200 || !body.result) { return callback(new Error(op.get(body, 'error', body))) }
        var properties = op.get(body, 'result.properties', {})
        var entity = {
            id: op.get(body, 'result.id', null),
            displayname: op.get(body, 'result.displayname', null),
            displayinfo: op.get(body, 'result.displayinfo', null),
            definition: op.get(body, 'result.definition.keyname', null),
            picture: APP_ENTU_URL + '/entity-' + op.get(body, 'result.id', null) + '/picture',
            right: op.get(body, 'result.right', null),
            properties: {}
        }
        for (var p in properties) {
            if (op.has(properties, [p, 'values'])) {
                for (var v in op.get(properties, [p, 'values'])) {
                    if (op.get(properties, [p, 'datatype']) === 'file') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            file: APP_ENTU_URL + '/file-' + op.get(properties, [p, 'values', v, 'db_value'])
                        })
                    } else if (op.get(properties, [p, 'datatype']) === 'text') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            md: md(op.get(properties, [p, 'values', v, 'db_value']))
                        })
                    } else if (op.get(properties, [p, 'datatype']) === 'reference') {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                            reference: op.get(properties, [p, 'values', v, 'db_value'])
                        })
                    } else {
                        op.push(entity, ['properties', p], {
                            id: op.get(properties, [p, 'values', v, 'id']),
                            value: op.get(properties, [p, 'values', v, 'value']),
                        })
                    }
                }
                if (op.get(properties, [p, 'multiplicity']) === 1) { op.set(entity, ['properties', p], op.get(entity, ['properties', p, 0])) }
            }
        }
        // debug(JSON.stringify(entity, null, '  '))
        callback(null, op(entity))
    })
}


//Get entities by definition
function get_entities(definition, limit, auth_id, auth_token, callback) {
    if (!definition) {
        callback(new Error('Missing "definition"'))
        return
    }

    var qs = {definition: definition}
    var headers = {}
    if (limit) {
        qs['limit'] = limit
    }
    if (auth_id && auth_token) {
        headers = {'X-Auth-UserId': auth_id, 'X-Auth-Token': auth_token}
    } else {
        qs = sign_data(qs)
    }
{'fraction_non_alphanumeric': 0.16412009830110055, 'fraction_numerical': 0.002778074580617587, 'mean_word_length': 1.906832298136646, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7080864', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2782, 'n_words': 658}