<commit_before>/* jslint node:true */

'use strict';

exports = module.exports = {
    getProfileByIdentifier: getProfileByIdentifier,
    getAll: getAll,
    getAllLean: getAllLean,
    get: get,
    add: add,
    put: put,
    del: del,
    exp: exp,
    imp: imp,
    publicLink: publicLink,
    getByShareId: getByShareId,
    extractURLs: extractURLs,
    extractTags: extractTags,
    facelift: facelift,
    cleanupTags: cleanupTags,
    importThings: importThings,

    // TODO remove eventually
    hasOldData: null,
    expOldData: expOldData,
    cleanupOldData: cleanupOldData,

    TYPE_IMAGE: 'image',
    TYPE_UNKNOWN: 'unknown'
};

var assert = require('assert'),
    async = require('async'),
    config = require('./config.js'),
    path = require('path'),
    fs = require('fs'),
    ldapjs = require('ldapjs'),
    mkdirp = require('mkdirp'),
    url = require('url'),
    tags = require('./database/tags.js'),
    tar = require('tar-fs'),
    things = require('./database/things.js'),
    rimraf = require('rimraf'),
    safe = require('safetydance'),
    shares = require('./database/shares.js'),
    superagent = require('superagent');

var GET_URL = new RegExp('(^|[ \t\r\n])((ftp|http|https|gopher|mailto|news|nntp|telnet|wais|file|prospero|aim|webcal):(([A-Za-z0-9$_.+!*(),;/?:@&~=-])|%[A-Fa-f0-9]{2}){2,}(#([a-zA-Z0-9$_.+!*(),;/?:@&~=%-]*))?([A-Za-z0-9$_+!*();/?:~-]))', 'g');
var PRETTY_URL_LENGTH = 40;

// identifier may be userId, email, username
function getProfileByIdentifier(identifier, callback) {
    assert.strictEqual(typeof identifier, 'string');
    assert.strictEqual(typeof callback, 'function');

    var ldapClient = ldapjs.createClient({ url: process.env.LDAP_URL });
    ldapClient.on('error', function (error) {
        console.error('LDAP error', error);
        callback(error);
    });

    ldapClient.search(process.env.LDAP_USERS_BASE_DN, { filter: '(|(uid=' + identifier + ')(mail=' + identifier + ')(username=' + identifier + '))' }, function (error, result) {
        if (error) return callback(error);

        var items = [];

        result.on('searchEntry', function(entry) {
            items.push(entry.object);
        });

        result.on('error', function (error) {
            callback(error);
        });

        result.on('end', function (result) {
            if (result.status !== 0) return callback(new Error('non-zero status from LDAP search: ' + result.status));
            if (items.length === 0) return callback(new Error('Duplicate entries found'));

            var out = {
                id: items[0].uid,
                username: items[0].username,
                displayName: items[0].displayname,
                email: items[0].mail
            };

            callback(null, out);
        });
    });
}

function extractURLs(content) {
    var lines = content.split('\n');
    var urls = [];

    lines.forEach(function (line) {
        var tmp = line.match(GET_URL);
        if (tmp === null) return;

        urls = urls.concat(tmp.map(function (url) {
            return url.trim();
        }));
    });

    return urls.filter(function (item, pos, self) {
        return self.indexOf(item) === pos;
    });
}

function escapeRegExp(str) {
    return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
}

function extractTags(content) {
    var tagObjects = [];

    // first replace all urls which might contain # with placeholders
    var urls = extractURLs(content);
    urls.forEach(function (u) {
        content = content.replace(new RegExp(escapeRegExp(u), 'gmi'), ' --URL_PLACEHOLDER-- ');
    });

    var lines = content.split('\n');
    lines.forEach(function (line) {
        var tmp = line.match(/#([\u00C0-\u017Fa-zA-Z0-9]+)/g);
        if (tmp === null) return;

        tagObjects = tagObjects.concat(tmp.map(function (tag) {
            return tag.slice(1).toLowerCase();
        }));
    });

    return tagObjects;
}

function extractExternalContent(content, callback) {
    var urls = extractURLs(content);
    var externalContent = [];

    async.each(urls, function (url, callback) {
        superagent.get(url).end(function (error, result) {
            var obj = { url: url, type: exports.TYPE_UNKNOWN };

            if (error) {
                console.log('[WARN] failed to fetch external content %s', url);
            } else {
                if (result.type.indexOf('image/') === 0) {
                    obj = { url: url, type: exports.TYPE_IMAGE };
                }

                console.log('[INFO] external content type %s - %s', obj.type, obj.url);
            }

            externalContent.push(obj);

            callback(null);<commit_msg>Only fetch the HEAD for external content requests<commit_after>/* jslint node:true */

'use strict';

exports = module.exports = {
    getProfileByIdentifier: getProfileByIdentifier,
    getAll: getAll,
    getAllLean: getAllLean,
    get: get,
    add: add,
    put: put,
    del: del,
    exp: exp,
    imp: imp,
    publicLink: publicLink,
    getByShareId: getByShareId,
    extractURLs: extractURLs,
    extractTags: extractTags,
    facelift: facelift,
    cleanupTags: cleanupTags,
    importThings: importThings,

    // TODO remove eventually
    hasOldData: null,
    expOldData: expOldData,
    cleanupOldData: cleanupOldData,

    TYPE_IMAGE: 'image',
    TYPE_UNKNOWN: 'unknown'
};

var assert = require('assert'),
    async = require('async'),
    config = require('./config.js'),
    path = require('path'),
    fs = require('fs'),
    ldapjs = require('ldapjs'),
    mkdirp = require('mkdirp'),
    url = require('url'),
    tags = require('./database/tags.js'),
    tar = require('tar-fs'),
    things = require('./database/things.js'),
    rimraf = require('rimraf'),
    safe = require('safetydance'),
    shares = require('./database/shares.js'),
    superagent = require('superagent');

var GET_URL = new RegExp('(^|[ \t\r\n])((ftp|http|https|gopher|mailto|news|nntp|telnet|wais|file|prospero|aim|webcal):(([A-Za-z0-9$_.+!*(),;/?:@&~=-])|%[A-Fa-f0-9]{2}){2,}(#([a-zA-Z0-9$_.+!*(),;/?:@&~=%-]*))?([A-Za-z0-9$_+!*();/?:~-]))', 'g');
var PRETTY_URL_LENGTH = 40;

// identifier may be userId, email, username
function getProfileByIdentifier(identifier, callback) {
    assert.strictEqual(typeof identifier, 'string');
    assert.strictEqual(typeof callback, 'function');

    var ldapClient = ldapjs.createClient({ url: process.env.LDAP_URL });
    ldapClient.on('error', function (error) {
        console.error('LDAP error', error);
        callback(error);
    });

    ldapClient.search(process.env.LDAP_USERS_BASE_DN, { filter: '(|(uid=' + identifier + ')(mail=' + identifier + ')(username=' + identifier + '))' }, function (error, result) {
        if (error) return callback(error);

        var items = [];

        result.on('searchEntry', function(entry) {
            items.push(entry.object);
        });

        result.on('error', function (error) {
            callback(error);
        });

        result.on('end', function (result) {
            if (result.status !== 0) return callback(new Error('non-zero status from LDAP search: ' + result.status));
            if (items.length === 0) return callback(new Error('Duplicate entries found'));

            var out = {
                id: items[0].uid,
                username: items[0].username,
                displayName: items[0].displayname,
                email: items[0].mail
            };

            callback(null, out);
        });
    });
}

function extractURLs(content) {
    var lines = content.split('\n');
    var urls = [];

    lines.forEach(function (line) {
        var tmp = line.match(GET_URL);
        if (tmp === null) return;

        urls = urls.concat(tmp.map(function (url) {
            return url.trim();
        }));
    });

    return urls.filter(function (item, pos, self) {
        return self.indexOf(item) === pos;
    });
}

function escapeRegExp(str) {
    return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
}

function extractTags(content) {
    var tagObjects = [];

    // first replace all urls which might contain # with placeholders
    var urls = extractURLs(content);
    urls.forEach(function (u) {
        content = content.replace(new RegExp(escapeRegExp(u), 'gmi'), ' --URL_PLACEHOLDER-- ');
    });

    var lines = content.split('\n');
    lines.forEach(function (line) {
        var tmp = line.match(/#([\u00C0-\u017Fa-zA-Z0-9]+)/g);
        if (tmp === null) return;

        tagObjects = tagObjects.concat(tmp.map(function (tag) {
            return tag.slice(1).toLowerCase();
        }));
    });

    return tagObjects;
}

function extractExternalContent(content, callback) {
    var urls = extractURLs(content);
    var externalContent = [];

    async.each(urls, function (url, callback) {
        superagent.head(url).timeout(20000).end(function (error, result) {
            var obj = { url: url, type: exports.TYPE_UNKNOWN };

            if (error) {
                console.log('[WARN] failed to fetch external content %s', url);
            } else {
                if (result.type.indexOf('image/') === 0) {
                    obj = { url: url, type: exports.TYPE_IMAGE };
                }

                console.log('[INFO] external content type %s - %s', obj.type, obj.url);
            }

            externalContent.push(obj);

            callback(null);
{'fraction_non_alphanumeric': 0.17335456475583863, 'fraction_numerical': 0.006475583864118896, 'mean_word_length': 2.5271433919880195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4459167', 'n_tokens_mistral': 3041, 'n_tokens_neox': 2944, 'n_words': 610}