<commit_before>"use strict";

var extend = require('extend');

var logger = require('../logger');
var mycache = require('../cache');
var myhash = require("../hash");
var uri_scanner = require('./uri');

var urlRegex = /((https?:\/\/(?:(?:[a-zA-Z0-9\$\-\_\.\+\!\*\'\(\)\,\;\?\&\=]|(?:\%[a-fA-F0-9]{2})){1,64}(?:\:(?:[a-zA-Z0-9\$\-\_\.\+\!\*\'\(\)\,\;\?\&\=]|(?:\%[a-fA-F0-9]{2})){1,25})?\@)?)?((?:(?:[a-zA-Z0-9][a-zA-Z0-9\-]{0,64}\.)+(?:(?:aero|arpa|asia|a[cdefgilmnoqrstuwxz])|(?:biz|b[abdefghijmnorstvwyz])|(?:cat|com|coop|c[acdfghiklmnoruvxyz])|d[ejkmoz]|(?:edu|e[cegrstu])|f[ijkmor]|(?:gov|g[abdefghilmnpqrstuwy])|h[kmnrtu]|(?:info|int|i[delmnoqrst])|(?:jobs|j[emop])|k[eghimnrwyz]|l[abcikrstuvy]|(?:mil|mobi|museum|m[acdghklmnopqrstuvwxyz])|(?:name|net|n[acefgilopruz])|(?:org|om)|(?:pro|p[aefghklmnrstwy])|qa|r[eouw]|s[abcdeghijklmnortuvyz]|(?:tel|travel|t[cdfghjklmnoprtvwz])|u[agkmsyz]|v[aceginu]|w[fs]|y[etu]|z[amw]))|(?:(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[0-9])))(?:\:\d{1,5})?)(\/(?:(?:[a-zA-Z0-9\;\/\?\:\@\&\=\#\~\-\.\+\!\*\'\(\)\,\_])|(?:\%[a-fA-F0-9]{2}))*)?(?:\b|$)/gi;


function fix_uri(resource) {
  if (resource.startsWith('//')) {
    return "http:" + resource;
  }

  if (!(
      resource.startsWith('http://') ||
      resource.startsWith('https://') ||
      resource.startsWith('ftp://')
    )) {
    return "http://" + resource
  }

  return resource
}


function check_text(payload) {
  if (payload.results) {
    logger.debug("Skip checking text");
    return Promise.resolve(payload);
  }

  return new Promise((fulfill, reject) => {
    let links = payload.resource.match(urlRegex) || [];
    let scanner_promises = [];

    logger.debug(`Found links in text: ${links}`);
    payload.resource = links;

    for (let idx = 0, len = links.length; idx < len; idx++) {
      let link = fix_uri(links[idx]);
      scanner_promises.push(uri_scanner.scan_uri(link, payload.options));
    }

    Promise.all(scanner_promises).then((values) => {
      extend(payload, {
        malicious: false,
        results: {},
      });

      for (let idx = 0, len = values.length; idx < len; idx++) {
        let v = values[idx];

        payload.malicious = payload.malicious || (!!v.malicious);
        payload.results[v.resource] = v.result
      }
      logger.debug(`Text scan result: ${payload}`);
      fulfill(payload);<commit_msg>refactor: result instead of results for scan/text<commit_after>"use strict";

var extend = require('extend');

var logger = require('../logger');
var mycache = require('../cache');
var myhash = require("../hash");
var uri_scanner = require('./uri');

var urlRegex = /((https?:\/\/(?:(?:[a-zA-Z0-9\$\-\_\.\+\!\*\'\(\)\,\;\?\&\=]|(?:\%[a-fA-F0-9]{2})){1,64}(?:\:(?:[a-zA-Z0-9\$\-\_\.\+\!\*\'\(\)\,\;\?\&\=]|(?:\%[a-fA-F0-9]{2})){1,25})?\@)?)?((?:(?:[a-zA-Z0-9][a-zA-Z0-9\-]{0,64}\.)+(?:(?:aero|arpa|asia|a[cdefgilmnoqrstuwxz])|(?:biz|b[abdefghijmnorstvwyz])|(?:cat|com|coop|c[acdfghiklmnoruvxyz])|d[ejkmoz]|(?:edu|e[cegrstu])|f[ijkmor]|(?:gov|g[abdefghilmnpqrstuwy])|h[kmnrtu]|(?:info|int|i[delmnoqrst])|(?:jobs|j[emop])|k[eghimnrwyz]|l[abcikrstuvy]|(?:mil|mobi|museum|m[acdghklmnopqrstuvwxyz])|(?:name|net|n[acefgilopruz])|(?:org|om)|(?:pro|p[aefghklmnrstwy])|qa|r[eouw]|s[abcdeghijklmnortuvyz]|(?:tel|travel|t[cdfghjklmnoprtvwz])|u[agkmsyz]|v[aceginu]|w[fs]|y[etu]|z[amw]))|(?:(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[0-9])))(?:\:\d{1,5})?)(\/(?:(?:[a-zA-Z0-9\;\/\?\:\@\&\=\#\~\-\.\+\!\*\'\(\)\,\_])|(?:\%[a-fA-F0-9]{2}))*)?(?:\b|$)/gi;


function fix_uri(resource) {
  if (resource.startsWith('//')) {
    return "http:" + resource;
  }

  if (!(
      resource.startsWith('http://') ||
      resource.startsWith('https://') ||
      resource.startsWith('ftp://')
    )) {
    return "http://" + resource
  }

  return resource
}


function check_text(payload) {
  if (payload.result) {
    logger.debug("Skip checking text");
    return Promise.resolve(payload);
  }

  return new Promise((fulfill, reject) => {
    let links = payload.resource.match(urlRegex) || [];
    let scanner_promises = [];

    logger.debug(`Found links in text: ${links}`);
    payload.resource = links;

    for (let idx = 0, len = links.length; idx < len; idx++) {
      let link = fix_uri(links[idx]);
      scanner_promises.push(uri_scanner.scan_uri(link, payload.options));
    }

    Promise.all(scanner_promises).then((values) => {
      extend(payload, {
        malicious: false,
        result: {},
      });

      for (let idx = 0, len = values.length; idx < len; idx++) {
        let v = values[idx];

        payload.malicious = payload.malicious || (!!v.malicious);
        payload.result[v.resource] = v.result
      }
      logger.debug(`Text scan result: ${payload}`);
      fulfill(payload);
{'fraction_non_alphanumeric': 0.31537708129285014, 'fraction_numerical': 0.04466209598432909, 'mean_word_length': 3.881453154875717, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2203368', 'n_tokens_mistral': 2381, 'n_tokens_neox': 2318, 'n_words': 209}