<commit_before>import { ALIAS_AS_RELATIVE, ALIAS_AS_RESOLVABLE,
  STORE_BLOB, STORE_CONTENT, STORE_LINKS, STORE_STAT,
  isDotJS, isDotJSON, isDotNODE, isPackageJson, normalizePath
} from '../prelude/common.js';

import { log, wasReported } from './log.js';
import assert from 'assert';
import detector from './detector.js';
import follow from './follow.js';
import fs from 'fs-extra';
import globby from 'globby';
import natives from './natives.js';
import path from 'path';

function shortFromAlias (alias) {
  // alias = fs-promise or @types/node
  if (alias[0] === '@') {
    return alias.match(/^([^\\/]+[\\/][^\\/]+)/)[0];
  } else {
    return alias.match(/^[^\\/]+/)[0];
  }
}

function isPermissive (config) {
  if (config.private) return false;
  let { license, licenses } = config;
  if (license) {
    license = license.type || license;
  } else
  if (Array.isArray(licenses)) {
    license = licenses.map((c) => String(c.type || c)).join(',');
  }
  if (!license) return false;
  if (/^\(/.test(license)) license = license.slice(1);
  if (/\)$/.test(license)) license = license.slice(0, -1);
  license = license.toLowerCase();
  licenses = Array.prototype.concat(
    license.split(' or '), license.split(' and '),
    license.split('/'), license.split(',')
  );
  let result = false;
  const foss = [ 'isc', 'mit', 'apache-2.0', 'apache 2.0',
    'public domain', 'bsd', 'bsd-2-clause', 'bsd-3-clause', 'wtfpl',
    'cc-by-3.0', 'x11', 'artistic-2.0', 'gplv3', 'mpl', 'mplv2.0',
    'unlicense' ];
  for (const c of licenses) {
    result = foss.indexOf(c) >= 0;
    if (result) break;
  }
  return result;
}

function upon (p, base) {
  if (typeof p !== 'string') {
    throw wasReported(
      'Config items must be strings. See examples'
    );
  }
  let negate = false;
  if (p[0] === '!') {
    p = p.slice(1);
    negate = true;
  }
  if (!path.isAbsolute(p)) {
    p = path.join(base, p);
  }
  if (negate) {
    p = '!' + p;
  }
  return p;
}

function collect (ps) {
  return globby.sync(
    ps, { dot: true }
  );
}

function expandFiles (efs, base) {
  if (!Array.isArray(efs)) {
    efs = [ efs ];
  }
  efs = efs.concat(
    efs.map((ef) =>
      // npm/node_modules/fstream-npm/fstream-npm.js
      // Packer.prototype.readRules
      ef.replace(/\/+$/, '') + '/**'
    )
  );
  efs = collect(
    efs.map((p) => upon(p, base))
  );
  return efs;
}

class Walker {
  appendSub (task) {
    const { file, marker } = task;
    assert(!this.records[file]);
    const record = this.records[file] = { file };
    if (marker) record.marker = marker;
    if (task.reason) {
      log.debug('File %1 added to queue. It was required from %2', [ task.file, task.reason ]);
    } else {
      log.debug('File %1 added to queue', [ task.file ]);
    }
  }

  append (task) {
    task.file = normalizePath(task.file);
    const record = this.records[task.file];
    if (!record) this.appendSub(task);
    this.tasks.push(task);
  }

  async appendFilesFromConfig (marker) {
    const { config, base } = marker;
    const pkgConfig = config.pkg;

    if (pkgConfig) {
      let { scripts } = pkgConfig;

      if (scripts) {
        scripts = expandFiles(scripts, base);
        for (const script of scripts) {
          const stat = await fs.stat(script);
          if (stat.isFile()) {
            this.append({
              file: script,
              marker: marker,
              store: STORE_BLOB<commit_msg>show a warning that css gets into 'scripts'. addresses #79<commit_after>import { ALIAS_AS_RELATIVE, ALIAS_AS_RESOLVABLE,
  STORE_BLOB, STORE_CONTENT, STORE_LINKS, STORE_STAT,
  isDotJS, isDotJSON, isDotNODE, isPackageJson, normalizePath
} from '../prelude/common.js';

import { log, wasReported } from './log.js';
import assert from 'assert';
import detector from './detector.js';
import follow from './follow.js';
import fs from 'fs-extra';
import globby from 'globby';
import natives from './natives.js';
import path from 'path';

function shortFromAlias (alias) {
  // alias = fs-promise or @types/node
  if (alias[0] === '@') {
    return alias.match(/^([^\\/]+[\\/][^\\/]+)/)[0];
  } else {
    return alias.match(/^[^\\/]+/)[0];
  }
}

function isPermissive (config) {
  if (config.private) return false;
  let { license, licenses } = config;
  if (license) {
    license = license.type || license;
  } else
  if (Array.isArray(licenses)) {
    license = licenses.map((c) => String(c.type || c)).join(',');
  }
  if (!license) return false;
  if (/^\(/.test(license)) license = license.slice(1);
  if (/\)$/.test(license)) license = license.slice(0, -1);
  license = license.toLowerCase();
  licenses = Array.prototype.concat(
    license.split(' or '), license.split(' and '),
    license.split('/'), license.split(',')
  );
  let result = false;
  const foss = [ 'isc', 'mit', 'apache-2.0', 'apache 2.0',
    'public domain', 'bsd', 'bsd-2-clause', 'bsd-3-clause', 'wtfpl',
    'cc-by-3.0', 'x11', 'artistic-2.0', 'gplv3', 'mpl', 'mplv2.0',
    'unlicense' ];
  for (const c of licenses) {
    result = foss.indexOf(c) >= 0;
    if (result) break;
  }
  return result;
}

function upon (p, base) {
  if (typeof p !== 'string') {
    throw wasReported(
      'Config items must be strings. See examples'
    );
  }
  let negate = false;
  if (p[0] === '!') {
    p = p.slice(1);
    negate = true;
  }
  if (!path.isAbsolute(p)) {
    p = path.join(base, p);
  }
  if (negate) {
    p = '!' + p;
  }
  return p;
}

function collect (ps) {
  return globby.sync(
    ps, { dot: true }
  );
}

function expandFiles (efs, base) {
  if (!Array.isArray(efs)) {
    efs = [ efs ];
  }
  efs = efs.concat(
    efs.map((ef) =>
      // npm/node_modules/fstream-npm/fstream-npm.js
      // Packer.prototype.readRules
      ef.replace(/\/+$/, '') + '/**'
    )
  );
  efs = collect(
    efs.map((p) => upon(p, base))
  );
  return efs;
}

class Walker {
  appendSub (task) {
    const { file, marker } = task;
    assert(!this.records[file]);
    const record = this.records[file] = { file };
    if (marker) record.marker = marker;
    if (task.reason) {
      log.debug('File %1 added to queue. It was required from %2', [ task.file, task.reason ]);
    } else {
      log.debug('File %1 added to queue', [ task.file ]);
    }
  }

  append (task) {
    task.file = normalizePath(task.file);
    const record = this.records[task.file];
    if (!record) this.appendSub(task);
    this.tasks.push(task);
  }

  async appendFilesFromConfig (marker) {
    const { config, base } = marker;
    const pkgConfig = config.pkg;

    if (pkgConfig) {
      let { scripts } = pkgConfig;

      if (scripts) {
        scripts = expandFiles(scripts, base);
        for (const script of scripts) {
          const stat = await fs.stat(script);
          if (stat.isFile()) {
            if (!isDotJS(script)) {
              log.warn('Non-javascript file is specified in \'scripts\'.', [
                'Pkg will probably fail to parse. Specify *.js in glob.',
                script ]);
            }

            this.append({
              file: script,
              marker: marker,
              store: STORE_BLOB
{'fraction_non_alphanumeric': 0.18017386427369603, 'fraction_numerical': 0.007851934941110488, 'mean_word_length': 2.6007067137809186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1144898', 'n_tokens_mistral': 2573, 'n_tokens_neox': 2495, 'n_words': 619}