<commit_before>const { promisify } = require('util');
const exec = promisify(require('child_process').exec);
const readdir = promisify(require('fs').readdir);

const mac = process.platform === 'darwin';

const filetype = mac ? '.dylib' : '.so';

/* these libraries will always have dynamic dependencies, so ignore */
const ignoredLibraries = mac
  ? new Set([])
  : new Set([
      'libalsaout.so',
      'libmpris.so',
      'libpipewireout.so',
      'libpulseout.so',
    ]);

/* we assume these dependencies will always be available on the user's
target machine. */
const validLibraries = mac
  ? new Set([
      '/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit',
      '/System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices',
      '/System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox',
      '/System/Library/Frameworks/Carbon.framework/Versions/A/Carbon',
      '/System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio',
      '/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation',
      '/System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics',
      '/System/Library/Frameworks/CoreMedia.framework/Versions/A/CoreMedia',
      '/System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo',
      '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation',
      '/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration',
      '/usr/lib/libc++.1.dylib',
      '/usr/lib/libobjc.A.dylib',
      '/usr/lib/libSystem.B.dylib',
      '/usr/lib/libz.1.dylib',
    ])
  : new Set([
      '/lib64/ld-linux-x86-64.so.2',
      'libc.so.6',
      'libdl.so.2',
      'libgcc_s.so.1',
      'libm.so.6',
      'libstdc++.so.6',
      'libz.so.1',
      'linux-vdso.so.1',
    ]);

let errors = 0;

const path = process.argv[2];

if (!path) {
  console.log('\n\nusage: node scan-standalone.js <path>\n\n');
  process.exit(1);
}

const ls = async (leaf) => {
  const output = await readdir(`${path}/${leaf}`);
  const files = output
    .filter((fn) => fn.indexOf(filetype) !== -1)
    .filter((fn) => !ignoredLibraries.has(fn));
  return files;
};

const lddLinux = async (fn) => {<commit_msg>Added a couple more libraries to the ignore list.<commit_after>const { promisify } = require('util');
const exec = promisify(require('child_process').exec);
const readdir = promisify(require('fs').readdir);

const mac = process.platform === 'darwin';

const filetype = mac ? '.dylib' : '.so';

/* these libraries will always have dynamic dependencies, so ignore */
const ignoredLibraries = mac
  ? new Set([])
  : new Set([
      'libalsaout.so',
      'libmpris.so',
      'libpipewireout.so',
      'libpulseout.so',
    ]);

/* we assume these dependencies will always be available on the user's
target machine. */
const validLibraries = mac
  ? new Set([
      '/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit',
      '/System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices',
      '/System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox',
      '/System/Library/Frameworks/Carbon.framework/Versions/A/Carbon',
      '/System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio',
      '/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation',
      '/System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics',
      '/System/Library/Frameworks/CoreMedia.framework/Versions/A/CoreMedia',
      '/System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo',
      '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation',
      '/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration',
      '/usr/lib/libc++.1.dylib',
      '/usr/lib/libobjc.A.dylib',
      '/usr/lib/libSystem.B.dylib',
      '/usr/lib/libz.1.dylib',
    ])
  : new Set([
      '/lib64/ld-linux-x86-64.so.2',
      'libc.so.6',
      'libdl.so.2',
      'libgcc_s.so.1',
      'libm.so.6',
      'libpthread.so.0',
      'librt.so.1',
      'libstdc++.so.6',
      'libz.so.1',
      'linux-vdso.so.1',
    ]);

let errors = 0;

const path = process.argv[2];

if (!path) {
  console.log('\n\nusage: node scan-standalone.js <path>\n\n');
  process.exit(1);
}

const ls = async (leaf) => {
  const output = await readdir(`${path}/${leaf}`);
  const files = output
    .filter((fn) => fn.indexOf(filetype) !== -1)
    .filter((fn) => !ignoredLibraries.has(fn));
  return files;
};

const lddLinux = async (fn) => {
{'fraction_non_alphanumeric': 0.16980315812243132, 'fraction_numerical': 0.009085009733939001, 'mean_word_length': 3.3134328358208953, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6312326', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1559, 'n_words': 242}