<commit_before>import { promises as fs } from 'node:fs'
import regenerate from 'regenerate'
// @ts-expect-error: untyped
import alphabetics from '@unicode/unicode-13.0.0/Binary_Property/Alphabetic/code-points.js'

const categoryBase = new URL('../node_modules/@unicode/unicode-13.0.0/General_Category/', import.meta.url)

// Unicode General Categories to remove.
const ranges = [
  // Some numbers:
  'Other_Number',

  // Some punctuation:
  'Close_Punctuation',
  'Final_Punctuation',
  'Initial_Punctuation',
  'Open_Punctuation',
  'Other_Punctuation',
  // All except a normal `-` (dash)
  'Dash_Punctuation',

  // All:
  'Symbol',
  'Control',
  'Private_Use',
  'Format',
  'Unassigned',

  // All except a normal ` ` (space)
  'Separator'
]

const generator = regenerate()

let index = -1

// Add code points to strip.
while (++index < ranges.length) {
  const name = ranges[index]
  const fp = `./${name}/code-points.js`
  /** @type {{default: Array<number>}} */
  const { default: codePoints } = await import(new URL(fp, categoryBase).href)

  generator.add(codePoints)
}

generator
  // Some overlap between letters and Other Symbol.
  .remove(alphabetics)<commit_msg>Refactor some more<commit_after>import { promises as fs } from 'node:fs'
import regenerate from 'regenerate'
// @ts-expect-error: untyped
import alphabetics from '@unicode/unicode-13.0.0/Binary_Property/Alphabetic/code-points.js'

const categoryBase = new URL('../node_modules/@unicode/unicode-13.0.0/General_Category/', import.meta.url)

// Unicode General Categories to remove.
const ranges = [
  // Some numbers:
  'Other_Number',

  // Some punctuation:
  'Close_Punctuation',
  'Final_Punctuation',
  'Initial_Punctuation',
  'Open_Punctuation',
  'Other_Punctuation',
  // All except a normal `-` (dash)
  'Dash_Punctuation',

  // All:
  'Symbol',
  'Control',
  'Private_Use',
  'Format',
  'Unassigned',

  // All except a normal ` ` (space)
  'Separator'
]

const generator = regenerate()

let index = -1

// Add code points to strip.
while (++index < ranges.length) {
  const name = ranges[index]
  /** @type {{default: Array<number>}} */
  const { default: codePoints } = await import(new URL(name + '/code-points.js', categoryBase).href)

  generator.add(codePoints)
}

generator
  // Some overlap between letters and Other Symbol.
  .remove(alphabetics)
{'fraction_non_alphanumeric': 0.1469961653174265, 'fraction_numerical': 0.007669365146996165, 'mean_word_length': 3.8814968814968815, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2326268', 'n_tokens_mistral': 811, 'n_tokens_neox': 783, 'n_words': 191}