<commit_before>const fs = require('fs');

const ditty = require('parse-ditty');

const dataPath = `${__dirname}/../data`;
const rawCategories = fs.readdirSync(dataPath);
const categoryPattern = new RegExp(
  '^' +
    '(\\d+)' +    // (1) Category ID,
    '\\.\\s*' +   // followed by a dot and optional whitespace,
    '(.+)' +      // followed by (2) the category name.
  '$'
);
const categoriesData = rawCategories.map((dirname) => {
  const matches = dirname.match(categoryPattern);
  return {
    id: parseInt(matches[1], 10),
    name: matches[2],
  };
});

const categoryDirs = rawCategories.map((dirname, index) => ({
  id: categoriesData[index].id,
  dirname,
}));

const songsData = categoryDirs.reduce((result, category) => {
  const categoryPath = `${dataPath}/${category.dirname}`;
  const songFiles = fs.readdirSync(categoryPath);
  const parsedSongs = songFiles.map(filename => {
    const rawSong = fs.readFileSync(`${categoryPath}/${filename}`, 'utf8');
    return Object.assign({},
      ditty(rawSong),
      { category: category.id }<commit_msg>Update the backend<commit_after>const fs = require('fs');

const ditty = require('parse-ditty');
const pick = require('1-liners/pick');

const dataPath = `${__dirname}/../data`;
const rawCategories = fs.readdirSync(dataPath);
const categoryPattern = new RegExp(
  '^' +
    '(\\d+)' +    // (1) Category ID,
    '\\.\\s*' +   // followed by a dot and optional whitespace,
    '(.+)' +      // followed by (2) the category name.
  '$'
);
const categoriesData = rawCategories.map((dirname) => {
  const matches = dirname.match(categoryPattern);
  return {
    id: parseInt(matches[1], 10),
    name: matches[2],
  };
});

const categoryDirs = rawCategories.map((dirname, index) => ({
  id: categoriesData[index].id,
  dirname,
}));

const songsData = categoryDirs.reduce((result, category) => {
  const categoryPath = `${dataPath}/${category.dirname}`;
  const songFiles = fs.readdirSync(categoryPath);
  const parsedSongs = songFiles.map(filename => {
    const rawSong = fs.readFileSync(`${categoryPath}/${filename}`, 'utf8');
    return Object.assign({},
      pick(['number', 'title', 'blocks'], ditty(rawSong)),
      { category: category.id }
{'fraction_non_alphanumeric': 0.1822704658525554, 'fraction_numerical': 0.0067842605156037995, 'mean_word_length': 3.893805309734513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7018680', 'n_tokens_mistral': 720, 'n_tokens_neox': 703, 'n_words': 158}