<commit_before>/* eslint-disable no-console */
const colors = require('colors/safe');
const fs = require('fs');
const prettyStringify = require('json-stringify-pretty-compact');
const shell = require('shelljs');
const YAML = require('js-yaml');
const fetch = require('node-fetch');

const languageNames = require('./language_names.js');

// fontawesome icons
const fontawesome = require('@fortawesome/fontawesome-svg-core');
const fas = require('@fortawesome/free-solid-svg-icons').fas;
const far = require('@fortawesome/free-regular-svg-icons').far;
const fab = require('@fortawesome/free-brands-svg-icons').fab;
fontawesome.library.add(fas, far, fab);

let _currBuild = null;


// if called directly, do the thing.
if (process.argv[1].indexOf('build_data.js') > -1) {
  buildData();
} else {
  module.exports = buildData;
}


function buildData() {
  if (_currBuild) return _currBuild;

  const START = '🏗   ' + colors.yellow('Building data...');
  const END = '👍  ' + colors.green('data built');

  console.log('');
  console.log(START);
  console.time(END);

  // Create symlinks if necessary..  { 'target': 'source' }
  const symlinks = {
    'land.html': 'dist/land.html',
    img: 'dist/img'
  };

  for (let target of Object.keys(symlinks)) {
    if (!shell.test('-L', target)) {
      console.log(`Creating symlink:  ${target} -> ${symlinks[target]}`);
      shell.ln('-sf', symlinks[target], target);
    }
  }

  // Start clean
  shell.rm('-f', [
    'data/territory_languages.json',
    'dist/locales/en.json',
    'dist/data/*',
    'svg/fontawesome/*.svg',
  ]);

  // compile Font Awesome icons
  let faIcons = new Set([
    // list here the icons we want to use in the UI that aren't tied to other data
    'fas-i-cursor',
    'fas-lock',
    'fas-th-list',
    'fas-user-cog'
  ]);
  // add icons for QA integrations
  readQAIssueIcons(faIcons);

  let territoryLanguages = generateTerritoryLanguages();
  fs.writeFileSync('data/territory_languages.json', prettyStringify(territoryLanguages, { maxLength: 9999 }) );
  writeEnJson();

  const languageInfo = languageNames.langNamesInNativeLang;
  fs.writeFileSync('data/languages.json', prettyStringify(languageInfo, { maxLength: 200 }));
  fs.writeFileSync('dist/data/languages.min.json', JSON.stringify(languageInfo));

  // Save individual data files
  let tasks = [
    minifyJSON('data/address_formats.json', 'dist/data/address_formats.min.json'),
    minifyJSON('data/imagery.json', 'dist/data/imagery.min.json'),
    minifyJSON('data/intro_graph.json', 'dist/data/intro_graph.min.json'),
    minifyJSON('data/keepRight.json', 'dist/data/keepRight.min.json'),
    minifyJSON('data/languages.json', 'dist/data/languages.min.json'),
    minifyJSON('data/phone_formats.json', 'dist/data/phone_formats.min.json'),
    minifyJSON('data/qa_data.json', 'dist/data/qa_data.min.json'),
    minifyJSON('data/shortcuts.json', 'dist/data/shortcuts.min.json'),
    minifyJSON('data/territory_languages.json', 'dist/data/territory_languages.min.json'),
    Promise.all([
      // Fetch the icons that are needed by the expected tagging schema version 
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/presets.min.json'),
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/preset_categories.min.json'),
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/fields.min.json'),
      // WARNING: we fetch the bleeding edge data too to make sure we're always hosting the
      // latest icons, but note that the format could break at any time
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/presets.min.json'),
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/preset_categories.min.json'),
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/fields.min.json')
    ])
    .then(responses => Promise.all(responses.map(response => response.json())))
    .then((results) => {
      // compile the icons used by all the presets<commit_msg>Remove trailing whitespace<commit_after>/* eslint-disable no-console */
const colors = require('colors/safe');
const fs = require('fs');
const prettyStringify = require('json-stringify-pretty-compact');
const shell = require('shelljs');
const YAML = require('js-yaml');
const fetch = require('node-fetch');

const languageNames = require('./language_names.js');

// fontawesome icons
const fontawesome = require('@fortawesome/fontawesome-svg-core');
const fas = require('@fortawesome/free-solid-svg-icons').fas;
const far = require('@fortawesome/free-regular-svg-icons').far;
const fab = require('@fortawesome/free-brands-svg-icons').fab;
fontawesome.library.add(fas, far, fab);

let _currBuild = null;


// if called directly, do the thing.
if (process.argv[1].indexOf('build_data.js') > -1) {
  buildData();
} else {
  module.exports = buildData;
}


function buildData() {
  if (_currBuild) return _currBuild;

  const START = '🏗   ' + colors.yellow('Building data...');
  const END = '👍  ' + colors.green('data built');

  console.log('');
  console.log(START);
  console.time(END);

  // Create symlinks if necessary..  { 'target': 'source' }
  const symlinks = {
    'land.html': 'dist/land.html',
    img: 'dist/img'
  };

  for (let target of Object.keys(symlinks)) {
    if (!shell.test('-L', target)) {
      console.log(`Creating symlink:  ${target} -> ${symlinks[target]}`);
      shell.ln('-sf', symlinks[target], target);
    }
  }

  // Start clean
  shell.rm('-f', [
    'data/territory_languages.json',
    'dist/locales/en.json',
    'dist/data/*',
    'svg/fontawesome/*.svg',
  ]);

  // compile Font Awesome icons
  let faIcons = new Set([
    // list here the icons we want to use in the UI that aren't tied to other data
    'fas-i-cursor',
    'fas-lock',
    'fas-th-list',
    'fas-user-cog'
  ]);
  // add icons for QA integrations
  readQAIssueIcons(faIcons);

  let territoryLanguages = generateTerritoryLanguages();
  fs.writeFileSync('data/territory_languages.json', prettyStringify(territoryLanguages, { maxLength: 9999 }) );
  writeEnJson();

  const languageInfo = languageNames.langNamesInNativeLang;
  fs.writeFileSync('data/languages.json', prettyStringify(languageInfo, { maxLength: 200 }));
  fs.writeFileSync('dist/data/languages.min.json', JSON.stringify(languageInfo));

  // Save individual data files
  let tasks = [
    minifyJSON('data/address_formats.json', 'dist/data/address_formats.min.json'),
    minifyJSON('data/imagery.json', 'dist/data/imagery.min.json'),
    minifyJSON('data/intro_graph.json', 'dist/data/intro_graph.min.json'),
    minifyJSON('data/keepRight.json', 'dist/data/keepRight.min.json'),
    minifyJSON('data/languages.json', 'dist/data/languages.min.json'),
    minifyJSON('data/phone_formats.json', 'dist/data/phone_formats.min.json'),
    minifyJSON('data/qa_data.json', 'dist/data/qa_data.min.json'),
    minifyJSON('data/shortcuts.json', 'dist/data/shortcuts.min.json'),
    minifyJSON('data/territory_languages.json', 'dist/data/territory_languages.min.json'),
    Promise.all([
      // Fetch the icons that are needed by the expected tagging schema version
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/presets.min.json'),
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/preset_categories.min.json'),
      fetch('https://cdn.jsdelivr.net/npm/@openstreetmap/id-tagging-schema@2/dist/fields.min.json'),
      // WARNING: we fetch the bleeding edge data too to make sure we're always hosting the
      // latest icons, but note that the format could break at any time
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/presets.min.json'),
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/preset_categories.min.json'),
      fetch('https://raw.githubusercontent.com/openstreetmap/id-tagging-schema/main/dist/fields.min.json')
    ])
    .then(responses => Promise.all(responses.map(response => response.json())))
    .then((results) => {
      // compile the icons used by all the presets
{'fraction_non_alphanumeric': 0.16471734892787523, 'fraction_numerical': 0.0029239766081871343, 'mean_word_length': 4.005487804878049, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3703795', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2709, 'n_words': 489}