<commit_before>/* eslint-disable no-console */
const colors = require('colors/safe');
const fs = require('fs');
const glob = require('glob');
const jsonschema = require('jsonschema');
const path = require('path');
const prettyStringify = require('json-stringify-pretty-compact');
const shell = require('shelljs');
const YAML = require('js-yaml');

const fieldSchema = require('./data/presets/schema/field.json');
const presetSchema = require('./data/presets/schema/preset.json');
const nsi = require('name-suggestion-index');
const deprecated = require('./data/deprecated.json').dataDeprecated;

// fontawesome icons
const fontawesome = require('@fortawesome/fontawesome-svg-core');
const fas = require('@fortawesome/free-solid-svg-icons').fas;
const far = require('@fortawesome/free-regular-svg-icons').far;
const fab = require('@fortawesome/free-brands-svg-icons').fab;
fontawesome.library.add(fas, far, fab);

const request = require('request').defaults({ maxSockets: 1 });

let _currBuild = null;


function buildData() {
  if (_currBuild) return _currBuild;

  const START = '🏗   ' + colors.yellow('Building data...');
  const END = '👍  ' + colors.green('data built');

  console.log('');
  console.log(START);
  console.time(END);

  // Create symlinks if necessary..  { 'target': 'source' }
  const symlinks = {
    'land.html': 'dist/land.html',
    img: 'dist/img'
  };

  for (let target of Object.keys(symlinks)) {
    if (!shell.test('-L', target)) {
      console.log(`Creating symlink:  ${target} -> ${symlinks[target]}`);
      shell.ln('-sf', symlinks[target], target);
    }
  }

  // Translation strings
  let tstrings = {
    categories: {},
    fields: {},
    presets: {}
  };

  // Font Awesome icons used
  let faIcons = {
    'fas-i-cursor': {},
    'fas-lock': {},
    'fas-long-arrow-alt-right': {},
    'fas-th-list': {},
    'fas-user-cog': {}
  };

  // The Noun Project icons used
  let tnpIcons = {};

  // all fields searchable under "add field"
  let searchableFieldIDs = {};

  // Start clean
  shell.rm('-f', [
    'data/presets/categories.json',
    'data/presets/fields.json',
    'data/presets/presets.json',
    'data/presets.yaml',
    'data/taginfo.json',
    'data/territory-languages.json',
    'dist/locales/en.json',
    'svg/fontawesome/*.svg',
  ]);

  let categories = generateCategories(tstrings, faIcons, tnpIcons);
  let fields = generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs);
  let presets = generatePresets(tstrings, faIcons, tnpIcons, searchableFieldIDs);
  let defaults = read('data/presets/defaults.json');
  let translations = generateTranslations(fields, presets, tstrings, searchableFieldIDs);
  let taginfo = generateTaginfo(presets, fields);
  let territoryLanguages = generateTerritoryLanguages();

  // Additional consistency checks
  validateCategoryPresets(categories, presets);
  validatePresetFields(presets, fields);
  validateDefaults(defaults, categories, presets);

  // Save individual data files
  let tasks = [
    writeFileProm('data/presets/categories.json', prettyStringify({ categories: categories }) ),
    writeFileProm('data/presets/fields.json', prettyStringify({ fields: fields }, { maxLength: 9999 }) ),
    writeFileProm('data/presets/presets.json', prettyStringify({ presets: presets }, { maxLength: 9999 }) ),
    writeFileProm('data/presets.yaml', translationsToYAML(translations) ),
    writeFileProm('data/taginfo.json', prettyStringify(taginfo, { maxLength: 9999 }) ),
    writeFileProm('data/territory-languages.json', prettyStringify({ dataTerritoryLanguages: territoryLanguages }, { maxLength: 9999 }) ),
    writeEnJson(tstrings),
    writeFaIcons(faIcons),
    writeTnpIcons(tnpIcons)
  ];

  return _currBuild =
    Promise.all(tasks)
    .then(() => {
      console.timeEnd(END);
      console.log('');
      _currBuild = null;
    })
    .catch((err) => {
      console.error(err);
      console.log('');
      _currBuild = null;
      process.exit(1);
    });
}


function read(f) {
  return JSON.parse(fs.readFileSync(f, 'utf8'));
}


function validate(file, instance, schema) {
  let validationErrors = jsonschema.validate(instance, schema).errors;

  if (validationErrors.length) {
    console.error(file + ': ');
    validationErrors.forEach((error) => {
      if (error.property) {
        console.error(error.property + ' ' + error.message);
      } else {
        console.error(error);
      }
    });
    console.log('');
    process.exit(1);
  }
}


function generateCategories(tstrings, faIcons, tnpIcons) {
  let categories = {};

  glob.sync(__dirname + '/data/presets/categories/*.json').forEach(file => {
    let category = read(file);
    let id = 'category-' + path.basename(file, '.json');
    tstrings.categories[id] = { name: category.name };
    categories[id] = category;

    // fontawesome icon, remember for later
    if (/^fa[srb]-/.test(category.icon)) {
      faIcons[category.icon] = {};
    }
    // noun project icon, remember for later
    if (/^tnp-/.test(category.icon)) {
      tnpIcons[category.icon] = {};
    }
  });

  return categories;
}


function generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs) {
  let fields = {};

  glob.sync(__dirname + '/data/presets/fields/**/*.json').forEach(file => {
    let field = read(file);
    let id = stripLeadingUnderscores(file.match(/presets\/fields\/([^.]*)\.json/)[1]);

    validate(file, field, fieldSchema);

    let t = tstrings.fields[id] = {
      label: field.label,
      terms: (field.terms || []).join(',')
    };

    if (field.universal) {
      searchableFieldIDs[id] = true;
    }

    if (field.placeholder) {
      t.placeholder = field.placeholder;
    }

    if (field.strings) {
      for (let i in field.strings) {
        t[i] = field.strings[i];
      }
    }

    fields[id] = field;

    // fontawesome icon, remember for later
    if (/^fa[srb]-/.test(field.icon)) {
      faIcons[field.icon] = {};
    }
    // noun project icon, remember for later
    if (/^tnp-/.test(field.icon)) {
      tnpIcons[field.icon] = {};
    }
  });

  return fields;
}


function suggestionsToPresets(presets) {
  const brands = nsi.brands.brands;
  const wikidata = nsi.wikidata.wikidata;

  Object.keys(brands).forEach(kvnd => {
    const suggestion = brands[kvnd];
    const qid = suggestion.tags['brand:wikidata'];
    if (!qid || !/^Q\d+$/.test(qid)) return;   // wikidata tag missing or looks wrong..

    const parts = kvnd.split('|', 2);
    const kv = parts[0];
    const name = parts[1].replace('~', ' ');

    let presetID, preset;

    // sometimes we can choose a more specific preset then key/value..
    if (suggestion.tags.cuisine) {
      // cuisine can contain multiple values, so try them all in order
      let cuisines = suggestion.tags.cuisine.split(';');
      for (let i = 0; i < cuisines.length; i++) {
        presetID = kv + '/' + cuisines[i].trim();
        preset = presets[presetID];
        if (preset) break;  // we matched one
      }

    } else if (suggestion.tags.vending) {
      if (suggestion.tags.vending === 'parcel_pickup;parcel_mail_in') {
        presetID = kv + '/parcel_pickup_dropoff';
      } else {
        presetID = kv + '/' + suggestion.tags.vending;
      }
      preset = presets[presetID];
    }

    // A few exceptions where the NSI tagging doesn't exactly match iD tagging..
    if (kv === 'healthcare/clinic') {
      presetID = 'amenity/clinic';
      preset = presets[presetID];
    } else if (kv === 'leisure/tanning_salon') {
      presetID = 'shop/beauty/tanning';
      preset = presets[presetID];
    }

    // fallback to key/value
    if (!preset) {
      presetID = kv;
      preset = presets[presetID];
    }

    // still no match?
    if (!preset) {
      console.log('Warning:  No preset "' + presetID + '" for name-suggestion "' + name + '"');
      return;
    }

    let suggestionID = presetID + '/' + name.replace('/', '');

    let tags = { 'brand:wikidata': qid };
    for (let k in preset.tags) {
      // prioritize suggestion tags over preset tags (for `vending`,`cuisine`, etc)
      tags[k] = suggestion.tags[k] || preset.tags[k];
    }

    // Prefer a wiki commons logo sometimes.. #6361
    const preferCommons = {
      Q177054: true,    // Burger King
      Q524757: true,    // KFC
      Q779845: true,    // CBA
      Q1205312: true    // In-N-Out
    };

    let logoURL;
    let logoURLs = wikidata[qid] && wikidata[qid].logos;
    if (logoURLs) {<commit_msg>Add Carlings to wiki commons logo list<commit_after>/* eslint-disable no-console */
const colors = require('colors/safe');
const fs = require('fs');
const glob = require('glob');
const jsonschema = require('jsonschema');
const path = require('path');
const prettyStringify = require('json-stringify-pretty-compact');
const shell = require('shelljs');
const YAML = require('js-yaml');

const fieldSchema = require('./data/presets/schema/field.json');
const presetSchema = require('./data/presets/schema/preset.json');
const nsi = require('name-suggestion-index');
const deprecated = require('./data/deprecated.json').dataDeprecated;

// fontawesome icons
const fontawesome = require('@fortawesome/fontawesome-svg-core');
const fas = require('@fortawesome/free-solid-svg-icons').fas;
const far = require('@fortawesome/free-regular-svg-icons').far;
const fab = require('@fortawesome/free-brands-svg-icons').fab;
fontawesome.library.add(fas, far, fab);

const request = require('request').defaults({ maxSockets: 1 });

let _currBuild = null;


function buildData() {
  if (_currBuild) return _currBuild;

  const START = '🏗   ' + colors.yellow('Building data...');
  const END = '👍  ' + colors.green('data built');

  console.log('');
  console.log(START);
  console.time(END);

  // Create symlinks if necessary..  { 'target': 'source' }
  const symlinks = {
    'land.html': 'dist/land.html',
    img: 'dist/img'
  };

  for (let target of Object.keys(symlinks)) {
    if (!shell.test('-L', target)) {
      console.log(`Creating symlink:  ${target} -> ${symlinks[target]}`);
      shell.ln('-sf', symlinks[target], target);
    }
  }

  // Translation strings
  let tstrings = {
    categories: {},
    fields: {},
    presets: {}
  };

  // Font Awesome icons used
  let faIcons = {
    'fas-i-cursor': {},
    'fas-lock': {},
    'fas-long-arrow-alt-right': {},
    'fas-th-list': {},
    'fas-user-cog': {}
  };

  // The Noun Project icons used
  let tnpIcons = {};

  // all fields searchable under "add field"
  let searchableFieldIDs = {};

  // Start clean
  shell.rm('-f', [
    'data/presets/categories.json',
    'data/presets/fields.json',
    'data/presets/presets.json',
    'data/presets.yaml',
    'data/taginfo.json',
    'data/territory-languages.json',
    'dist/locales/en.json',
    'svg/fontawesome/*.svg',
  ]);

  let categories = generateCategories(tstrings, faIcons, tnpIcons);
  let fields = generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs);
  let presets = generatePresets(tstrings, faIcons, tnpIcons, searchableFieldIDs);
  let defaults = read('data/presets/defaults.json');
  let translations = generateTranslations(fields, presets, tstrings, searchableFieldIDs);
  let taginfo = generateTaginfo(presets, fields);
  let territoryLanguages = generateTerritoryLanguages();

  // Additional consistency checks
  validateCategoryPresets(categories, presets);
  validatePresetFields(presets, fields);
  validateDefaults(defaults, categories, presets);

  // Save individual data files
  let tasks = [
    writeFileProm('data/presets/categories.json', prettyStringify({ categories: categories }) ),
    writeFileProm('data/presets/fields.json', prettyStringify({ fields: fields }, { maxLength: 9999 }) ),
    writeFileProm('data/presets/presets.json', prettyStringify({ presets: presets }, { maxLength: 9999 }) ),
    writeFileProm('data/presets.yaml', translationsToYAML(translations) ),
    writeFileProm('data/taginfo.json', prettyStringify(taginfo, { maxLength: 9999 }) ),
    writeFileProm('data/territory-languages.json', prettyStringify({ dataTerritoryLanguages: territoryLanguages }, { maxLength: 9999 }) ),
    writeEnJson(tstrings),
    writeFaIcons(faIcons),
    writeTnpIcons(tnpIcons)
  ];

  return _currBuild =
    Promise.all(tasks)
    .then(() => {
      console.timeEnd(END);
      console.log('');
      _currBuild = null;
    })
    .catch((err) => {
      console.error(err);
      console.log('');
      _currBuild = null;
      process.exit(1);
    });
}


function read(f) {
  return JSON.parse(fs.readFileSync(f, 'utf8'));
}


function validate(file, instance, schema) {
  let validationErrors = jsonschema.validate(instance, schema).errors;

  if (validationErrors.length) {
    console.error(file + ': ');
    validationErrors.forEach((error) => {
      if (error.property) {
        console.error(error.property + ' ' + error.message);
      } else {
        console.error(error);
      }
    });
    console.log('');
    process.exit(1);
  }
}


function generateCategories(tstrings, faIcons, tnpIcons) {
  let categories = {};

  glob.sync(__dirname + '/data/presets/categories/*.json').forEach(file => {
    let category = read(file);
    let id = 'category-' + path.basename(file, '.json');
    tstrings.categories[id] = { name: category.name };
    categories[id] = category;

    // fontawesome icon, remember for later
    if (/^fa[srb]-/.test(category.icon)) {
      faIcons[category.icon] = {};
    }
    // noun project icon, remember for later
    if (/^tnp-/.test(category.icon)) {
      tnpIcons[category.icon] = {};
    }
  });

  return categories;
}


function generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs) {
  let fields = {};

  glob.sync(__dirname + '/data/presets/fields/**/*.json').forEach(file => {
    let field = read(file);
    let id = stripLeadingUnderscores(file.match(/presets\/fields\/([^.]*)\.json/)[1]);

    validate(file, field, fieldSchema);

    let t = tstrings.fields[id] = {
      label: field.label,
      terms: (field.terms || []).join(',')
    };

    if (field.universal) {
      searchableFieldIDs[id] = true;
    }

    if (field.placeholder) {
      t.placeholder = field.placeholder;
    }

    if (field.strings) {
      for (let i in field.strings) {
        t[i] = field.strings[i];
      }
    }

    fields[id] = field;

    // fontawesome icon, remember for later
    if (/^fa[srb]-/.test(field.icon)) {
      faIcons[field.icon] = {};
    }
    // noun project icon, remember for later
    if (/^tnp-/.test(field.icon)) {
      tnpIcons[field.icon] = {};
    }
  });

  return fields;
}


function suggestionsToPresets(presets) {
  const brands = nsi.brands.brands;
  const wikidata = nsi.wikidata.wikidata;

  Object.keys(brands).forEach(kvnd => {
    const suggestion = brands[kvnd];
    const qid = suggestion.tags['brand:wikidata'];
    if (!qid || !/^Q\d+$/.test(qid)) return;   // wikidata tag missing or looks wrong..

    const parts = kvnd.split('|', 2);
    const kv = parts[0];
    const name = parts[1].replace('~', ' ');

    let presetID, preset;

    // sometimes we can choose a more specific preset then key/value..
    if (suggestion.tags.cuisine) {
      // cuisine can contain multiple values, so try them all in order
      let cuisines = suggestion.tags.cuisine.split(';');
      for (let i = 0; i < cuisines.length; i++) {
        presetID = kv + '/' + cuisines[i].trim();
        preset = presets[presetID];
        if (preset) break;  // we matched one
      }

    } else if (suggestion.tags.vending) {
      if (suggestion.tags.vending === 'parcel_pickup;parcel_mail_in') {
        presetID = kv + '/parcel_pickup_dropoff';
      } else {
        presetID = kv + '/' + suggestion.tags.vending;
      }
      preset = presets[presetID];
    }

    // A few exceptions where the NSI tagging doesn't exactly match iD tagging..
    if (kv === 'healthcare/clinic') {
      presetID = 'amenity/clinic';
      preset = presets[presetID];
    } else if (kv === 'leisure/tanning_salon') {
      presetID = 'shop/beauty/tanning';
      preset = presets[presetID];
    }

    // fallback to key/value
    if (!preset) {
      presetID = kv;
      preset = presets[presetID];
    }

    // still no match?
    if (!preset) {
      console.log('Warning:  No preset "' + presetID + '" for name-suggestion "' + name + '"');
      return;
    }

    let suggestionID = presetID + '/' + name.replace('/', '');

    let tags = { 'brand:wikidata': qid };
    for (let k in preset.tags) {
      // prioritize suggestion tags over preset tags (for `vending`,`cuisine`, etc)
      tags[k] = suggestion.tags[k] || preset.tags[k];
    }

    // Prefer a wiki commons logo sometimes.. #6361
    const preferCommons = {
      Q177054: true,    // Burger King
      Q524757: true,    // KFC
      Q779845: true,    // CBA
      Q1205312: true,   // In-N-Out
      Q10443115: true   // Carlings
    };

    let logoURL;
    let logoURLs = wikidata[qid] && wikidata[qid].logos;
    if (logoURLs) {
{'fraction_non_alphanumeric': 0.1607384325945088, 'fraction_numerical': 0.00681991886648245, 'mean_word_length': 3.071325993298229, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1246467', 'n_tokens_mistral': 5822, 'n_tokens_neox': 5587, 'n_words': 1165}