<commit_before>#!/usr/bin/env babel-node
import request from 'superagent';
import AdmZip from 'adm-zip';
import tmp from 'tmp';
import fs from 'fs';
import http from 'http';
import knex from 'knex';


let exec_env = process.env.DB_ENV || 'development';
let root = fs.realpathSync(`${__dirname}/..`);

let Knex = knex(require(`${root}/knexfile.js`)[exec_env]);

async function countries() {
  let req = await request.get('http://download.geonames.org/export/dump/countryInfo.txt');

  let content_lines = req.text.split("\n");

  content_lines.pop();

  let countries_data = content_lines.slice(51).map((line) => {
    let country_fields = line.split("\t");
    try {
      country_fields.map((field, index) => {
        if(index === 15 || index === 17) {
          country_fields[index] = `'${JSON.stringify(field.split(','))}'`;
        } else if(!field.match(/^\d+$/)) {
          field = field.replace(/'/g, "''");
          country_fields[index] = `'${field}'`;
        }
      });

      country_fields.splice(16, 1);
      country_fields.pop();
      return country_fields;
    } catch (ex) {
      return [];
    }

  });

  let q = countries_data.map((row) => {
    return `INSERT INTO geonames_countries VALUES (${row.join(',')});`;
  }).join("\n");

  await Knex.raw(q);

}

function cities() {
  let tmp_file = tmp.fileSync();

  let output = fs.createWriteStream(tmp_file.name);

  output.on('finish', function () {
    let zip = new AdmZip(tmp_file.name);

    let content_lines = zip.readAsText("cities1000.txt").split("\n");

    let cities_data = content_lines.map((line) => {
      let city_fields = line.split("\t");
      try {
        city_fields.map((field, index) => {
          field = field.replace(/'/g, "''");
          if(index === 3) {
            city_fields[index] = `'${JSON.stringify(field.split(','))}'`;
          } else if(!field.match(/^\d+$/)) {
            city_fields[index] = `'${field}'`;
          }
        });

        return city_fields;
      } catch (ex) {
        return [];
      }

    });

    cities_data.pop();

    let q = cities_data.map((row) => {
      return `INSERT INTO geonames_cities VALUES (${row.join(',')});`;
    }).join("\n");

    Knex.raw(q)
      .then(() => {
      process.stdout.write("=== CITIES DONE ===\n");
      process.exit();
    }).catch(e => {
      console.error(e);  // eslint-disable-line no-console
      process.exit(1);
    });
  });

  request.get('http://download.geonames.org/export/dump/cities1000.zip').pipe(output);
}

countries()
  .then(() => {
    process.stdout.write("=== COUNTRIES DONE ===\n");
    
    cities();
  })
  .catch(e => {
    console.error(e);  // eslint-disable-line no-console
    process.exit(1);
  });<commit_msg>Truncate tables to allow re-imports<commit_after>#!/usr/bin/env babel-node
import request from 'superagent';
import AdmZip from 'adm-zip';
import tmp from 'tmp';
import fs from 'fs';
import http from 'http';
import knex from 'knex';


let exec_env = process.env.DB_ENV || 'development';
let root = fs.realpathSync(`${__dirname}/..`);

let Knex = knex(require(`${root}/knexfile.js`)[exec_env]);

async function countries() {
  let req = await request.get('http://download.geonames.org/export/dump/countryInfo.txt');

  let content_lines = req.text.split("\n");

  content_lines.pop();

  let countries_data = content_lines.slice(51).map((line) => {
    let country_fields = line.split("\t");
    try {
      country_fields.map((field, index) => {
        if(index === 15 || index === 17) {
          country_fields[index] = `'${JSON.stringify(field.split(','))}'`;
        } else if(!field.match(/^\d+$/)) {
          field = field.replace(/'/g, "''");
          country_fields[index] = `'${field}'`;
        }
      });

      country_fields.splice(16, 1);
      country_fields.pop();
      return country_fields;
    } catch (ex) {
      return [];
    }

  });

  let q = countries_data.map((row) => {
    return `INSERT INTO geonames_countries VALUES (${row.join(',')});`;
  }).join("\n");

  process.stdout.write("=== TRUNCATING COUNTRIES TABLE ===\n");

  await Knex.raw('TRUNCATE geonames_countries CASCADE');

  process.stdout.write("=== IMPORTING ===\n");

  await Knex.raw(q);

}

function cities() {
  let tmp_file = tmp.fileSync();

  let output = fs.createWriteStream(tmp_file.name);

  output.on('finish', function () {
    let zip = new AdmZip(tmp_file.name);

    let content_lines = zip.readAsText("cities1000.txt").split("\n");

    let cities_data = content_lines.map((line) => {
      let city_fields = line.split("\t");
      try {
        city_fields.map((field, index) => {
          field = field.replace(/'/g, "''");
          if(index === 3) {
            city_fields[index] = `'${JSON.stringify(field.split(','))}'`;
          } else if(!field.match(/^\d+$/)) {
            city_fields[index] = `'${field}'`;
          }
        });

        return city_fields;
      } catch (ex) {
        return [];
      }

    });

    cities_data.pop();

    let q = cities_data.map((row) => {
      return `INSERT INTO geonames_cities VALUES (${row.join(',')});`;
    }).join("\n");

    process.stdout.write("=== TRUNCATING CITIES TABLE ===\n");

    Knex.raw('TRUNCATE geonames_cities CASCADE').then(() => {

      process.stdout.write("=== IMPORTING ===\n");

      Knex.raw(q)
          .then(() => {
            process.stdout.write("=== CITIES DONE ===\n");
            process.exit();
          }).catch(e => {
        console.error(e);  // eslint-disable-line no-console
        process.exit(1);
      });
    });


  });

  request.get('http://download.geonames.org/export/dump/cities1000.zip').pipe(output);
}

countries()
  .then(() => {
    process.stdout.write("=== COUNTRIES DONE ===\n");

    cities();
  })
  .catch(e => {
    console.error(e);  // eslint-disable-line no-console
    process.exit(1);
  });
{'fraction_non_alphanumeric': 0.2038173142467621, 'fraction_numerical': 0.006816632583503749, 'mean_word_length': 2.688874921433061, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2533785', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1969, 'n_words': 357}