<commit_before>const request = require('superagent')
const decompress = require('decompress')
const parse = require('csv-parser')
const pumpify = require('pumpify').obj
const through = require('through2').obj
const {decodeStream} = require('iconv-lite')
const toArray = require('get-stream').array
const {trimStart, trimEnd, keyBy} = require('lodash')
const {bufferToStream} = require('./buffer-stream')

const COMMUNES_2017_URL = 'https://www.insee.fr/fr/statistiques/fichier/2666684/france2017-txt.zip'

async function getCommunesBuffer() {
  const response = await request.get(COMMUNES_2017_URL)
    .buffer(true)
    .parse(request.parse['application/octet-stream'])
  const archiveBuffer = response.body
  const decompressedFiles = await decompress(archiveBuffer)
  return decompressedFiles[0].data
}

function buildNom(NCCENR, ARTMIN) {
  if (!ARTMIN) return NCCENR
  const trimmedArticle = trimStart(trimEnd(ARTMIN, ')'), '(')
  return trimmedArticle.endsWith('\'') ? trimmedArticle + NCCENR : trimmedArticle + ' ' + NCCENR
}

function eachCommune(commune, enc, cb) {
  const {ACTUAL, DEP, COM, NCCENR, ARTMIN} = commune
  if (ACTUAL === '9') return cb() // On ignore les fractions cantonales
  const code = DEP + COM
  const nom = buildNom(NCCENR, ARTMIN)
  cb(null, {code, nom, status: ACTUAL})
}

async function getCommunes() {
  const communesBuffer = await getCommunesBuffer()
  return toArray(
    pumpify(
      bufferToStream(communesBuffer),
      decodeStream('win1252'),
      parse({separator: '\t', strict: true}),
      through(eachCommune)
    )
  )<commit_msg>COG 2018<commit_after>const request = require('superagent')
const decompress = require('decompress')
const parse = require('csv-parser')
const pumpify = require('pumpify').obj
const through = require('through2').obj
const {decodeStream} = require('iconv-lite')
const toArray = require('get-stream').array
const {trimStart, trimEnd, keyBy} = require('lodash')
const {bufferToStream} = require('./buffer-stream')

const COMMUNES_2018_URL = 'https://www.insee.fr/fr/statistiques/fichier/3363419/France2018-txt.zip'

async function getCommunesBuffer() {
  const response = await request.get(COMMUNES_2018_URL)
    .buffer(true)
    .parse(request.parse['application/octet-stream'])
  const archiveBuffer = response.body
  const decompressedFiles = await decompress(archiveBuffer)
  return decompressedFiles[0].data
}

function buildNom(NCCENR, ARTMIN) {
  if (!ARTMIN) return NCCENR
  const trimmedArticle = trimStart(trimEnd(ARTMIN, ')'), '(')
  return trimmedArticle.endsWith('\'') ? trimmedArticle + NCCENR : trimmedArticle + ' ' + NCCENR
}

function eachCommune(commune, enc, cb) {
  const {ACTUAL, DEP, COM, NCCENR, ARTMIN} = commune
  if (ACTUAL === '9') return cb() // On ignore les fractions cantonales
  const code = DEP + COM
  const nom = buildNom(NCCENR, ARTMIN)
  cb(null, {code, nom, status: ACTUAL})
}

async function getCommunes() {
  const communesBuffer = await getCommunesBuffer()
  return toArray(
    pumpify(
      bufferToStream(communesBuffer),
      decodeStream('win1252'),
      parse({separator: '\t', strict: true}),
      through(eachCommune)
    )
  )
{'fraction_non_alphanumeric': 0.1347674786459981, 'fraction_numerical': 0.017715912685858905, 'mean_word_length': 4.988636363636363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '595118', 'n_tokens_mistral': 1079, 'n_tokens_neox': 1010, 'n_words': 235}