<commit_before>"use strict";

const {extract} = require("./index");

/**
 * @module DataParser/wards
 *
 * Writes wards.json to stdout
 */

const CODE_OFFSET = 0;
const VALUE_OFFSET = 1;

const transform = row => {
	const code = row[CODE_OFFSET];
	const value = row[VALUE_OFFSET];
	if (code === "WD19CD") return []; // Escape if header
	return [code, value];
};

const configs = [
	{
		file: "Ward names and codes UK as at 05_19_NSPD.csv",
		transform,
		parseOptions: {
			delimiter: ",",
		},
		encoding: "utf8",
	}
];

extract({ configs });<commit_msg>chore(GSS Codes): Update Wards<commit_after>"use strict";

const { extract } = require("./index");

/**
 * @module DataParser/wards
 *
 * Writes wards.json to stdout
 */

const CODE_OFFSET = 0;
const VALUE_OFFSET = 1;

const transform = row => {
  const code = row[CODE_OFFSET];
  const value = row[VALUE_OFFSET];
  if (code === "WD19CD") return []; // Escape if header
  return [code, value];
};

const appendMissing = {
  E05012396: "Darenth",
};

const configs = [
  {
    file: "Ward names and codes UK as at 05_19_NSPD.csv",
    transform,
    parseOptions: {
      delimiter: ",",
    },
    encoding: "utf8",
  },
];

extract({ configs });
{'fraction_non_alphanumeric': 0.16153205661948378, 'fraction_numerical': 0.02164862614487927, 'mean_word_length': 3.4684014869888475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7518356', 'n_tokens_mistral': 454, 'n_tokens_neox': 425, 'n_words': 115}