<commit_before>"use strict";

const {extract, isPseudoCode} = require("./index");

/**
 * @module DataParser/ccgs
 *
 * Writes ccgs.json to stdout
 *
 * Notes:
 * - Pseudocodes are ignored
 * - "CCG" suffix is dropped from values
 */


const CODE_OFFSET = 0;
const VALUE_OFFSET = 2;

const ccgRegex = /\sCCG$/;

const transform = row => {
	const code = row[CODE_OFFSET];
	const value = row[VALUE_OFFSET];
	if (code === "CCG19CD") return []; // Escape if header
	if (isPseudoCode(code)) return [];
	return [code, value.replace(ccgRegex, "").trim()];
};

const configs = [
	{
		file: "CCG names and codes UK as at 04_19.csv",
		transform,
		parseOptions: {
			delimiter: ",",
		},
		encoding: "utf8",
	}
];

extract({ configs });<commit_msg>feat(CCG): Parse ccg files for ccg short code<commit_after>"use strict";

const {extract, isPseudoCode} = require("./index");

/**
 * @module DataParser/ccgs
 *
 * Writes ccgs.json to stdout
 *
 * Notes:
 * - Pseudocodes are ignored
 * - "CCG" suffix is dropped from values
 */


const CODE_OFFSET = 0;
const SECOND_CODE_OFFSET = 1;
const VALUE_OFFSET = 2;

const ccgRegex = /\sCCG$/;

const transform = row => {
	const code = row[CODE_OFFSET];
  const second_code = row[SECOND_CODE_OFFSET];
	const value = row[VALUE_OFFSET];
	if (code === "CCG19CD") return []; // Escape if header
	if (isPseudoCode(code)) return [];
	return [code, {
	  "code": second_code,
    "name": value.replace(ccgRegex, "").trim()
  }];
};

const configs = [
	{
		file: "CCG names and codes UK as at 04_19.csv",
		transform,
		parseOptions: {
			delimiter: ",",
		},
		encoding: "utf8",
	}
];

extract({ configs });
{'fraction_non_alphanumeric': 0.16206261510128914, 'fraction_numerical': 0.011663597298956415, 'mean_word_length': 3.7109826589595376, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095698', 'n_tokens_mistral': 627, 'n_tokens_neox': 587, 'n_words': 160}