<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License.
 *  REQUIREMENT: This definition is dependent on the @types/node definition.
 *  Install with `npm install @types/node --save-dev`
 *--------------------------------------------------------------------------------------------*/

declare module 'iconv-lite' {
	export function decode(buffer: Buffer, encoding: string, options?: Options): string;

	export function encode(content: string, encoding: string, options?: Options): Buffer;

	export function encodingExists(encoding: string): boolean;

	export function decodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream;

	export function encodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream;
}

export interface Options {
    stripBOM?: boolean;
    addBOM?: boolean;
    defaultEncoding?: string;
}
<commit_msg>Add getEncoder/getDecoder to typescript definitions. Fixes #229<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License.
 *  REQUIREMENT: This definition is dependent on the @types/node definition.
 *  Install with `npm install @types/node --save-dev`
 *--------------------------------------------------------------------------------------------*/

declare module 'iconv-lite' {
	// Basic API
	export function decode(buffer: Buffer, encoding: string, options?: Options): string;

	export function encode(content: string, encoding: string, options?: Options): Buffer;

	export function encodingExists(encoding: string): boolean;

	// Stream API
	export function decodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream;

	export function encodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream;

	// Low-level stream APIs
	export function getEncoder(encoding: string, options?: Options): EncoderStream;

	export function getDecoder(encoding: string, options?: Options): DecoderStream;
}

export interface Options {
    stripBOM?: boolean;
    addBOM?: boolean;
    defaultEncoding?: string;
}

export interface EncoderStream {
	write(str: string): Buffer;
	end(): Buffer | undefined;
}

export interface DecoderStream {
	write(buf: Buffer): string;
	end(): string | undefined;
}

{'fraction_non_alphanumeric': 0.24019409623938537, 'fraction_numerical': 0.001213101496158512, 'mean_word_length': 2.479606188466948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2703934', 'n_tokens_mistral': 580, 'n_tokens_neox': 560, 'n_words': 199}