<commit_before>/* global Uint8Array */

import tags from './type-tags.js';
import Extendable from './extendable.js';
import type { ExtensionMap, ExtensionPoint } from './extendable.js';

type SuperPackedValue = Array<number>;
type Keyset = Array<string>;

/*
todo:
mapl, bmapl
timestamp
extension
factor out various common threads
*/

export default class Decoder extends Extendable {
  strings: Array<string>
  keysets: Array<Keyset>
  buffer : SuperPackedValue
  ptr: number

  constructor() {
    super();
    this.strings = [];
    this.keysets = [];
    this.buffer = [];
    this.ptr = 0;
  }

  static decode(buffer : SuperPackedValue, options? : { omittedKeysets? : Array<Keyset>, extensions? : ExtensionMap } = {}) : any {
    let d = new Decoder;
    if (options.extensions != null) {
      // $FlowFixMe: flow doesn't understand that ext is an ExtensionPoint
      Object.keys(options.extensions).forEach((ext : ExtensionPoint) => {
        // $FlowFixMe: flow doesn't understand that options.extensions is non-null here
        let extension = options.extensions[ext];
        d.extend(ext, extension.detector, extension.serialiser, extension.deserialiser);
      });
    }
    return d.decode(buffer, options);
  }

  decode(buffer : SuperPackedValue, options? : { omittedKeysets? : Array<Keyset> } = {}) : any {
    this.buffer = buffer;
    if (buffer[0] === tags.STRLUT) {
      this.strings = (this.decodeValue(): Array<string>);
      this.keysets = (this.decodeValue(): Array<Keyset>);
    }
    if (options.omittedKeysets != null) {<commit_msg>use a const to make flow happy<commit_after>/* global Uint8Array */

import tags from './type-tags.js';
import Extendable from './extendable.js';
import type { ExtensionMap, ExtensionPoint } from './extendable.js';

type SuperPackedValue = Array<number>;
type Keyset = Array<string>;

/*
todo:
mapl, bmapl
timestamp
extension
factor out various common threads
*/

export default class Decoder extends Extendable {
  strings: Array<string>
  keysets: Array<Keyset>
  buffer : SuperPackedValue
  ptr: number

  constructor() {
    super();
    this.strings = [];
    this.keysets = [];
    this.buffer = [];
    this.ptr = 0;
  }

  static decode(buffer : SuperPackedValue, options? : { omittedKeysets? : Array<Keyset>, extensions? : ExtensionMap } = {}) : any {
    let d = new Decoder;
    const extensions = options.extensions;
    if (extensions != null) {
      // $FlowFixMe: flow doesn't understand that ext is an ExtensionPoint
      Object.keys(extensions).forEach((ext : ExtensionPoint) => {
        let extension = extensions[ext];
        d.extend(ext, extension.detector, extension.serialiser, extension.deserialiser);
      });
    }
    return d.decode(buffer, options);
  }

  decode(buffer : SuperPackedValue, options? : { omittedKeysets? : Array<Keyset> } = {}) : any {
    this.buffer = buffer;
    if (buffer[0] === tags.STRLUT) {
      this.strings = (this.decodeValue(): Array<string>);
      this.keysets = (this.decodeValue(): Array<Keyset>);
    }
    if (options.omittedKeysets != null) {
{'fraction_non_alphanumeric': 0.13556566970091027, 'fraction_numerical': 0.0019505851755526658, 'mean_word_length': 3.4788937409024747, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5345597', 'n_tokens_mistral': 936, 'n_tokens_neox': 924, 'n_words': 240}