<commit_before>/*
 * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at:
 *
 *     http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */

// Binary reader.  This is a user reader built on top of
// the IonParserBinaryRaw parser.
//
// Handles system symbols, and conversion from the parsed
// input byte array  to the desired Javascript value (scalar or
// object, such as IonValue).
import { Catalog } from "./IonCatalog";
import { Decimal } from "./IonDecimal";
import { defaultLocalSymbolTable } from "./IonLocalSymbolTable";
import { getSystemSymbolTable } from "./IonSystemSymbolTable";
import { Import } from "./IonImport";
import { ion_symbol_table_sid } from "./IonSymbols";
import { IonType } from "./IonType";
import { IonTypes } from "./IonTypes";
import { IVM } from "./IonConstants";
import { LocalSymbolTable } from "./IonLocalSymbolTable";
import { makeSymbolTable } from "./IonSymbols";
import { ParserBinaryRaw } from "./IonParserBinaryRaw";
import { Reader } from "./IonReader";
import { SharedSymbolTable } from "./IonSharedSymbolTable";
import { BinarySpan } from "./IonSpan";
import { Timestamp } from "./IonTimestamp";

const RAW_STRING = new IonType( -1, "raw_input", true,  false, false, false );

const ERROR            = -3;
const BOC              = -2;//beginning of container?
const EOF              = -1;
const TB_NULL          =  0;
const TB_BOOL          =  1;
const TB_INT           =  2;
const TB_NEG_INT       =  3;
const TB_FLOAT         =  4;
const TB_DECIMAL       =  5;
const TB_TIMESTAMP     =  6;
const TB_SYMBOL        =  7;
const TB_STRING        =  8;
const TB_CLOB          =  9;
const TB_BLOB          = 10;
const TB_SEXP          = 11;
const TB_LIST          = 12;
const TB_STRUCT        = 13;
const TB_ANNOTATION    = 14;
const TB_UNUSED__      = 15;
const TB_SEXP_CLOSE    = 21;
const TB_LIST_CLOSE    = 22;
const TB_STRUCT_CLOSE  = 23;

function get_ion_type(t: number) : IonType {
  switch(t) {
    case TB_NULL:          return IonTypes.NULL;
    case TB_BOOL:          return IonTypes.BOOL;
    case TB_INT:           return IonTypes.INT;
    case TB_NEG_INT:       return IonTypes.INT;
    case TB_FLOAT:         return IonTypes.FLOAT;
    case TB_DECIMAL:       return IonTypes.DECIMAL;
    case TB_TIMESTAMP:     return IonTypes.TIMESTAMP;
    case TB_SYMBOL:        return IonTypes.SYMBOL;
    case TB_STRING:        return IonTypes.STRING;
    case TB_CLOB:          return IonTypes.CLOB;
    case TB_BLOB:          return IonTypes.BLOB;
    case TB_SEXP:          return IonTypes.SEXP;
    case TB_LIST:          return IonTypes.LIST;
    case TB_STRUCT:        return IonTypes.STRUCT;<commit_msg>fixes tid values for TB_LIST and TB_SEXP (#257)<commit_after>/*
 * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at:
 *
 *     http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */

// Binary reader.  This is a user reader built on top of
// the IonParserBinaryRaw parser.
//
// Handles system symbols, and conversion from the parsed
// input byte array  to the desired Javascript value (scalar or
// object, such as IonValue).
import { Catalog } from "./IonCatalog";
import { Decimal } from "./IonDecimal";
import { defaultLocalSymbolTable } from "./IonLocalSymbolTable";
import { getSystemSymbolTable } from "./IonSystemSymbolTable";
import { Import } from "./IonImport";
import { ion_symbol_table_sid } from "./IonSymbols";
import { IonType } from "./IonType";
import { IonTypes } from "./IonTypes";
import { IVM } from "./IonConstants";
import { LocalSymbolTable } from "./IonLocalSymbolTable";
import { makeSymbolTable } from "./IonSymbols";
import { ParserBinaryRaw } from "./IonParserBinaryRaw";
import { Reader } from "./IonReader";
import { SharedSymbolTable } from "./IonSharedSymbolTable";
import { BinarySpan } from "./IonSpan";
import { Timestamp } from "./IonTimestamp";

const RAW_STRING = new IonType( -1, "raw_input", true,  false, false, false );

const ERROR            = -3;
const BOC              = -2;//beginning of container?
const EOF              = -1;
const TB_NULL          =  0;
const TB_BOOL          =  1;
const TB_INT           =  2;
const TB_NEG_INT       =  3;
const TB_FLOAT         =  4;
const TB_DECIMAL       =  5;
const TB_TIMESTAMP     =  6;
const TB_SYMBOL        =  7;
const TB_STRING        =  8;
const TB_CLOB          =  9;
const TB_BLOB          = 10;
const TB_LIST          = 11;
const TB_SEXP          = 12;
const TB_STRUCT        = 13;
const TB_ANNOTATION    = 14;
const TB_UNUSED__      = 15;
const TB_SEXP_CLOSE    = 21;
const TB_LIST_CLOSE    = 22;
const TB_STRUCT_CLOSE  = 23;

function get_ion_type(t: number) : IonType {
  switch(t) {
    case TB_NULL:          return IonTypes.NULL;
    case TB_BOOL:          return IonTypes.BOOL;
    case TB_INT:           return IonTypes.INT;
    case TB_NEG_INT:       return IonTypes.INT;
    case TB_FLOAT:         return IonTypes.FLOAT;
    case TB_DECIMAL:       return IonTypes.DECIMAL;
    case TB_TIMESTAMP:     return IonTypes.TIMESTAMP;
    case TB_SYMBOL:        return IonTypes.SYMBOL;
    case TB_STRING:        return IonTypes.STRING;
    case TB_CLOB:          return IonTypes.CLOB;
    case TB_BLOB:          return IonTypes.BLOB;
    case TB_LIST:          return IonTypes.LIST;
    case TB_SEXP:          return IonTypes.SEXP;
    case TB_STRUCT:        return IonTypes.STRUCT;
{'fraction_non_alphanumeric': 0.09387950270959515, 'fraction_numerical': 0.014504303474657317, 'mean_word_length': 2.6911764705882355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2449593', 'n_tokens_mistral': 2068, 'n_tokens_neox': 1889, 'n_words': 650}