<commit_before>import CodeGenerator from './util/codeGenerator';
import createStringEncoder from './util/stringEncoder';

const builtInNamespace = {
  i8: {
    name: 'i8',
    size: () => 1,
    encodeImpl: (value, dataView) => dataView.setInt8(value),
    decodeImpl: (dataView) => dataView.getInt8(),
    sizeCode: 'size += 1;\n',
    encodeCode: 'dataView.setInt8(#value#);\n',
    decodeCode: '#value# = dataView.getInt8();\n',
  },
  u8: {
    name: 'u8',
    size: () => 1,
    encodeImpl: (value, dataView) => dataView.setUint8(value),
    decodeImpl: (dataView) => dataView.getUint8(),
    sizeCode: 'size += 1;\n',
    encodeCode: 'dataView.setUint8(#value#);\n',
    decodeCode: '#value# = dataView.getUint8();\n',
  },
  i16: {
    name: 'i16',
    size: () => 2,
    encodeImpl: (value, dataView) => dataView.setInt16(value),
    decodeImpl: (dataView) => dataView.getInt16(),
    sizeCode: 'size += 2;\n',
    encodeCode: 'dataView.setInt16(#value#);\n',
    decodeCode: '#value# = dataView.getInt16();\n',
  },
  u16: {
    name: 'u16',
    size: () => 2,
    encodeImpl: (value, dataView) => dataView.setUint16(value),
    decodeImpl: (dataView) => dataView.getUint16(),
    sizeCode: 'size += 2;\n',
    encodeCode: 'dataView.setUint16(#value#);\n',
    decodeCode: '#value# = dataView.getUint16();\n',
  },
  i32: {
    name: 'i32',
    size: () => 4,
    encodeImpl: (value, dataView) => dataView.setInt32(value),
    decodeImpl: (dataView) => dataView.getInt32(),
    sizeCode: 'size += 4;\n',
    encodeCode: 'dataView.setInt32(#value#);\n',
    decodeCode: '#value# = dataView.getInt32();\n',
  },
  u32: {
    name: 'u32',
    size: () => 4,
    encodeImpl: (value, dataView) => dataView.setUint32(value),
    decodeImpl: (dataView) => dataView.getUint32(),
    sizeCode: 'size += 4;\n',
    encodeCode: 'dataView.setUint32(#value#);\n',
    decodeCode: '#value# = dataView.getUint32();\n',
  },
  'Array<_>': (state, generics) => {
    let type = state.resolveType(generics[0]);
    let numType = state.resolveType({ name: 'u32' });
    let codeGen = new CodeGenerator();
    // TODO Even if it's randomized, maybe it can cause a problem? Not sure yet.
    let varName = 'arraySize' + (Math.random() * 100000 | 0);
    codeGen.pushTypeDecode(varName, numType, true);
    codeGen.pushEncode(`var ${varName} = #value#.length;`);
    codeGen.pushTypeEncode(varName, numType);
    codeGen.pushDecode(`#value# = new Array(${varName});`);
    codeGen.push(`for (var i = 0; i < ${varName}; ++i) {`);
    codeGen.pushType('#value#[i]', type);
    codeGen.push('}');
    return codeGen.compile();
  },
  String: createStringEncoder(),
};
<commit_msg>Implement dynamic array nullable support<commit_after>import CodeGenerator from './util/codeGenerator';
import createStringEncoder from './util/stringEncoder';

const builtInNamespace = {
  i8: {
    name: 'i8',
    size: () => 1,
    encodeImpl: (value, dataView) => dataView.setInt8(value),
    decodeImpl: (dataView) => dataView.getInt8(),
    sizeCode: 'size += 1;\n',
    encodeCode: 'dataView.setInt8(#value#);\n',
    decodeCode: '#value# = dataView.getInt8();\n',
  },
  u8: {
    name: 'u8',
    size: () => 1,
    encodeImpl: (value, dataView) => dataView.setUint8(value),
    decodeImpl: (dataView) => dataView.getUint8(),
    sizeCode: 'size += 1;\n',
    encodeCode: 'dataView.setUint8(#value#);\n',
    decodeCode: '#value# = dataView.getUint8();\n',
  },
  i16: {
    name: 'i16',
    size: () => 2,
    encodeImpl: (value, dataView) => dataView.setInt16(value),
    decodeImpl: (dataView) => dataView.getInt16(),
    sizeCode: 'size += 2;\n',
    encodeCode: 'dataView.setInt16(#value#);\n',
    decodeCode: '#value# = dataView.getInt16();\n',
  },
  u16: {
    name: 'u16',
    size: () => 2,
    encodeImpl: (value, dataView) => dataView.setUint16(value),
    decodeImpl: (dataView) => dataView.getUint16(),
    sizeCode: 'size += 2;\n',
    encodeCode: 'dataView.setUint16(#value#);\n',
    decodeCode: '#value# = dataView.getUint16();\n',
  },
  i32: {
    name: 'i32',
    size: () => 4,
    encodeImpl: (value, dataView) => dataView.setInt32(value),
    decodeImpl: (dataView) => dataView.getInt32(),
    sizeCode: 'size += 4;\n',
    encodeCode: 'dataView.setInt32(#value#);\n',
    decodeCode: '#value# = dataView.getInt32();\n',
  },
  u32: {
    name: 'u32',
    size: () => 4,
    encodeImpl: (value, dataView) => dataView.setUint32(value),
    decodeImpl: (dataView) => dataView.getUint32(),
    sizeCode: 'size += 4;\n',
    encodeCode: 'dataView.setUint32(#value#);\n',
    decodeCode: '#value# = dataView.getUint32();\n',
  },
  'Array<_>': (state, generics) => {
    let type = state.resolveType(generics[0]);
    let numType = state.resolveType({ name: 'u32' });
    let codeGen = new CodeGenerator();
    let u8, nullFieldName;
    let nullable = generics[0].nullable;
    if (nullable) {
      // If the type is nullable, we have to use separate nullable fields to
      // spare some bits
      u8 = state.resolveType({ name: 'u8' });
      nullFieldName = 'nullCheck' + (Math.random() * 100000 | 0);
      codeGen.push(`var ${nullFieldName} = 0;`);
    }
    // TODO Even if it's randomized, maybe it can cause a problem? Not sure yet.
    let varName = 'arraySize' + (Math.random() * 100000 | 0);
    codeGen.pushTypeDecode(varName, numType, true);
    codeGen.pushEncode(`var ${varName} = #value#.length;`);
    codeGen.pushTypeEncode(varName, numType);
    codeGen.pushDecode(`#value# = new Array(${varName});`);
    codeGen.push(`for (var i = 0; i < ${varName}; ++i) {`);
    if (nullable) {
      // To match with decoding order, encoder must look ahead of the array
      // and save flags beforehand.
      codeGen.push('if (i % 8 === 0) {');
      codeGen.pushEncode(`${nullFieldName} = 0;`);
      codeGen.pushEncode(`var maxIdx = Math.min(8, ${varName} - i);`);
      codeGen.pushEncode('for (var j = 0; j < maxIdx; ++j) {');
      // Check flag / insert flag.
      codeGen.pushEncode(
        `${nullFieldName} |= #value#[i + j] == null ? 0 : (1 << j);`);
      codeGen.pushEncode('}');
      codeGen.pushType(nullFieldName, u8);
      codeGen.push('}');
      codeGen.push(`if (${nullFieldName} & (1 << (i % 8)) !== 0) {`);
    }
    codeGen.pushType('#value#[i]', type);
    if (nullable) {
      codeGen.push('}');
    }
    codeGen.push('}');
    return codeGen.compile();
  },
  String: createStringEncoder(),
};

{'fraction_non_alphanumeric': 0.19807213930348258, 'fraction_numerical': 0.029384328358208957, 'mean_word_length': 3.397129186602871, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4845885', 'n_tokens_mistral': 2396, 'n_tokens_neox': 2287, 'n_words': 481}