<commit_before>import Chunk from '../chunked/chunk';

class MCAL {

  static ALPHA_MAP_SIZE = 4096;
  static ALPHA_MAP_HALF_SIZE = 2048;

  constructor() {
    this.chunk = Chunk();
  }

  decode(stream, parent) {
    const data = this.chunk.decode(stream, parent);

    // Skip the fully opaque initial layer
    const layers = parent.MCLY.layers.slice(1);
    data.alphaMaps = layers.map((layer) => {
      if (layer.compressed) {
        return this.decodeCompressed(stream);
      }
      return this.decodeUncompressed(stream, parent);
    });

    return data;
  }

  decodeCompressed(stream) {
    const size = this.constructor.ALPHA_MAP_SIZE;
    const buffer = stream.buffer;
    const alpha = new Buffer(size);

    let writePos = 0;
    while (writePos < size) {
      const fill = buffer[stream.pos] & 0x80;
      const count = buffer[stream.pos] & 0x7F;
      stream.pos++;
      for (let i = 0; i < count; ++i) {
        if (writePos === size) {
          break;
        }
        alpha[writePos] = buffer[stream.pos];
        writePos++;
        if (!fill) {
          stream.pos++;
        }
      }
      if (fill) {
        stream.pos++;
      }
    }

    return alpha;
  }

  decodeUncompressed(stream, parent) {
    const wdtFlags = parent.parent.wdtFlags;
    if (wdtFlags & 0x4 || wdtFlags & 0x80) {
      return stream.readBuffer(this.constructor.ALPHA_MAP_SIZE);
    }
    const size = this.constructor.ALPHA_MAP_HALF_SIZE;
    const buffer = stream.readBuffer(size);
    const alpha = new Buffer(this.constructor.ALPHA_MAP_SIZE);

    for (let i = 0; i < size; ++i) {
      const value = buffer[i];
      const offset = i * 2;
      alpha[offset] = (value >>> 4) * 17;
      alpha[offset + 1] = (value & 0x0F) * 17;
    }

    return alpha;<commit_msg>Correct 2048 alpha map decoding<commit_after>import Chunk from '../chunked/chunk';

class MCAL {

  static ALPHA_MAP_SIZE = 4096;
  static ALPHA_MAP_HALF_SIZE = 2048;

  constructor() {
    this.chunk = Chunk();
  }

  decode(stream, parent) {
    const data = this.chunk.decode(stream, parent);

    // Skip the fully opaque initial layer
    const layers = parent.MCLY.layers.slice(1);
    data.alphaMaps = layers.map((layer) => {
      if (layer.compressed) {
        return this.decodeCompressed(stream);
      }
      return this.decodeUncompressed(stream, parent);
    });

    return data;
  }

  decodeCompressed(stream) {
    const size = this.constructor.ALPHA_MAP_SIZE;
    const buffer = stream.buffer;
    const alpha = new Buffer(size);

    let writePos = 0;
    while (writePos < size) {
      const fill = buffer[stream.pos] & 0x80;
      const count = buffer[stream.pos] & 0x7F;
      stream.pos++;
      for (let i = 0; i < count; ++i) {
        if (writePos === size) {
          break;
        }
        alpha[writePos] = buffer[stream.pos];
        writePos++;
        if (!fill) {
          stream.pos++;
        }
      }
      if (fill) {
        stream.pos++;
      }
    }

    return alpha;
  }

  decodeUncompressed(stream, parent) {
    const wdtFlags = parent.parent.wdtFlags;
    if (wdtFlags & 0x4 || wdtFlags & 0x80) {
      return stream.readBuffer(this.constructor.ALPHA_MAP_SIZE);
    }
    const size = this.constructor.ALPHA_MAP_HALF_SIZE;
    const buffer = stream.readBuffer(size);
    const alpha = new Buffer(this.constructor.ALPHA_MAP_SIZE);

    for (let i = 0; i < size; ++i) {
      const value = buffer[i];
      const offset = i * 2;
      alpha[offset] = (value & 0x0F) * 17;
      alpha[offset + 1] = (value >> 4) * 17;
    }

    return alpha;
{'fraction_non_alphanumeric': 0.12846347607052896, 'fraction_numerical': 0.018471872376154493, 'mean_word_length': 2.3370681605975725, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '615766', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1170, 'n_words': 275}