<commit_before>/**
 * @license
 * Copyright 2016 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

goog.provide('shaka.util.EbmlElement');
goog.provide('shaka.util.EbmlParser');

goog.require('shaka.util.DataViewReader');
goog.require('shaka.util.Error');
goog.require('shaka.util.Uint8ArrayUtils');


/**
 * @summary
 * Extensible Binary Markup Language (EBML) parser.
 */
shaka.util.EbmlParser = class {
  /**
   * @param {!DataView} dataView The EBML data.
   */
  constructor(dataView) {
    /** @private {!DataView} */
    this.dataView_ = dataView;

    /** @private {!shaka.util.DataViewReader} */
    this.reader_ = new shaka.util.DataViewReader(
        dataView,
        shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
  }


  /**
   * @return {boolean} True if the parser has more data, false otherwise.
   */
  hasMoreData() {
    return this.reader_.hasMoreData();
  }


  /**
   * Parses an EBML element from the parser's current position, and advances
   * the parser.
   * @return {!shaka.util.EbmlElement} The EBML element.
   * @throws {shaka.util.Error}
   * @see http://matroska.org/technical/specs/rfc/index.html
   */
  parseElement() {
    const id = this.parseId_();

    // Parse the element's size.
    const vint = this.parseVint_();
    let size;
    if (shaka.util.EbmlParser.isDynamicSizeValue_(vint)) {
      // If this has an unknown size, assume that it takes up the rest of the
      // data.
      size = this.dataView_.byteLength - this.reader_.getPosition();
    } else {
      size = shaka.util.EbmlParser.getVintValue_(vint);
    }

    // Note that if the element's size is larger than the buffer then we are
    // parsing a "partial element". This may occur if for example we are
    // parsing the beginning of some WebM container data, but our buffer does
    // not contain the entire WebM container data.
    const elementSize =
        this.reader_.getPosition() + size <= this.dataView_.byteLength ?
        size :
        this.dataView_.byteLength - this.reader_.getPosition();

    const dataView = new DataView(
        this.dataView_.buffer,
        this.dataView_.byteOffset + this.reader_.getPosition(), elementSize);

    this.reader_.skip(elementSize);

    return new shaka.util.EbmlElement(id, dataView);
  }


  /**
   * Parses an EBML ID from the parser's current position, and advances the
   * parser.
   * @throws {shaka.util.Error}
   * @return {number} The EBML ID.
   * @private
   */
  parseId_() {
    const vint = this.parseVint_();

    if (vint.length > 7) {
      throw new shaka.util.Error(
          shaka.util.Error.Severity.CRITICAL,
          shaka.util.Error.Category.MEDIA,
          shaka.util.Error.Code.EBML_OVERFLOW);
    }

    let id = 0;
    for (const /* byte */ b of vint) {
      // Note that we cannot use << since |value| may exceed 32 bits.
      id = (256 * id) + b;
    }

    return id;
  }


  /**
   * Parses a variable sized integer from the parser's current position, and
   * advances the parser.
   * For example:
   *   1 byte  wide: 1xxx xxxx
   *   2 bytes wide: 01xx xxxx xxxx xxxx
   *   3 bytes wide: 001x xxxx xxxx xxxx xxxx xxxx
   * @throws {shaka.util.Error}
   * @return {!Uint8Array} The variable sized integer.
   * @private
   */
  parseVint_() {
    const firstByte = this.reader_.readUint8();
    let numBytes;

    // Determine the byte width of the variable sized integer.
    for (numBytes = 1; numBytes <= 8; numBytes++) {
      const mask = 0x1 << (8 - numBytes);
      if (firstByte & mask) {
        break;
      }
    }

    if (numBytes > 8) {
      throw new shaka.util.Error(
          shaka.util.Error.Severity.CRITICAL,
          shaka.util.Error.Category.MEDIA,
          shaka.util.Error.Code.EBML_OVERFLOW);
    }

    const vint = new Uint8Array(numBytes);
    vint[0] = firstByte;

    // Include the remaining bytes.
    for (let i = 1; i < numBytes; i++) {<commit_msg>Use log2 to avoid a loop.<commit_after>/**
 * @license
 * Copyright 2016 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

goog.provide('shaka.util.EbmlElement');
goog.provide('shaka.util.EbmlParser');

goog.require('goog.asserts');
goog.require('shaka.util.DataViewReader');
goog.require('shaka.util.Error');
goog.require('shaka.util.Uint8ArrayUtils');


/**
 * @summary
 * Extensible Binary Markup Language (EBML) parser.
 */
shaka.util.EbmlParser = class {
  /**
   * @param {!DataView} dataView The EBML data.
   */
  constructor(dataView) {
    /** @private {!DataView} */
    this.dataView_ = dataView;

    /** @private {!shaka.util.DataViewReader} */
    this.reader_ = new shaka.util.DataViewReader(
        dataView,
        shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
  }


  /**
   * @return {boolean} True if the parser has more data, false otherwise.
   */
  hasMoreData() {
    return this.reader_.hasMoreData();
  }


  /**
   * Parses an EBML element from the parser's current position, and advances
   * the parser.
   * @return {!shaka.util.EbmlElement} The EBML element.
   * @throws {shaka.util.Error}
   * @see http://matroska.org/technical/specs/rfc/index.html
   */
  parseElement() {
    const id = this.parseId_();

    // Parse the element's size.
    const vint = this.parseVint_();
    let size;
    if (shaka.util.EbmlParser.isDynamicSizeValue_(vint)) {
      // If this has an unknown size, assume that it takes up the rest of the
      // data.
      size = this.dataView_.byteLength - this.reader_.getPosition();
    } else {
      size = shaka.util.EbmlParser.getVintValue_(vint);
    }

    // Note that if the element's size is larger than the buffer then we are
    // parsing a "partial element". This may occur if for example we are
    // parsing the beginning of some WebM container data, but our buffer does
    // not contain the entire WebM container data.
    const elementSize =
        this.reader_.getPosition() + size <= this.dataView_.byteLength ?
        size :
        this.dataView_.byteLength - this.reader_.getPosition();

    const dataView = new DataView(
        this.dataView_.buffer,
        this.dataView_.byteOffset + this.reader_.getPosition(), elementSize);

    this.reader_.skip(elementSize);

    return new shaka.util.EbmlElement(id, dataView);
  }


  /**
   * Parses an EBML ID from the parser's current position, and advances the
   * parser.
   * @throws {shaka.util.Error}
   * @return {number} The EBML ID.
   * @private
   */
  parseId_() {
    const vint = this.parseVint_();

    if (vint.length > 7) {
      throw new shaka.util.Error(
          shaka.util.Error.Severity.CRITICAL,
          shaka.util.Error.Category.MEDIA,
          shaka.util.Error.Code.EBML_OVERFLOW);
    }

    let id = 0;
    for (const /* byte */ b of vint) {
      // Note that we cannot use << since |value| may exceed 32 bits.
      id = (256 * id) + b;
    }

    return id;
  }


  /**
   * Parses a variable sized integer from the parser's current position, and
   * advances the parser.
   * For example:
   *   1 byte  wide: 1xxx xxxx
   *   2 bytes wide: 01xx xxxx xxxx xxxx
   *   3 bytes wide: 001x xxxx xxxx xxxx xxxx xxxx
   * @return {!Uint8Array} The variable sized integer.
   * @private
   */
  parseVint_() {
    const firstByte = this.reader_.readUint8();
    if (firstByte == 0) {
      throw new shaka.util.Error(
          shaka.util.Error.Severity.CRITICAL,
          shaka.util.Error.Category.MEDIA,
          shaka.util.Error.Code.EBML_OVERFLOW);
    }

    // Determine the index of the highest bit set.
    const index = Math.floor(Math.log2(firstByte));
    const numBytes = 8 - index;
    goog.asserts.assert(numBytes <= 8 && numBytes >= 1, 'Incorrect log2 value');

    const vint = new Uint8Array(numBytes);
    vint[0] = firstByte;

    // Include the remaining bytes.
    for (let i = 1; i < numBytes; i++) {
{'fraction_non_alphanumeric': 0.11640510330811787, 'fraction_numerical': 0.008242068420458394, 'mean_word_length': 2.664873810508895, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1169417', 'n_tokens_mistral': 2927, 'n_tokens_neox': 2780, 'n_words': 836}