<commit_before>/**
 * Licensed to Big Data Genomics (BDG) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The BDG licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.bdgenomics.adam.util

import java.nio.{ ByteBuffer, ByteOrder }
import com.esotericsoftware.kryo.{ Kryo, Serializer }
import com.esotericsoftware.kryo.io.{ Input, Output }
import org.bdgenomics.utils.io.{ ByteAccess, ByteArrayByteAccess }
import org.bdgenomics.adam.models._

private object TwoBitFile {
  val MAGIC_NUMBER: Int = 0x1A412743
  val BASES_PER_BYTE: Int = 4
  val BYTE_SIZE: Int = 8
  val MASK: Byte = 3 // 00000011

  // file-level byte offsets
  val VERSION_OFFSET: Int = 4
  val SEQUENCE_COUNT_OFFSET: Int = 8
  val HEADER_RESERVED_OFFSET: Int = 12
  val FILE_INDEX_OFFSET: Int = 16

  // index record-related sizes (bytes)
  val NAME_SIZE_SIZE: Int = 1
  val OFFSET_SIZE: Int = 4

  // sequence record-related sizes (bytes)
  val INT_SIZE: Int = 4
  val DNA_SIZE_SIZE: Int = INT_SIZE
  val BLOCK_COUNT_SIZE: Int = 4
  // 4-byte int for Starts array and 4-byte int for Sizes array
  val PER_BLOCK_SIZE: Int = 8
  val SEQ_RECORD_RESERVED_SIZE: Int = 4
}

/**
 * Represents a set of reference sequences backed by a .2bit file.
 *
 * See http://genome.ucsc.edu/FAQ/FAQformat.html#format7 for the spec.
 *
 * @param byteAccess ByteAccess pointing to a .2bit file.
 */
class TwoBitFile(byteAccess: ByteAccess) extends ReferenceFile {

  // load file into memory
  private[util] val bytes = ByteBuffer.wrap(byteAccess.readFully(0, byteAccess.length().toInt))
  private[util] val numSeq = readHeader()
  // hold current byte position of start of current index record
  var indexRecordStart = TwoBitFile.FILE_INDEX_OFFSET
  private[util] val seqRecords = (0 until numSeq).map(idx => {
    val (name, startIdx) = readIndexEntry(indexRecordStart)
    indexRecordStart += TwoBitFile.NAME_SIZE_SIZE + name.length + TwoBitFile.OFFSET_SIZE
    (name, TwoBitRecord(bytes, name, startIdx, idx))
  })
  private val seqRecordsMap = seqRecords.toMap

  /**
   * The sequence dictionary corresponding to the contigs in this two bit file.
   */
  val sequences = new SequenceDictionary(seqRecords.toVector
    .map(r => SequenceRecord(r._1,
      r._2.dnaSize,
      referenceIndex = Some(r._2.seqIdx))))

  private def readHeader(): Int = {
    // figure out proper byte order
    bytes.order(ByteOrder.LITTLE_ENDIAN)<commit_msg>[ADAM-1610] Mark non-serializable field in TwoBitFile as transient.<commit_after>/**
 * Licensed to Big Data Genomics (BDG) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The BDG licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.bdgenomics.adam.util

import java.nio.{ ByteBuffer, ByteOrder }
import com.esotericsoftware.kryo.{ Kryo, Serializer }
import com.esotericsoftware.kryo.io.{ Input, Output }
import org.bdgenomics.utils.io.{ ByteAccess, ByteArrayByteAccess }
import org.bdgenomics.adam.models._

private object TwoBitFile {
  val MAGIC_NUMBER: Int = 0x1A412743
  val BASES_PER_BYTE: Int = 4
  val BYTE_SIZE: Int = 8
  val MASK: Byte = 3 // 00000011

  // file-level byte offsets
  val VERSION_OFFSET: Int = 4
  val SEQUENCE_COUNT_OFFSET: Int = 8
  val HEADER_RESERVED_OFFSET: Int = 12
  val FILE_INDEX_OFFSET: Int = 16

  // index record-related sizes (bytes)
  val NAME_SIZE_SIZE: Int = 1
  val OFFSET_SIZE: Int = 4

  // sequence record-related sizes (bytes)
  val INT_SIZE: Int = 4
  val DNA_SIZE_SIZE: Int = INT_SIZE
  val BLOCK_COUNT_SIZE: Int = 4
  // 4-byte int for Starts array and 4-byte int for Sizes array
  val PER_BLOCK_SIZE: Int = 8
  val SEQ_RECORD_RESERVED_SIZE: Int = 4
}

/**
 * Represents a set of reference sequences backed by a .2bit file.
 *
 * See http://genome.ucsc.edu/FAQ/FAQformat.html#format7 for the spec.
 *
 * @param byteAccess ByteAccess pointing to a .2bit file.
 */
class TwoBitFile(byteAccess: ByteAccess) extends ReferenceFile {

  // load file into memory
  @transient private[util] val bytes = ByteBuffer.wrap(byteAccess.readFully(0, byteAccess.length().toInt))
  private[util] val numSeq = readHeader()
  // hold current byte position of start of current index record
  var indexRecordStart = TwoBitFile.FILE_INDEX_OFFSET
  private[util] val seqRecords = (0 until numSeq).map(idx => {
    val (name, startIdx) = readIndexEntry(indexRecordStart)
    indexRecordStart += TwoBitFile.NAME_SIZE_SIZE + name.length + TwoBitFile.OFFSET_SIZE
    (name, TwoBitRecord(bytes, name, startIdx, idx))
  })
  private val seqRecordsMap = seqRecords.toMap

  /**
   * The sequence dictionary corresponding to the contigs in this two bit file.
   */
  val sequences = new SequenceDictionary(seqRecords.toVector
    .map(r => SequenceRecord(r._1,
      r._2.dnaSize,
      referenceIndex = Some(r._2.seqIdx))))

  private def readHeader(): Int = {
    // figure out proper byte order
    bytes.order(ByteOrder.LITTLE_ENDIAN)
{'fraction_non_alphanumeric': 0.08605293067056341, 'fraction_numerical': 0.015262217892515018, 'mean_word_length': 3.820031298904538, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5711798', 'n_tokens_mistral': 1973, 'n_tokens_neox': 1861, 'n_words': 661}