<commit_before>package latis.reader

import scala.annotation.migration
import scala.io.Source
import scala.util.parsing.combinator.JavaTokenParsers
import latis.dm.Binary
import latis.dm.Dataset
import latis.dm.Function
import latis.dm.Index
import latis.dm.Integer
import latis.dm.Real
import latis.dm.Sample
import latis.dm.Scalar
import latis.dm.Text
import latis.dm.Tuple
import latis.dm.Variable
import latis.metadata.Metadata
import scala.collection.mutable.MapBuilder
import latis.ops.Operation
import java.net.URL

class DapReader(url: String) extends DatasetAccessor{
  
  override def close = {}
  
  lazy val dds = Source.fromURL(url + ".dds").getLines.mkString(" ")
  lazy val das = Source.fromURL(url + ".das").getLines.mkString(" ")
  lazy val data = {
    val chars = Source.fromURL(url + ".dods").drop(dds.length + "\nData:\n".length)
    //The data should be formatted in 4-byte chunks, but Source reads one byte at a time.
    //Each char read corresponds to one byte. Combine 16-bit Chars into 32-bit Ints. 
    chars.grouped(4).map(cs => (cs(0).toInt << 24) + (cs(1).toInt << 16) + (cs(2).toInt << 8) + (cs(3).toInt))
  }
  val metadata = collection.mutable.Map[String, Metadata]()
  
  lazy val origDataset = {
    DasParser.getOrigMetadata
    DdsParser.getOrigDataset
  }
  def getOrigDataset: Dataset = origDataset<commit_msg>Specify codec in DapReader<commit_after>package latis.reader

import scala.io.Codec
import scala.io.Codec.ISO8859
import scala.io.Source
import scala.util.parsing.combinator.JavaTokenParsers

import latis.dm.Binary
import latis.dm.Dataset
import latis.dm.Function
import latis.dm.Index
import latis.dm.Integer
import latis.dm.Real
import latis.dm.Sample
import latis.dm.Scalar
import latis.dm.Text
import latis.dm.Tuple
import latis.dm.Variable
import latis.metadata.Metadata
import latis.ops.Operation

class DapReader(url: String) extends DatasetAccessor {
  
  override def close = {}
  
  val codec: Codec = ISO8859
  
  lazy val dds = Source.fromURL(url + ".dds")(codec).getLines.mkString(" ")
  lazy val das = Source.fromURL(url + ".das")(codec).getLines.mkString(" ")
  lazy val data = {
    val chars = Source.fromURL(url + ".dods")(codec).drop(dds.length + "\nData:\n".length)
    //The data should be formatted in 4-byte chunks, but Source reads one byte at a time.
    //Each char read corresponds to one byte. Combine 16-bit Chars into 32-bit Ints. 
    chars.grouped(4).map(cs => (cs(0).toInt << 24) + (cs(1).toInt << 16) + (cs(2).toInt << 8) + (cs(3).toInt))
  }
  val metadata = collection.mutable.Map[String, Metadata]()
  
  lazy val origDataset = {
    DasParser.getOrigMetadata
    DdsParser.getOrigDataset
  }
  def getOrigDataset: Dataset = origDataset
{'fraction_non_alphanumeric': 0.12192434814542784, 'fraction_numerical': 0.01395519647447668, 'mean_word_length': 3.9708029197080292, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '485', 'n_tokens_mistral': 961, 'n_tokens_neox': 941, 'n_words': 232}