<commit_before>/*
 * Copyright (c) 2012 Crown Copyright 
 *                    Animal Health and Veterinary Laboratories Agency
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sampler.io

import java.nio.file.Path
import sampler.data.Types._
import java.io.FileOutputStream
import java.io.PrintStream
import scala.io.Source
import java.io.FileNotFoundException
import scala.reflect.Manifest
import scala.collection.immutable.WrappedString
import java.io.FileWriter

trait TableReader{
	def get[T](params: Header[T]): Column[T]
}
trait TableWriter{
	def apply(columns: Column[_]*): Unit
}

class CSVTableReader(path: Path) extends TableReader{
	
	val source = Source.fromFile(path.toString())
			.getLines()
			.map(_.split(",").map(_.trim))
			.toIterable
	
	def get[T](header: Header[T]): Column[T] = {
		val it = source.iterator
		val headers = it.next()
		
		val columnIdx = headers.indexWhere(_ == header.name) match{
			case i: Int if i<0 => throw new UnsupportedOperationException("Header not found, TODO: improve exception")
			case i => i 
		}
		
		val values = it.map(row => header.cType(row(columnIdx))).toIndexedSeq
		new Column(values, Option(header.name))(header.cType)
	}
}

class CSVTableWriter(path: Path, overwrite: Boolean = false, append: Boolean = false) extends TableWriter{
	def apply(columns: Column[_]*){
		def makeCSVLine(tokens: Iterable[String]) = {
			val newLine = System.getProperty("line.separator")
			val it = tokens.iterator
			val builder = new StringBuilder()
			it.foreach(value => {
				builder.append(value)
				if(it.hasNext) builder.append(",")
			})
			builder.append(newLine)
			builder.toString
		}
		
		val writer = new FileWriter(path.toFile)
		
		columns.map{col =>
			if(col.name.isEmpty) throw new TableWriterException("Found unnamed column")
			col.toStringColumn.values
		}.transpose.foreach{row =>
			writer.append(makeCSVLine(row))
		}
		
		writer.close
	}
}

class TableWriterException(msg: String) extends RuntimeException(msg)
class ReaderFileException(msg: String) extends RuntimeException(msg)<commit_msg>Add header line writing to csv table writer<commit_after>/*
 * Copyright (c) 2012 Crown Copyright 
 *                    Animal Health and Veterinary Laboratories Agency
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sampler.io

import java.nio.file.Path
import sampler.data.Types._
import java.io.FileOutputStream
import java.io.PrintStream
import scala.io.Source
import java.io.FileNotFoundException
import scala.reflect.Manifest
import scala.collection.immutable.WrappedString
import java.io.FileWriter

trait TableReader{
	def get[T](params: Header[T]): Column[T]
}
trait TableWriter{
	def apply(columns: Column[_]*): Unit
}

class CSVTableReader(path: Path) extends TableReader{
	
	val source = Source.fromFile(path.toString())
			.getLines()
			.map(_.split(",").map(_.trim))
			.toIterable
	
	def get[T](header: Header[T]): Column[T] = {
		val it = source.iterator
		val headers = it.next()
		
		val columnIdx = headers.indexWhere(_ == header.name) match{
			case i: Int if i<0 => throw new UnsupportedOperationException("Header not found, TODO: improve exception")
			case i => i 
		}
		
		val values = it.map(row => header.cType(row(columnIdx))).toIndexedSeq
		new Column(values, Option(header.name))(header.cType)
	}
}

class CSVTableWriter(path: Path, overwrite: Boolean = false, append: Boolean = false) extends TableWriter{
	def apply(columns: Column[_]*){
		def makeCSVLine(tokens: Iterable[String]) = {
			val newLine = System.getProperty("line.separator")
			val it = tokens.iterator
			val builder = new StringBuilder()
			it.foreach(value => {
				builder.append(value)
				if(it.hasNext) builder.append(",")
			})
			builder.append(newLine)
			builder.toString
		}
		
		val writer = new FileWriter(path.toFile)
		
		val headerLine = makeCSVLine(columns.map(col => 
			col.name.getOrElse(throw new TableWriterException("Found unnamed column"))
		))
		
		writer.append(headerLine)
		
		columns.map(col => col.toStringColumn.values).transpose.foreach{row =>
			writer.append(makeCSVLine(row))
		}
		
		writer.close
	}
}

class TableWriterException(msg: String) extends RuntimeException(msg)
class ReaderFileException(msg: String) extends RuntimeException(msg)
{'fraction_non_alphanumeric': 0.10917691579943235, 'fraction_numerical': 0.0034058656575212867, 'mean_word_length': 4.177277179236043, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '915401', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1530, 'n_words': 466}