<commit_before>
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** @author  <NAME>
  *  @version 1.2
  *  @date    Sun Aug 23 15:42:06 EDT 2015
  *  @see     LICENSE (MIT style license file).
  *
  *  An implementation supporting columnar relational databases faciltating easy
  *  and rapid analytics.  The columns in a relation are vectors from the
  *  `scalation.linalgebra` package.  Vectors and matrices may be readily extracted
  *  from a relation and feed into any of the numerous analytics techniques provided
  *  in `scalation.analytics`.  The implementation provides most of the columnar
  *  relational algebra operators given in the following paper:
  *  @see db.csail.mit.edu/projects/cstore/vldb.pdf
  *
  *  Some of the operators have unicode versions: @see `scalation.util.UnicodeTest`
  */

package scalation.relalgebra

import java.io.{FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream, PrintWriter}
import java.net.URL

import collection.immutable.StringOps
import collection.mutable.Map
import io.Source.{fromFile, fromURL}

import scalation.analytics.Regression
import scalation.linalgebra._
import scalation.linalgebra.MatrixKind._
import scalation.math.{Complex, Rational, Real}
import scalation.math.StrO._
import scalation.util.{Error, SEP}

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `Relation` companion object provides additional functions for the `Relation`
  *  class.
  */
object Relation
{
    /** Type definition for a row/tuple
      */
    type Row = Vector [Any]

    /** Relative path for database storage directory
      */
    private val DB_DIR = "store" + SEP

    /** Relative path for input/output data directory
      */
    private val DATA_DIR = "data" + SEP

    /** Filename extension for serialized relations
      *  FIX: investigate using more efficient serialization, e.g.,
      *  @see github.com/EsotericSoftware/kryo
      */
    private val SER = ".ser"

    /** Filename extension for CSV data files
      */
    private val CSV = ".csv"

    /** Filename extension for JSON data files
      */
    private val JSON = ".json"

    /** The token/element separation character (',' for CSV)
      */
    private val SP = ','

    /** Counter for making unique relation names
      */
    private var _count = 0

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Return the next count.
      */
    def count (): Int = { _count += 1; _count }

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a relation from a sequence of row/tuples.  These rows must be converted
      *  to columns.
      *  @param name     the name of the relation
      *  @param colName  the names of columns
      *  @param row      the sequence of rows to be converted to columns for the columnar relation
      *  @param key      the column number for the primary key (< 0 => no primary key)
      *  @param domain   an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      */
    def apply (name: String, colName: Seq [String], row: Seq [Row], key: Int, domain: String): Relation =
    {
        val equivCol = Vector.fill [Vec] (colName.length)(null)
        val r2 = Relation (name, colName, equivCol, key, domain)
        for (tuple <- row) r2.add (tuple)
        r2
    } // apply

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory using serialization.
      *  @param name  the name of the relation to load
      */
    def apply (name: String): Relation =
    {
        val ois = new ObjectInputStream (new FileInputStream (DB_DIR + name + SER))
        val obj = ois.readObject ()
        ois.close ()
        obj.asInstanceOf [Relation]
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      *  @param skip      the number of lines in the CSV file to skip (e.g., header line(s))
      *  @param eSep      the element separation string/regex (e.g., "," ";" " +")
      */
    def apply (fileName: String, name: String, colName: Seq [String], key: Int,
               domain: String, skip: Int, eSep: String): Relation =
    {
        var cnt    = skip
        val lines = if (fileName contains "://") fromURL(new URL(fileName)).getLines
        else fromFile (DATA_DIR + fileName).getLines
        val newCol = Vector.fill [Vec] (colName.length)(null)
        val r3     = Relation (name, colName, newCol, key, domain)
        for (ln <- lines) {
            if (cnt <= 0) r3.add (r3.row (ln.split (eSep), domain)) else cnt -= 1
        } // for
        r3
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      *  @param skip      the number of lines in the CSV file to skip (e.g., header line(s))
      *  @param eSep      the element separation string/regex (e.g., "," ";" " +")
      */
    def apply (fileName: String, name: String, key: Int,
               domain: String, eSep: String): Relation =
    {
        var first = true
        val lines = if (fileName contains "://") fromURL(new URL(fileName)).getLines
        else fromFile (DATA_DIR + fileName).getLines
        var r3: Relation = null
        for (ln <- lines) {
            if (first) {
                val colName = ln.split (eSep)
                val newCol = Vector.fill [Vec] (colName.length)(null)
                r3 = Relation (name, colName, newCol, key, domain)
                first = false
            }
            else r3.add (r3.row (ln.split (eSep), domain))
        } // for
        r3
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.  This version assumes
      *  defaults for 'eSep' and 'skip' of ("," and 0).
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      */
    def apply (fileName: String, name: String, colName: Seq [String], key: Int,
               domain: String): Relation =
    {
        var eSep   = ","
        val lines  = fromFile (DATA_DIR + fileName).getLines
        val newCol = Vector.fill [Vec] (colName.length)(null)
        val r3     = Relation (name, colName, newCol, key, domain)
        for (ln <- lines) r3.add (r3.row (ln.split (eSep), domain))<commit_msg>Scripted auto-commit on change (2015-09-28 16:45:00) by gitwatch.sh<commit_after>
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** @author  <NAME>
  *  @version 1.2
  *  @date    Sun Aug 23 15:42:06 EDT 2015
  *  @see     LICENSE (MIT style license file).
  *
  *  An implementation supporting columnar relational databases faciltating easy
  *  and rapid analytics.  The columns in a relation are vectors from the
  *  `scalation.linalgebra` package.  Vectors and matrices may be readily extracted
  *  from a relation and feed into any of the numerous analytics techniques provided
  *  in `scalation.analytics`.  The implementation provides most of the columnar
  *  relational algebra operators given in the following paper:
  *  @see db.csail.mit.edu/projects/cstore/vldb.pdf
  *
  *  Some of the operators have unicode versions: @see `scalation.util.UnicodeTest`
  */

package scalation.relalgebra

import java.io.{FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream, PrintWriter}
import java.net.URL

import collection.immutable.StringOps
import collection.mutable.Map
import io.Source.{fromFile, fromURL}

import scalation.analytics.Regression
import scalation.linalgebra._
import scalation.linalgebra.MatrixKind._
import scalation.math.{Complex, Rational, Real}
import scalation.math.StrO._
import scalation.util.{Error, SEP}

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `Relation` companion object provides additional functions for the `Relation`
  *  class.
  */
object Relation
{
    /** Type definition for a row/tuple
      */
    type Row = Vector [Any]

    /** Relative path for database storage directory
      */
    private val DB_DIR = "store" + SEP

    /** Relative path for input/output data directory
      */
    private val DATA_DIR = "data" + SEP

    /** Filename extension for serialized relations
      *  FIX: investigate using more efficient serialization, e.g.,
      *  @see github.com/EsotericSoftware/kryo
      */
    private val SER = ".ser"

    /** Filename extension for CSV data files
      */
    private val CSV = ".csv"

    /** Filename extension for JSON data files
      */
    private val JSON = ".json"

    /** The token/element separation character (',' for CSV)
      */
    private val SP = ','

    /** Counter for making unique relation names
      */
    private var _count = 0

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Return the next count.
      */
    def count (): Int = { _count += 1; _count }

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a relation from a sequence of row/tuples.  These rows must be converted
      *  to columns.
      *  @param name     the name of the relation
      *  @param colName  the names of columns
      *  @param row      the sequence of rows to be converted to columns for the columnar relation
      *  @param key      the column number for the primary key (< 0 => no primary key)
      *  @param domain   an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      */
    def apply (name: String, colName: Seq [String], row: Seq [Row], key: Int, domain: String): Relation =
    {
        val equivCol = Vector.fill [Vec] (colName.length)(null)
        val r2 = Relation (name, colName, equivCol, key, domain)
        for (tuple <- row) r2.add (tuple)
        r2
    } // apply

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory using serialization.
      *  @param name  the name of the relation to load
      */
    def apply (name: String): Relation =
    {
        val ois = new ObjectInputStream (new FileInputStream (DB_DIR + name + SER))
        val obj = ois.readObject ()
        ois.close ()
        obj.asInstanceOf [Relation]
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      *  @param skip      the number of lines in the CSV file to skip (e.g., header line(s))
      *  @param eSep      the element separation string/regex (e.g., "," ";" " +")
      */
    def apply (fileName: String, name: String, colName: Seq [String], key: Int,
               domain: String, skip: Int, eSep: String): Relation =
    {
        var cnt    = skip
        val lines = if (fileName contains "://") fromURL(new URL(fileName)).getLines
        else fromFile (DATA_DIR + fileName).getLines
        val newCol = Vector.fill [Vec] (colName.length)(null)
        val r3     = Relation (name, colName, newCol, key, domain)
        for (ln <- lines) {
            if (cnt <= 0) r3.add (r3.row (ln.split (eSep), domain)) else cnt -= 1
        } // for
        r3
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      *  @param skip      the number of lines in the CSV file to skip (e.g., header line(s))
      *  @param eSep      the element separation string/regex (e.g., "," ";" " +")
      */
    def apply (fileName: String, name: String, key: Int,
               domain: String, eSep: String): Relation =
    {
        var first = true
        val lines = fromURL(new URL(fileName)).getLines
//        else fromFile (DATA_DIR + fileName).getLines
        var r3: Relation = null
        for (ln <- lines) {
            if (first) {
                val colName = ln.split (eSep)
                val newCol = Vector.fill [Vec] (colName.length)(null)
                r3 = Relation (name, colName, newCol, key, domain)
                first = false
            }
            else r3.add (r3.row (ln.split (eSep), domain))
        } // for
        r3
    } // apply

    //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Read the relation with the given 'name' into memory loading its columns
      *  with data from the CSV file named 'fileName'.  This version assumes
      *  defaults for 'eSep' and 'skip' of ("," and 0).
      *  @param fileName  the file name of the data file
      *  @param name      the name of the relation
      *  @param colName   the names of columns
      *  @param key       the column number for the primary key (< 0 => no primary key)
      *  @param domain    an optional string indicating domains for columns (e.g., SD = StrNum, Double)
      */
    def apply (fileName: String, name: String, colName: Seq [String], key: Int,
               domain: String): Relation =
    {
        var eSep   = ","
        val lines  = fromFile (DATA_DIR + fileName).getLines
        val newCol = Vector.fill [Vec] (colName.length)(null)
        val r3     = Relation (name, colName, newCol, key, domain)
        for (ln <- lines) r3.add (r3.row (ln.split (eSep), domain))
{'fraction_non_alphanumeric': 0.16907653749759013, 'fraction_numerical': 0.00578368999421631, 'mean_word_length': 2.527198549410698, 'pattern_counts': {'":': 3, '<': 23, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2007366', 'n_tokens_mistral': 4219, 'n_tokens_neox': 4118, 'n_words': 1607}