<commit_before>//
//  io.swift
//  swix
//
//  Created by <NAME> on 11/7/14.
//  Copyright (c) 2014 com.scott. All rights reserved.
//

import Foundation

// ndarray binary
func write_binary(x:ndarray, filename:String, prefix:String=S2_PREFIX){
    let N = x.n
    let data = NSData(bytes:!x, length:N*sizeof(Double))
    data.writeToFile(prefix+"../"+filename, atomically: false)
}
func read_binary(filename:String, prefix:String=S2_PREFIX) -> ndarray{
    let read = NSData(contentsOfFile: prefix+"../"+filename)
    let l:Int! = read?.length
    let sD:Int = sizeof(Double)
    let count = (l.double / sD.double)
    
    let y = zeros(count.int)
    read?.getBytes(!y, length: count.int*sizeof(Double))
    return y
}

// matrix binary
func write_binary(x:matrix, filename:String, prefix:String=S2_PREFIX){
    let y = concat(array(x.shape.0.double, x.shape.1.double), y: x.flat)
    write_binary(y, filename:filename, prefix:prefix)
}
func read_binary(filename:String, prefix:String=S2_PREFIX)->matrix{
    var a:ndarray = read_binary(filename, prefix:prefix)
    let (w, h) = (a[0], a[1])
    return reshape(a[2..<a.n], shape: (w.int,h.int))
}


// ndarray csv
func write_csv(x:ndarray, filename:String, prefix:String=S2_PREFIX){
    // write the array to CSV
    var seperator=","
    var str = ""
    for i in 0..<x.n{
        seperator = i == x.n-1 ? "" : ","
        str += String(format: "\(x[i])"+seperator)
    }
    str += "\n"
    do {
        try str.writeToFile(prefix+"../"+filename, atomically: false, encoding: NSUTF8StringEncoding)
    } catch {
        Swift.print("File probably wasn't recognized")
    }
    
}
func read_csv(filename:String, prefix:String=S2_PREFIX) -> ndarray{
    var x: String?
    do {
        x = try String(contentsOfFile: prefix+"../"+filename, encoding: NSUTF8StringEncoding)
    } catch _ {
        x = nil
    }
    var array:[Double] = []
    var columns:Int = 0
    var z = x!.componentsSeparatedByString(",")
    columns = 0
    for i in 0..<z.count{
        let num = z[i]
        array.append(num.doubleValue)
        columns += 1
    }
    var done = zeros(1 * columns)
    done.grid = array
    return done
}

// matrix csv
func read_csv(filename:String, prefix:String=S2_PREFIX) -> matrix{
    var x: String?
    do {
        x = try String(contentsOfFile: prefix+"../"+filename, encoding: NSUTF8StringEncoding)
    } catch _ {
        x = nil
    }
    var y = x!.componentsSeparatedByString("\n")
    let rows = y.count-1
    var array:[Double] = []
    var columns:Int = 0
    for i in 0..<rows{
        let z = y[i].componentsSeparatedByString(",")
        columns = 0
        for num in z{
            array.append(num.doubleValue)
            columns += 1
        }
    }
    var done = zeros((rows, columns))
    done.flat.grid = array
    return done
}
func write_csv(x:matrix, filename:String, prefix:String=S2_PREFIX){
    var seperator=","
    var str = ""
    for i in 0..<x.shape.0{
        for j in 0..<x.shape.1{
            seperator = j == x.shape.1-1 ? "" : ","
            str += String(format: "\(x[i, j])"+seperator)
        }
        str += "\n"
    }
    do {
        try str.writeToFile(prefix+"../"+filename, atomically: false, encoding: NSUTF8StringEncoding)
    } catch {
        Swift.print("File probably wasn't recognized")
    }
}


<commit_msg>Add alternative read csv function<commit_after>//
//  io.swift
//  swix
//
//  Created by <NAME> on 11/7/14.
//  Copyright (c) 2014 com.scott. All rights reserved.
//

import Foundation

// ndarray binary
func write_binary(x:ndarray, filename:String, prefix:String=S2_PREFIX){
    let N = x.n
    let data = NSData(bytes:!x, length:N*sizeof(Double))
    data.writeToFile(prefix+"../"+filename, atomically: false)
}
func read_binary(filename:String, prefix:String=S2_PREFIX) -> ndarray{
    let read = NSData(contentsOfFile: prefix+"../"+filename)
    let l:Int! = read?.length
    let sD:Int = sizeof(Double)
    let count = (l.double / sD.double)
    
    let y = zeros(count.int)
    read?.getBytes(!y, length: count.int*sizeof(Double))
    return y
}

// matrix binary
func write_binary(x:matrix, filename:String, prefix:String=S2_PREFIX){
    let y = concat(array(x.shape.0.double, x.shape.1.double), y: x.flat)
    write_binary(y, filename:filename, prefix:prefix)
}
func read_binary(filename:String, prefix:String=S2_PREFIX)->matrix{
    var a:ndarray = read_binary(filename, prefix:prefix)
    let (w, h) = (a[0], a[1])
    return reshape(a[2..<a.n], shape: (w.int,h.int))
}


// ndarray csv
func write_csv(x:ndarray, filename:String, prefix:String=S2_PREFIX){
    // write the array to CSV
    var seperator=","
    var str = ""
    for i in 0..<x.n{
        seperator = i == x.n-1 ? "" : ","
        str += String(format: "\(x[i])"+seperator)
    }
    str += "\n"
    do {
        try str.writeToFile(prefix+"../"+filename, atomically: false, encoding: NSUTF8StringEncoding)
    } catch {
        Swift.print("File probably wasn't recognized")
    }
    
}
func read_csv(filename:String, prefix:String=S2_PREFIX) -> ndarray{
    var x: String?
    do {
        x = try String(contentsOfFile: prefix+"../"+filename, encoding: NSUTF8StringEncoding)
    } catch _ {
        x = nil
    }
    var array:[Double] = []
    var columns:Int = 0
    var z = x!.componentsSeparatedByString(",")
    columns = 0
    for i in 0..<z.count{
        let num = z[i]
        array.append(num.doubleValue)
        columns += 1
    }
    var done = zeros(1 * columns)
    done.grid = array
    return done
}

// matrix csv
func read_csv(filename:String, prefix:String=S2_PREFIX) -> matrix{
    var x: String?
    do {
        x = try String(contentsOfFile: prefix+"../"+filename, encoding: NSUTF8StringEncoding)
    } catch _ {
        x = nil
    }
    var y = x!.componentsSeparatedByString("\n")
    let rows = y.count-1
    var array:[Double] = []
    var columns:Int = 0
    for i in 0..<rows{
        let z = y[i].componentsSeparatedByString(",")
        columns = 0
        for num in z{
            array.append(num.doubleValue)
            columns += 1
        }
    }
    var done = zeros((rows, columns))
    done.flat.grid = array
    return done
}
func write_csv(x:matrix, filename:String, prefix:String=S2_PREFIX){
    var seperator=","
    var str = ""
    for i in 0..<x.shape.0{
        for j in 0..<x.shape.1{
            seperator = j == x.shape.1-1 ? "" : ","
            str += String(format: "\(x[i, j])"+seperator)
        }
        str += "\n"
    }
    do {
        try str.writeToFile(prefix+"../"+filename, atomically: false, encoding: NSUTF8StringEncoding)
    } catch {
        Swift.print("File probably wasn't recognized")
    }
}


func read_csv2(filename:String,_ header:Bool=true) -> matrix{
    var x: String?
    do {
        x = try String(contentsOfFile: filename, encoding: NSUTF8StringEncoding)
    } catch _ {
        x = nil
    }
    var y = x!.componentsSeparatedByString("\n")
    let rows = y.count-1
    var array:[Double] = []
    var columns:Int = 0
    var startrow:Int = 0
    if(header==true)
    {
        startrow = 1
    }
    var factor = [String:Int]()
    var levels = Set<String>()
    var levelnum:Int = 0
    for i in startrow..<rows{
        let z = y[i].componentsSeparatedByString(",")
        columns = 0
        for num in z{
            let text=num.stringByReplacingOccurrencesOfString("\r",withString: "")
            if(Double(text) != nil)
            {
                array.append(Double(text)!)
            }
            else
            {
                if(levels.contains(text))
                {
                    array.append(Double(factor[text]!))
                }
                else{
                    factor[text]=levelnum
                    levels.insert(text)
                    levelnum=levelnum+1
                    array.append(Double(factor[text]!))
                }
            }
            columns += 1
        }
    }
    var done = zeros((rows-startrow, columns))
    done.flat.grid = array
    return done
}

{'fraction_non_alphanumeric': 0.14566488369200148, 'fraction_numerical': 0.012190570966538127, 'mean_word_length': 2.2856559051900285, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3793170', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2666, 'n_words': 613}