<commit_before><commit_msg>cleanup script<commit_after>package main

import(
	"archive/zip"
	"bufio"
	"encoding/csv"
	"path/filepath"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var in_data_dir string
var out_data_dir string
var zips []string

type FieldInfo struct {
	name string
	idx int
}

func getOutputFile(in_csv string) string {
	return out_data_dir + "/" + filepath.Base(in_csv)
}

func discoverZips(dir string) {
	files, err := ioutil.ReadDir(dir)
	check(err)
	for _,f := range files {
		fpath := dir + "/" + f.Name()
		if f.IsDir() {
			discoverZips(fpath)
		} else if (strings.HasSuffix(f.Name(), ".zip")) {
			zips = append(zips, fpath)
		}
	}
}

func processCSV(r io.Reader, out_csv string) {
	// Create output file.
	of, err := os.Create(out_csv)
	check(err)
	w := bufio.NewWriter(of)

	var field_info = []FieldInfo {
		FieldInfo { name: "Year", idx: -1,},
		FieldInfo { name: "Month", idx: -1,},
		FieldInfo { name: "DayofMonth", idx: -1,},
		FieldInfo { name: "DayOfWeek", idx: -1,},
		FieldInfo { name: "UniqueCarrier", idx: -1,},
		FieldInfo { name: "Origin",	idx: -1,},
		FieldInfo { name: "Dest",	idx: -1,},
		FieldInfo { name: "CRSDepTime",	idx: -1,},
		FieldInfo { name: "DepDelay",	idx: -1,},
		FieldInfo { name: "ArrDelay",	idx: -1,},
		FieldInfo { name: "Cancelled",	idx: -1,},
		FieldInfo { name: "Diverted",	idx: -1,},
	}
  rdr := csv.NewReader(r)
	headers, err := rdr.Read()
	check(err)
	// Populate 'idx' in field_info members
	for f_idx := range field_info {
		f := &field_info[f_idx]
		for h_idx,h := range headers {
			if f.name == h {
				f.idx = h_idx
				break
			}
		}
		if f.idx < 0 {
			fmt.Println(headers)
			log.Fatal(f)
		}
		// Write header to output file
		if f_idx > 0 {
			w.WriteString(",")
		}
		w.WriteString(f.name)
	}
	w.WriteString("\n")
	for {
		record, err := rdr.Read()
		if err == io.EOF {
			break
		}
		check(err)
		for f_idx := range field_info {
			f := &field_info[f_idx]
			if f_idx > 0 {
				w.WriteString(",")
			}
			val := record[f.idx]
			if len(val) == 0 {
				val = "_"
			} else if strings.HasSuffix(val, ".00") {
				// Input has several <int>.00
				// Convert them to <int>
				val = val[0:len(val)-3]
			}
			w.WriteString(val)
		}
		w.WriteString("\n")
	}
	w.Flush()
	of.Close()
	log.Println("Wrote", out_csv)
}

func processZip(zp string) {
	r, err := zip.OpenReader(zp)
	check(err)
	defer r.Close()
	for _, f := range r.File {
		if strings.HasSuffix(f.Name, ".csv") {
			rc, err := f.Open()
			check(err)
			out_csv := getOutputFile(f.Name)
			processCSV(rc, out_csv)
			rc.Close()
		}
	}
}

func main() {
	in_data_dir = "/home/kamalne.singh/tmp"
	out_data_dir = "/home/kamalne.singh/tmp/clean"
	discoverZips(in_data_dir)
	log.Println(zips)
	for _,zip := range zips {
		processZip(zip)
	}
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

{'fraction_non_alphanumeric': 0.17661603673613563, 'fraction_numerical': 0.007771105616389968, 'mean_word_length': 3.1103047895500726, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2578591', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1208, 'n_words': 271}