<commit_before>package main

import (
	"fmt"
	. "github.com/eaciit/hdc/hive"
	"time"
)

var h *Hive
var q string

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   int    `tag_name:"total_emp"`
	Salary      int    `tag_name:"salary"`
}

type SampleParse struct {
	Code        string    `tag_name:"code"`
	Description string    `tag_name:"description"`
	Total_emp   int       `tag_name:"total_emp"`
	Salary      int       `tag_name:"salary"`
	Date        time.Time `tag_name:"date"`
}

func main() {
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
	q = "select * from sample_07 limit 20;"

	//for now this function just provide  csv type
	TestParseOutput()

	//Exec Query and Process with DoSomething Function PerLine | EXECPERLINE only support for csv or tsv
	h.OutputType = "csv"
	TestExecPerLine()

	// Exec Query and wait until all line fetched | EXEC only support for csv or tsv
	h.OutputType = "tsv"
	TestExec()
}

func DoSomething(res string) {
	tmp := Sample7{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)
}

func TestExec() {
	res, e := h.Exec(q)

	if e != nil {
		fmt.Printf("error: \n%v\n", e)
	} else {
		fmt.Println(res)
	}
}

func TestExecPerLine() {
	e := h.ExecLine(q, DoSomething)

	if e != nil {
		fmt.Printf("error: \n%v\n", e)
	}
}

func TestParseOutput() {
	h.Header = []string{"code", "description", "total_emp", "salary", "Date"}

	h.OutputType = "csv"
	h.DateFormat = "YYYY-MM-DD"
	res := "'00-0000','All Occupations CSV','134354250','40690','2014-05-01'"
	tmp := SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)

	h.OutputType = "tsv"
	h.DateFormat = "YYYY-MMM-DD"
	res = "00-0000\tAll Occupations TSV\t134354250\t40690\t2014-Dec-05"
	tmp = SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)

	h.OutputType = "json"
	h.DateFormat = "YY-MMM-DD DDDD"
	res = "{ \"code\" : \"00-0000\" , \"description\" : \"All Occupations JSON\", \"total_emp\" : 134354, \"salary\" : 40690,\"Date\" : \"16-Feb-14 Sunday\" }"
	tmp = SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)
}<commit_msg>detect data type parse out<commit_after>package main

import (
	"fmt"
	. "github.com/eaciit/hdc/hive"
	"time"
)

var h *Hive
var q string

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   int    `tag_name:"total_emp"`
	Salary      int    `tag_name:"salary"`
}

type SampleParse struct {
	Code        string    `tag_name:"code"`
	Description string    `tag_name:"description"`
	Total_emp   int       `tag_name:"total_emp"`
	Salary      int       `tag_name:"salary"`
	Date        time.Time `tag_name:"date"`
}

func main() {
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@", "")
	q = "select * from sample_07 limit 20;"

	//for now this function just provide  csv type
	TestParseOutput()

	//Exec Query and Process with DoSomething Function PerLine | EXECPERLINE only support for csv or tsv
	h.OutputType = "csv"
	TestExecPerLine()

	// Exec Query and wait until all line fetched | EXEC only support for csv or tsv
	h.OutputType = "tsv"
	TestExec()
}

func DoSomething(res string) {
	tmp := Sample7{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)
}

func TestExec() {
	res, e := h.Exec(q)

	if e != nil {
		fmt.Printf("error: \n%v\n", e)
	} else {
		fmt.Println(res)
	}
}

func TestExecPerLine() {
	e := h.ExecLine(q, DoSomething)

	if e != nil {
		fmt.Printf("error: \n%v\n", e)
	}
}

func TestParseOutput() {
	h.Header = []string{"code", "description", "total_emp", "salary", "Date"}

	h.OutputType = "csv"
	h.DateFormat = "YYYY-MM-DD"
	res := "'00-0000','All Occupations CSV','134354250','40690','2014-05-01'"
	tmp := SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)

	h.OutputType = "tsv"
	h.DateFormat = "YYYY-MMM-DD"
	res = "00-0000\tAll Occupations TSV\t134354250\t40690\t2014-Dec-05"
	tmp = SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)

	h.OutputType = "json"
	res = "{ \"code\" : \"00-0000\" , \"description\" : \"All Occupations JSON\", \"total_emp\" : 134354, \"salary\" : 40690,\"Date\" : 2012-04-23T18:25:43.511Z }"
	tmp = SampleParse{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)
}
{'fraction_non_alphanumeric': 0.16769157994323558, 'fraction_numerical': 0.0489593188268685, 'mean_word_length': 3.6319824753559695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6290544', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1639, 'n_words': 384}