<commit_before>// Package twse - Fetch stock data from http://www.twse.com.tw/
// 擷取台灣股市上市股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

func NewTWSE(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "tse"
	return data
}

func NewOTC(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "otc"
	return data
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, time.UTC)
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {<commit_msg>Fixed time local.<commit_after>// Package twse - Fetch stock data from http://www.twse.com.tw/
// 擷取台灣股市上市股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

func NewTWSE(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "tse"
	return data
}

func NewOTC(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "otc"
	return data
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, time.Local)
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
{'fraction_non_alphanumeric': 0.1644581164221486, 'fraction_numerical': 0.009938476100331283, 'mean_word_length': 3.2610887096774195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5076661', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1706, 'n_words': 438}