package main import ( "encoding/json" "fmt" "io" "io/ioutil" "log" "net/http" "os" "strconv" "sync" "time" "flag" ) /* Data Mining Assignment - Group 5 */ type Symbols struct { Symbols []string `json:"symbols"` } type Downloader struct { symbols []string } /* * New downloader */ func NewDownloader(symbols []string) *Downloader { return &Downloader{ symbols: symbols, } } /* * Download dataset */ func (d *Downloader) Download(symbol string, wg *sync.WaitGroup) { defer wg.Done() unixTimestamp := getCurrentUnixTimestamp() endpoint := fmt.Sprintf( "https://query1.finance.yahoo.com/v7/finance/download/" + "%s?period1=1410912000&period2=%s&interval=1d&events=history&includeAdjustedClose=true", symbol, strconv.FormatInt(unixTimestamp, 10), ) resp, err := http.Get(endpoint) if err != nil { log.Printf("[ERROR] error downloading %s: %v\n", symbol, err) return } defer resp.Body.Close() filename := fmt.Sprintf("./datasets/%s.csv", symbol) file, err := os.Create(filename) if err != nil { log.Printf("[ERROR] error creating file for %s: %v\n", symbol, err) return } defer file.Close() _, err = io.Copy(file, resp.Body) if err != nil { log.Printf("[ERROR] error writing data for %s: %v\n", symbol, err) return } fmt.Printf("[SUCCESS] saved: %s\n", symbol) } /* * Get current UNIX timetamp */ func getCurrentUnixTimestamp() int64 { now := time.Now().UTC() return now.Unix() } func main() { symbols_file := flag.String("symbols-file", "default", "symbols file") flag.Parse() // jsonFile, err := os.Open("./postman/symbols.json") jsonFile, err := os.Open(*symbols_file) if err != nil { log.Fatalf("[ERROR] failed to open JSON file: %v", err) } defer jsonFile.Close() byteValue, err := ioutil.ReadAll(jsonFile) if err != nil { log.Fatalf("[ERROR] failed to read JSON file: %v", err) } var symbols Symbols if err := json.Unmarshal(byteValue, &symbols); err != nil { log.Fatalf("[ERROR] failed to unmarshal JSON: %v", err) } var wg sync.WaitGroup downloader := NewDownloader(symbols.Symbols) for _, symbol := range symbols.Symbols { wg.Add(1) go downloader.Download(symbol, &wg) } wg.Wait() }