<commit_before>// Copyright (c) 2013 by <NAME>. All Rights Reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
//-----------------------------------------------------------------------------

package mop

import (
	`bytes`
	`fmt`
	`io/ioutil`
	`net/http`
	`reflect`
	`strings`
)

// See http://www.gummy-stuff.org/Yahoo-stocks.htm
// Also http://query.yahooapis.com/v1/public/yql
// ?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in(%22ALU%22,%22AAPL%22)
// &env=http%3A%2F%2Fstockstables.org%2Falltables.env
// &format=json'
//
// Current, Change, Open, High, Low, 52-W High, 52-W Low, Volume, AvgVolume, P/E, Yield, Market Cap.
// l1: last trade
// c6: change rt
// k2: change % rt
// o: open
// g: day's low
// h: day's high
// j: 52w low
// k: 52w high
// v: volume
// a2: avg volume
// r2: p/e rt
// r: p/e
// d: dividend/share
// y: wield
// j3: market cap rt
// j1: market cap

const quotes_url = `http://download.finance.yahoo.com/d/quotes.csv?s=%s&f=,l1c6k2oghjkva2r2rdyj3j1`

type Stock struct {
	Ticker      string
	LastTrade   string
	Change      string
	ChangePct   string
	Open        string
	Low         string
	High        string
	Low52       string
	High52      string
	Volume      string
	AvgVolume   string
	PeRatio     string
	PeRatioX    string
	Dividend    string
	Yield       string
	MarketCap   string
	MarketCapX  string
	Advancing   bool
}

type Quotes struct {
	market	    *Market
	profile	    *Profile
	stocks	    []Stock
	errors      string
}

//-----------------------------------------------------------------------------
func (self *Quotes) Initialize(market *Market, profile *Profile) *Quotes {
	self.market = market
	self.profile = profile
	self.errors = ``

	return self
}

// Fetch the latest stock quotes and parse raw fetched data into array of
// []Stock structs.
//-----------------------------------------------------------------------------
func (self *Quotes) Fetch() (this *Quotes) {
	this = self // <-- This ensures we return correct self after recover() from panic() attack.
	if self.is_ready() {
		defer func() {
			if err := recover(); err != nil {
				self.errors = fmt.Sprintf("\n\n\n\nError fetching stock quotes...\n%s", err)
			}
		}()

		url := fmt.Sprintf(quotes_url, strings.Join(self.profile.Tickers, `+`))
		response, err := http.Get(url)
		if err != nil {
			panic(err)
		}

		defer response.Body.Close()
		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			panic(err)
		}

		self.parse(sanitize(body))
	}

	return self
}

//-----------------------------------------------------------------------------
func (self *Quotes) Ok() (bool, string) {
	return self.errors == ``, self.errors
}

//-----------------------------------------------------------------------------
func (self *Quotes) AddTickers(tickers []string) (added int, err error) {
	if added, err = self.profile.AddTickers(tickers); err == nil && added > 0 {
		self.stocks = nil	// Force fetch.
	}
	return
}

//-----------------------------------------------------------------------------
func (self *Quotes) RemoveTickers(tickers []string) (removed int, err error) {
	if removed, err = self.profile.RemoveTickers(tickers); err == nil && removed > 0 {
		self.stocks = nil	// Force fetch.
	}
	return
}

// "Private" methods.

// Return true if we haven't fetched the quotes yet *or* the stock market is
// still open and we might want to grab the latest quotes. In both cases we
// make sure the list of requested tickers is not empty.
//-----------------------------------------------------------------------------
func (self *Quotes) is_ready() bool {
	return (self.stocks == nil || !self.market.IsClosed) && len(self.profile.Tickers) > 0
}


//-----------------------------------------------------------------------------
func (self *Quotes) parse(body []byte) *Quotes {
	lines := bytes.Split(body, []byte{'\n'})
	self.stocks = make([]Stock, len(lines))
	//
	// Get the total number of fields in the Stock struct. Skip the last
	// Advanicing field which is not fetched.
	//
	number_of_fields := reflect.ValueOf(self.stocks[0]).NumField() - 1
	//
	// Split each line into columns, then iterate over the Stock struct
	// fields to assign column values.
	//
	for i, line := range lines {
		columns := bytes.Split(bytes.TrimSpace(line), []byte{','})
		for j := 0; j < number_of_fields; j++ {
			// ex. self.stocks[i].Ticker = string(columns[0])
			reflect.ValueOf(&self.stocks[i]).Elem().Field(j).SetString(string(columns[j]))
		}
		//
		// Try realtime value and revert to the last known if the
		// realtime is not available.
		//
		if self.stocks[i].PeRatio == `N/A` && self.stocks[i].PeRatioX != `N/A` {
			self.stocks[i].PeRatio = self.stocks[i].PeRatioX
		}
		if self.stocks[i].MarketCap == `N/A` && self.stocks[i].MarketCapX != `N/A` {
			self.stocks[i].MarketCap = self.stocks[i].MarketCapX
		}
		//
		// Stock is advancing if the change is not negative (i.e. $0.00
		// is also "advancing").
		//
		self.stocks[i].Advancing = (self.stocks[i].Change[0:1] != `-`)
	}

	return self
}

// Utility methods.

//-----------------------------------------------------------------------------
func sanitize(body []byte) []byte {
	return bytes.Replace(bytes.TrimSpace(body), []byte{'"'}, []byte{}, -1)
}<commit_msg>First golint pass for yahoo_quotes.go<commit_after>// Copyright (c) 2013 by <NAME>. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package mop

import (
	`bytes`
	`fmt`
	`io/ioutil`
	`net/http`
	`reflect`
	`strings`
)

// See http://www.gummy-stuff.org/Yahoo-stocks.htm
//
// Also http://query.yahooapis.com/v1/public/yql
// ?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in(%22ALU%22,%22AAPL%22)
// &env=http%3A%2F%2Fstockstables.org%2Falltables.env&format=json'
//
const quotesURL = `http://download.finance.yahoo.com/d/quotes.csv?s=%s&f=,l1c6k2oghjkva2r2rdyj3j1`

type Stock struct {
	Ticker      string  // Stock ticker.
	LastTrade   string  // l1: last trade.
	Change      string  // c6: change real time.
	ChangePct   string  // k2: percent change real time.
	Open        string  // o: market open price.
	Low         string  // g: day's low.
	High        string  // h: day's high.
	Low52       string  // j: 52-weeks low.
	High52      string  // k: 52-weeks high.
	Volume      string  // v: volume.
	AvgVolume   string  // a2: average volume.
	PeRatio     string  // r2: P/E ration real time.
	PeRatioX    string  // r: P/E ration (fallback when real time is N/A).
	Dividend    string  // d: dividend.
	Yield       string  // y: dividend yield.
	MarketCap   string  // j3: market cap real time.
	MarketCapX  string  // j1: market cap (fallback when real time is N/A).
	Advancing   bool    // True when change is >= $0.
}

type Quotes struct {
	market	    *Market
	profile	    *Profile
	stocks	    []Stock
	errors      string
}

//-----------------------------------------------------------------------------
func (quotes *Quotes) Initialize(market *Market, profile *Profile) *Quotes {
	quotes.market = market
	quotes.profile = profile
	quotes.errors = ``

	return quotes
}

// Fetch the latest stock quotes and parse raw fetched data into array of
// []Stock structs.
func (quotes *Quotes) Fetch() (this *Quotes) {
	this = quotes // <-- This ensures we return correct quotes after recover() from panic() attack.
	if quotes.isReady() {
		defer func() {
			if err := recover(); err != nil {
				quotes.errors = fmt.Sprintf("\n\n\n\nError fetching stock quotes...\n%s", err)
			}
		}()

		url := fmt.Sprintf(quotesURL, strings.Join(quotes.profile.Tickers, `+`))
		response, err := http.Get(url)
		if err != nil {
			panic(err)
		}

		defer response.Body.Close()
		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			panic(err)
		}

		quotes.parse(sanitize(body))
	}

	return quotes
}

//-----------------------------------------------------------------------------
func (quotes *Quotes) Ok() (bool, string) {
	return quotes.errors == ``, quotes.errors
}

//-----------------------------------------------------------------------------
func (quotes *Quotes) AddTickers(tickers []string) (added int, err error) {
	if added, err = quotes.profile.AddTickers(tickers); err == nil && added > 0 {
		quotes.stocks = nil	// Force fetch.
	}
	return
}

//-----------------------------------------------------------------------------
func (quotes *Quotes) RemoveTickers(tickers []string) (removed int, err error) {
	if removed, err = quotes.profile.RemoveTickers(tickers); err == nil && removed > 0 {
		quotes.stocks = nil	// Force fetch.
	}
	return
}

// isReady returns true if we haven't fetched the quotes yet *or* the stock
// market is still open and we might want to grab the latest quotes. In both
// cases we make sure the list of requested tickers is not empty.
func (quotes *Quotes) isReady() bool {
	return (quotes.stocks == nil || !quotes.market.IsClosed) && len(quotes.profile.Tickers) > 0
}


//-----------------------------------------------------------------------------
func (quotes *Quotes) parse(body []byte) *Quotes {
	lines := bytes.Split(body, []byte{'\n'})
	quotes.stocks = make([]Stock, len(lines))
	//
	// Get the total number of fields in the Stock struct. Skip the last
	// Advanicing field which is not fetched.
	//
	fieldsCount := reflect.ValueOf(quotes.stocks[0]).NumField() - 1
	//
	// Split each line into columns, then iterate over the Stock struct
	// fields to assign column values.
	//
	for i, line := range lines {
		columns := bytes.Split(bytes.TrimSpace(line), []byte{','})
		for j := 0; j < fieldsCount; j++ {
			// ex. quotes.stocks[i].Ticker = string(columns[0])
			reflect.ValueOf(&quotes.stocks[i]).Elem().Field(j).SetString(string(columns[j]))
		}
		//
		// Try realtime value and revert to the last known if the
		// realtime is not available.
		//
		if quotes.stocks[i].PeRatio == `N/A` && quotes.stocks[i].PeRatioX != `N/A` {
			quotes.stocks[i].PeRatio = quotes.stocks[i].PeRatioX
		}
		if quotes.stocks[i].MarketCap == `N/A` && quotes.stocks[i].MarketCapX != `N/A` {
			quotes.stocks[i].MarketCap = quotes.stocks[i].MarketCapX
		}
		//
		// Stock is advancing if the change is not negative (i.e. $0.00
		// is also "advancing").
		//
		quotes.stocks[i].Advancing = (quotes.stocks[i].Change[0:1] != `-`)
	}

	return quotes
}

//-----------------------------------------------------------------------------
func sanitize(body []byte) []byte {
	return bytes.Replace(bytes.TrimSpace(body), []byte{'"'}, []byte{}, -1)
}
{'fraction_non_alphanumeric': 0.24581955597543695, 'fraction_numerical': 0.012564950401511572, 'mean_word_length': 2.025435838811089, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5238386', 'n_tokens_mistral': 3646, 'n_tokens_neox': 3415, 'n_words': 960}