<commit_before>// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"
)

const (
	euroAPI      = "http://data.fixer.io/api/latest"
	xauAPI       = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI       = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI     = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest"
	dsdAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0xbd2f0cd039e0bfcf88901c98c0bfac5ab27566e3&vs_currencies=EUR"
	dsdContract  = "0xbd2f0cd039e0bfcf88901c98c0bfac5ab27566e3"
	esdAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x36f3fd68e7325a35eb768f1aedaae9ea0689d723&vs_currencies=EUR"
	esdContract  = "0x36f3fd68e7325a35eb768f1aedaae9ea0689d723"
	fraxAPI      = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x853d955acef822db058eb8505911ed77f175b99e&vs_currencies=EUR"
	fraxContract = "0x853d955acef822db058eb8505911ed77f175b99e"
	fxsAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0&vs_currencies=EUR"
	fxsContract  = "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0"
)

var (
	// CoinMarketCap API key can be set via environment variable COINMARKETCAP_API_KEY
	coinmarketcap = os.Getenv("COINMARKETCAP_API_KEY")
	// Fixer API key can be set via environment variable FIXER_API_KEY
	fixer = os.Getenv("FIXER_API_KEY")
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{
		"Bitcoin",
		"Bitcoin Cash",
		"Bitcoin Gold",
		"Bitcoin SV",
		"Dai",
		"Dash",
		"Decred",
		"Ethereum",
		"Frax",
		"Frax Shares",
		"Grin",
		"Litecoin",
		"NEAR Protocol",
		"Monero",
		"Particl",
		"Tezos",
		"USD Coin",
		"Zcash",
	}
)

type result struct {
	symbol string
	price  float64
}

func httpGetWithWarning(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", url, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return b, err
}

func getEuroExchangeRates(api string) (map[string]interface{}, error) {
	if fixer != "" {
		api += "?access_key=" + fixer
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "&api_key=" + quandl
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return 0, err
	}
	if b == nil {
		return 0, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getDSDPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[dsdContract].(map[string]interface{}), nil
}

func getESDPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[esdContract].(map[string]interface{}), nil
}

func getFRAXPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[fraxContract].(map[string]interface{}), nil
}

func getFXSPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[fxsContract].(map[string]interface{}), nil
}

func getCoinPrices() ([]interface{}, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", coinsAPI, nil)
	if err != nil {
		return nil, err
	}

	q := url.Values{}
	q.Add("start", "1")
	q.Add("limit", "2000")
	q.Add("convert", "EUR")

	req.Header.Set("Accepts", "application/json")
	req.Header.Add("X-CMC_PRO_API_KEY", coinmarketcap)
	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", coinmarketcap, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["data"].([]interface{}), nil<commit_msg>ccprices: add DPI<commit_after>// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"
)

const (
	euroAPI      = "http://data.fixer.io/api/latest"
	xauAPI       = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI       = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI     = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest"
	dsdAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0xbd2f0cd039e0bfcf88901c98c0bfac5ab27566e3&vs_currencies=EUR"
	dsdContract  = "0xbd2f0cd039e0bfcf88901c98c0bfac5ab27566e3"
	esdAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x36f3fd68e7325a35eb768f1aedaae9ea0689d723&vs_currencies=EUR"
	esdContract  = "0x36f3fd68e7325a35eb768f1aedaae9ea0689d723"
	fraxAPI      = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x853d955acef822db058eb8505911ed77f175b99e&vs_currencies=EUR"
	fraxContract = "0x853d955acef822db058eb8505911ed77f175b99e"
	fxsAPI       = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0&vs_currencies=EUR"
	fxsContract  = "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0"
)

var (
	// CoinMarketCap API key can be set via environment variable COINMARKETCAP_API_KEY
	coinmarketcap = os.Getenv("COINMARKETCAP_API_KEY")
	// Fixer API key can be set via environment variable FIXER_API_KEY
	fixer = os.Getenv("FIXER_API_KEY")
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{
		"Bitcoin",
		"Bitcoin Cash",
		"Bitcoin Gold",
		"Bitcoin SV",
		"Dai",
		"Dash",
		"Decred",
		"DeFi Pulse Index",
		"Ethereum",
		"Grin",
		"Litecoin",
		"NEAR Protocol",
		"Monero",
		"Particl",
		"Tezos",
		"USD Coin",
		"Zcash",
	}
)

type result struct {
	symbol string
	price  float64
}

func httpGetWithWarning(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", url, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return b, err
}

func getEuroExchangeRates(api string) (map[string]interface{}, error) {
	if fixer != "" {
		api += "?access_key=" + fixer
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "&api_key=" + quandl
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return 0, err
	}
	if b == nil {
		return 0, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getDSDPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[dsdContract].(map[string]interface{}), nil
}

func getESDPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[esdContract].(map[string]interface{}), nil
}

func getFRAXPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[fraxContract].(map[string]interface{}), nil
}

func getFXSPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[fxsContract].(map[string]interface{}), nil
}

func getCoinPrices() ([]interface{}, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", coinsAPI, nil)
	if err != nil {
		return nil, err
	}

	q := url.Values{}
	q.Add("start", "1")
	q.Add("limit", "3000")
	q.Add("convert", "EUR")

	req.Header.Set("Accepts", "application/json")
	req.Header.Add("X-CMC_PRO_API_KEY", coinmarketcap)
	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", coinmarketcap, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["data"].([]interface{}), nil
{'fraction_non_alphanumeric': 0.16060203629924746, 'fraction_numerical': 0.040548915449313855, 'mean_word_length': 4.217551963048499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 14, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '778751', 'n_tokens_mistral': 4949, 'n_tokens_neox': 4569, 'n_words': 939}