<commit_before>require "ledger_get_prices/version"
require 'date'
require 'open-uri'
require 'yahoo-finance'

module LedgerGetPrices

  # = Synopsis
  #
  # Tool that uses Yahoo finance to intelligently generate
  # a ledger price database based on your current ledger
  # commodities and time period.
  #
  # Ensure that you have set the +LEDGER+ and +LEDGER_PRICE_DB+
  # environment variables before proceeding. Alternatively, you
  # can make the same addition to your +.ledgerrc+, so long as
  # running +ledger+ vanilla knows where to get the journal and
  # pricedb.
  #
  # = Options
  #
  # +LEDGER_BASE_CURRENCY+: Defaults to USD, change this to your reporting currency.
  # +LEDGER_PRICE_DATE_FORMAT+: The date format of the outputted pricedb. Defaults to +%Y/%m/%d+.
  class GetPrices
    class << self

      # Yahoo finance works best in USD, if the base currency is
      # different we will also store the USD price of that currency
      # to allow for conversion.
      BASE_CURRENCY = ENV['LEDGER_BASE_CURRENCY'] || "USD"

      PRICE_DB_PATH = ENV['LEDGER_PRICE_DB'] || ENV['PRICE_HIST'] # PRICE_HIST is <v3
      DATE_FORMAT = ENV['LEDGER_PRICE_DATE_FORMAT'] || "%Y/%m/%d"
      PRICE_FORMAT = "P %{date} %{time} %{symbol} %{price}"
      COMMODITY_BLACKLIST = (ENV['LEDGER_PRICE_COMMODITY_BLACKLIST'] || '').split(" ")

      # With a bang because it does a file write.
      def run!
        File.write(PRICE_DB_PATH, new_prices.join("\n"))
      end

      # We work with the database as an array of price definitions
      # @return [Array] an array of formatted prices
      def existing_prices
        @existing_prices ||= File.read(PRICE_DB_PATH)
                                .split("\n")
                                .reject { |x| (/^P.*$/ =~ x) != 0 }
      end

      # This method builds a new price database intelligently.
      #
      # @return [Array] an array of formatted prices
      def new_prices
        commodities.reduce(existing_prices) do |db, c|
          # `|` is a shortcut for merge
          db | prices_for_symbol(c, start_date: start_date, end_date: end_date)
              .map { |x| price_string_from_result(x, symbol: c) }
        end
      end

      # @return [Array] of YahooFinance results (OpenStruct)
      def prices_for_symbol(symbol, start_date: start_date, end_date: end_date) # -> Array
        puts "Getting historical quotes for: #{symbol}"

        if COMMODITY_BLACKLIST.include?(symbol)
          puts "Skipping #{symbol}: blacklisted."
          puts "Use `LEDGER_PRICE_COMMODITY_BLACKLIST` to configure the blacklist."
          return []
        end

        result = nil
        quote_strings = possible_quote_strings(commodity: symbol)
        err = nil

        while quote_strings.length > 0 && result.nil?
          begin
            result = YahooFinance::Client.new.historical_quotes(
              quote_strings.shift, start_date: start_date, end_date: end_date, period: :daily)
          rescue OpenURI::HTTPError => e
            err = e
          end
        end

        if result.nil?
          puts "Could not get quotes from Yahoo for: #{symbol}"
          puts "It may be worthwhile getting prices for this manually."
          []
        else result
        end<commit_msg>Revert "Remove BTC from default blacklist"<commit_after>require "ledger_get_prices/version"
require 'date'
require 'open-uri'
require 'yahoo-finance'

module LedgerGetPrices

  # = Synopsis
  #
  # Tool that uses Yahoo finance to intelligently generate
  # a ledger price database based on your current ledger
  # commodities and time period.
  #
  # Ensure that you have set the +LEDGER+ and +LEDGER_PRICE_DB+
  # environment variables before proceeding. Alternatively, you
  # can make the same addition to your +.ledgerrc+, so long as
  # running +ledger+ vanilla knows where to get the journal and
  # pricedb.
  #
  # = Options
  #
  # +LEDGER_BASE_CURRENCY+: Defaults to USD, change this to your reporting currency.
  # +LEDGER_PRICE_DATE_FORMAT+: The date format of the outputted pricedb. Defaults to +%Y/%m/%d+.
  class GetPrices
    class << self

      # Yahoo finance works best in USD, if the base currency is
      # different we will also store the USD price of that currency
      # to allow for conversion.
      BASE_CURRENCY = ENV['LEDGER_BASE_CURRENCY'] || "USD"

      PRICE_DB_PATH = ENV['LEDGER_PRICE_DB'] || ENV['PRICE_HIST'] # PRICE_HIST is <v3
      DATE_FORMAT = ENV['LEDGER_PRICE_DATE_FORMAT'] || "%Y/%m/%d"
      PRICE_FORMAT = "P %{date} %{time} %{symbol} %{price}"
      COMMODITY_BLACKLIST = (ENV['LEDGER_PRICE_COMMODITY_BLACKLIST'] || 'BTC').split(" ")

      # With a bang because it does a file write.
      def run!
        File.write(PRICE_DB_PATH, new_prices.join("\n"))
      end

      # We work with the database as an array of price definitions
      # @return [Array] an array of formatted prices
      def existing_prices
        @existing_prices ||= File.read(PRICE_DB_PATH)
                                .split("\n")
                                .reject { |x| (/^P.*$/ =~ x) != 0 }
      end

      # This method builds a new price database intelligently.
      #
      # @return [Array] an array of formatted prices
      def new_prices
        commodities.reduce(existing_prices) do |db, c|
          # `|` is a shortcut for merge
          db | prices_for_symbol(c, start_date: start_date, end_date: end_date)
              .map { |x| price_string_from_result(x, symbol: c) }
        end
      end

      # @return [Array] of YahooFinance results (OpenStruct)
      def prices_for_symbol(symbol, start_date: start_date, end_date: end_date) # -> Array
        puts "Getting historical quotes for: #{symbol}"

        if COMMODITY_BLACKLIST.include?(symbol)
          puts "Skipping #{symbol}: blacklisted."
          puts "Use `LEDGER_PRICE_COMMODITY_BLACKLIST` to configure the blacklist."
          puts "BTC is included by default because yahoo doesn't provide a way to " +
              "get historical data for it."
          return []
        end

        result = nil
        quote_strings = possible_quote_strings(commodity: symbol)
        err = nil

        while quote_strings.length > 0 && result.nil?
          begin
            result = YahooFinance::Client.new.historical_quotes(
              quote_strings.shift, start_date: start_date, end_date: end_date, period: :daily)
          rescue OpenURI::HTTPError => e
            err = e
          end
        end

        if result.nil?
          puts "Could not get quotes from Yahoo for: #{symbol}"
          puts "It may be worthwhile getting prices for this manually."
          []
        else result
        end
{'fraction_non_alphanumeric': 0.09143794749403342, 'fraction_numerical': 0.0008949880668257757, 'mean_word_length': 2.4722941481097878, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2065805', 'n_tokens_mistral': 2075, 'n_tokens_neox': 2005, 'n_words': 642}