<commit_before>require "open-uri"

module Yomou
  module Rankapi

    class Daily < Thor
      namespace "rank daily"

      include Yomou::Helper

      desc "list", ""
      def list
        @conf = Yomou::Config.new
        date = Date.today
        url = daily_url(date)
        yaml_gz(url).each do |entry|
          printf("%3d: %3dpt %s\n",
                 entry["rank"], entry["pt"], entry["ncode"])
        end
      end

      desc "download [--since YYYYMMDD]", ""
      option :since
      def download
        @conf = Yomou::Config.new
        date = nil
        if options.has_key?("since")
          date = Date.parse(options["since"])
        else
          date = Date.new(2013, 5, 7)
        end
        while date < Date.today
          url = daily_url(date)
          path = daily_path(date)
          p url
          p path
          unless path.exist?
            save_as(url, path)
          end
          date = date.next_day
        end
      end

      private

      def daily_url(date)
        rtype = date.strftime("%Y%m%d-d")
        [
          "#{BASE_URL}/?rtype=#{rtype}",
          "gzip=#{@conf.gzip}",
          "out=#{@conf.out}"
        ].join("&")<commit_msg>Fetch only daily ranking data since 2013/05/01<commit_after>require "open-uri"

module Yomou
  module Rankapi

    class Daily < Thor
      namespace "rank daily"

      include Yomou::Helper

      desc "list", ""
      def list
        @conf = Yomou::Config.new
        date = Date.today
        url = daily_url(date)
        yaml_gz(url).each do |entry|
          printf("%3d: %3dpt %s\n",
                 entry["rank"], entry["pt"], entry["ncode"])
        end
      end

      desc "download [--since YYYYMMDD]", ""
      option :since
      def download
        @conf = Yomou::Config.new
        date = nil
        if options.has_key?("since")
          date = Date.parse(options["since"])
        else
          date = Date.new(2013, 5, 1)
        end
        while date < Date.today
          url = daily_url(date)
          path = daily_path(date)
          unless path.exist?
            if date >= Date.new(2013, 5, 1)
              p url
              p path
              save_as(url, path)
            end
          end
          date = date.next_day
        end
      end

      private

      def daily_url(date)
        rtype = date.strftime("%Y%m%d-d")
        [
          "#{BASE_URL}/?rtype=#{rtype}",
          "gzip=#{@conf.gzip}",
          "out=#{@conf.out}"
        ].join("&")
{'fraction_non_alphanumeric': 0.12744706352377147, 'fraction_numerical': 0.011985617259288853, 'mean_word_length': 1.481665014866204, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2628592', 'n_tokens_mistral': 816, 'n_tokens_neox': 774, 'n_words': 183}