<commit_before>require "open-uri"

module Yomou
  module Rankapi

    class Rank < Thor

      private

      def weekly
        date = Date.new(2013, 5, 7)
        while date < Date.today
          rtype = date.strftime("%Y%m%d-w")
          path = Pathname.new(File.join(@conf.directory, "rankapi",
                                        date.strftime("weekly/%Y/#{rtype}.yaml.gz")))
          url = [
            "#{BASE_URL}/?rtype=#{rtype}",
            "gzip=#{@conf.gzip}",
            "out=#{@conf.out}"
          ].join("&")
          FileUtils.mkdir_p(path.dirname)
          if date.tuesday?
            p url
            p path.to_s
            open(url) do |context|
              File.open(path.to_s, "w+") do |file|
                file.puts(context.read)
              end
            end
          end
          date = date.next_day(7)
        end
      end

    end
  end
end<commit_msg>Rewrite weekly list,download command<commit_after>require "open-uri"

module Yomou
  module Rankapi

    class Weekly < Thor

      include Yomou::Helper

      desc "list", ""
      def list
        @conf = Yomou::Config.new
        date = Date.today
        until date.tuesday?
          date = date.prev_day(1)
        end
        url = weekly_url(date)
        yaml_gz(url).each do |entry|
          printf("%3d: %3dpt %s\n",
                 entry["rank"], entry["pt"], entry["ncode"])
        end
      end

      desc "download [OPTIONS]", ""
      option :since
      def download
        @conf = Yomou::Config.new
        date = nil
        if options.has_key?("since")
          date = Date.parse(options["since"])
          until date.tuesday?
            date = date.next_day(1)
          end
        else
          date = Date.new(2013, 5, 7)
        end
        while date < Date.today
          rtype = date.strftime("%Y%m%d-w")
          url = weekly_url(date)
          path = weekly_path(date)
          p url
          p path
          save_as(url, path)
          date = date.next_day(7)
        end
      end

      private

      def weekly_url(date)
        rtype = date.strftime("%Y%m%d-w")
        [
          "#{BASE_URL}/?rtype=#{rtype}",
          "gzip=#{@conf.gzip}",
          "out=#{@conf.out}"
        ].join("&")
      end

      def weekly_path(date)
        rtype = date.strftime("%Y%m%d-w")
        path = Pathname.new(File.join(@conf.directory, "rankapi",
                                      date.strftime("weekly/%Y/#{rtype}.yaml.gz")))
        FileUtils.mkdir_p(path.dirname)
        path
      end

    end
  end
end
{'fraction_non_alphanumeric': 0.13021442495126706, 'fraction_numerical': 0.007017543859649123, 'mean_word_length': 1.4345351043643264, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4092552', 'n_tokens_mistral': 842, 'n_tokens_neox': 804, 'n_words': 163}