<commit_before># -*- encoding: utf-8 -*-
module EnjuNdl
  module Crd
    def self.included(base)
      base.extend ClassMethods
    end

    module ClassMethods
      def get_crd_response(options)
        params = {:query_logic => 1, :results_num => 1, :results_num => 200, :sort => 10}.merge(options)
        query = []
        query << "01_#{params[:query_01].to_s.gsub('　', ' ')}" if params[:query_01]
        query << "02_#{params[:query_02].to_s.gsub('　', ' ')}" if params[:query_02]
        delimiter = '.'
        url = "http://crd.ndl.go.jp/refapi/servlet/refapi.RSearchAPI?query=#{URI.escape(query.join(delimiter))}&query_logic=#{params[:query_logic]}&results_get_position=#{params[:results_get_position]}&results_num=#{params[:results_num]}&sort=#{params[:sort]}"

        xml = open(url).read.to_s
      end

      def search_crd(options)
        params = {:page => 1}.merge(options)
        crd_page = params[:page].to_i
        crd_page = 1 if crd_page <= 0
        crd_startrecord = (crd_page - 1) * Question.crd_per_page + 1
        params[:results_get_position] = crd_startrecord
        params[:results_num] = Question.crd_per_page

        xml = get_crd_response(params)
        doc = Nokogiri::XML(xml)

        response = {
          :results_num => doc.at('//xmlns:hit_num').try(:content).to_i,
          :results => []
        }
        doc.xpath('//xmlns:result').each do |result|
          set = {<commit_msg>fixed duplicate key<commit_after># -*- encoding: utf-8 -*-
module EnjuNdl
  module Crd
    def self.included(base)
      base.extend ClassMethods
    end

    module ClassMethods
      def get_crd_response(options)
        params = {:query_logic => 1, :results_get_position => 1, :results_num => 200, :sort => 10}.merge(options)
        query = []
        query << "01_#{params[:query_01].to_s.gsub('　', ' ')}" if params[:query_01]
        query << "02_#{params[:query_02].to_s.gsub('　', ' ')}" if params[:query_02]
        delimiter = '.'
        url = "http://crd.ndl.go.jp/refapi/servlet/refapi.RSearchAPI?query=#{URI.escape(query.join(delimiter))}&query_logic=#{params[:query_logic]}&results_get_position=#{params[:results_get_position]}&results_num=#{params[:results_num]}&sort=#{params[:sort]}"

        xml = open(url).read.to_s
      end

      def search_crd(options)
        params = {:page => 1}.merge(options)
        crd_page = params[:page].to_i
        crd_page = 1 if crd_page <= 0
        crd_startrecord = (crd_page - 1) * Question.crd_per_page + 1
        params[:results_get_position] = crd_startrecord
        params[:results_num] = Question.crd_per_page

        xml = get_crd_response(params)
        doc = Nokogiri::XML(xml)

        response = {
          :results_num => doc.at('//xmlns:hit_num').try(:content).to_i,
          :results => []
        }
        doc.xpath('//xmlns:result').each do |result|
          set = {
{'fraction_non_alphanumeric': 0.16105518917042694, 'fraction_numerical': 0.017355085039916694, 'mean_word_length': 2.8684563758389263, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5174190', 'n_tokens_mistral': 1039, 'n_tokens_neox': 982, 'n_words': 148}