<commit_before>module GoogleApis
  class Api
    class BigQuery
      extend Base

      api "bigquery"
      version 2
      auth_scope "https://www.googleapis.com/auth/bigquery"
      default_parameters :projectId, :datasetId

      def project
        default_params[:projectId]
      end

      def dataset
        default_params[:datasetId]
      end

      def select_rows(statement)
        result = jobs.query :query => statement
        types = result["schema"]["fields"].collect{|x| x["type"].downcase.to_sym}

        (result["rows"] || []).collect do |row|
          row["f"].inject([]) do |values, x|
            values << parse_value(types[values.size], x["v"])
          end
        end
      end

      def select_values(query)
        select_rows(query).collect{|x| x[0]}
      end

      def select_value(query)
        row = select_rows(query)[0]
        row[0] if row
      end

    private

      def parse_value(type, value)
        unless value == "NULL"
          case type
          when :string
            parse_string_value value
          when :integer
            parse_integer_value value
          when :float
            parse_float_value value
          when :boolean
            parse_boolean_value value
          when :timestamp
            parse_timestamp_value value
          else
            raise NotImplementedError, "Cannot parse value of type #{type.inspect}"
          end
        end
      end

      def parse_string_value(value)
        value
      end

      def parse_integer_value(value)
        value.to_i
      end

      def parse_float_value(value)
        value.to_f
      end

      def parse_boolean_value(value)
        (value == "true") || (value == "1")
      end

      def parse_timestamp_value(value)
        Time.at value.to_f.to_i
      end

    end
  end
end
<commit_msg>Handling Google BigQuery NULL values correctly<commit_after>module GoogleApis
  class Api
    class BigQuery
      extend Base

      api "bigquery"
      version 2
      auth_scope "https://www.googleapis.com/auth/bigquery"
      default_parameters :projectId, :datasetId

      def project
        default_params[:projectId]
      end

      def dataset
        default_params[:datasetId]
      end

      def select_rows(statement)
        result = jobs.query :query => statement
        types = result["schema"]["fields"].collect{|x| x["type"].downcase.to_sym}

        (result["rows"] || []).collect do |row|
          row["f"].inject([]) do |values, x|
            values << parse_value(types[values.size], x["v"])
          end
        end
      end

      def select_values(query)
        select_rows(query).collect{|x| x[0]}
      end

      def select_value(query)
        row = select_rows(query)[0]
        row[0] if row
      end

    private

      def parse_value(type, value)
        unless [nil, "NULL"].include?(value)
          case type
          when :string
            parse_string_value value
          when :integer
            parse_integer_value value
          when :float
            parse_float_value value
          when :boolean
            parse_boolean_value value
          when :timestamp
            parse_timestamp_value value
          else
            raise NotImplementedError, "Cannot parse value of type #{type.inspect}"
          end
        end
      end

      def parse_string_value(value)
        value
      end

      def parse_integer_value(value)
        value.to_i
      end

      def parse_float_value(value)
        value.to_f
      end

      def parse_boolean_value(value)
        (value == "true") || (value == "1")
      end

      def parse_timestamp_value(value)
        Time.at value.to_f.to_i
      end

    end
  end
end

{'fraction_non_alphanumeric': 0.08456375838926175, 'fraction_numerical': 0.0026845637583892616, 'mean_word_length': 1.793103448275862, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4717143', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1037, 'n_words': 265}