<commit_before>module Lurker
  class SchemaModifier
    class Atom
      TYPE_MAP = {
        "Fixnum" => "integer",
        "Float" => "number",
        "Hash" => "object",
        "Time" => "string",
        "TrueClass" => "boolean",
        "FalseClass" => "boolean",
        "NilClass" => "null"
      }.freeze

      DATE_TIME_FORMAT = "date-time".freeze
      COLOR_FORMAT = "color".freeze
      URI_FORMAT = "uri".freeze

      def initialize(json_schema_hash)
        @atom = json_schema_hash
      end

      def merge!(data)
        atom = build_atom(data)
        combined? ? append_new(atom) : combine_with(atom)
      end

      private

      def combine_with(atom)
        return @atom.clear.merge!(atom) unless @atom.key?("type")

        if @atom["type"] != atom["type"]
          existing_atom = @atom.dup
          @atom.clear
          @atom["anyOf"] = [existing_atom, atom]
        end
      end

      def append_new(atom)
        # TODO : If example empty - fill from duplicate
        return if !!@atom["anyOf"].detect { |atm| atm["type"] == atom["type"] }

        @atom["anyOf"] << atom
      end

      def combined?
        !!@atom["anyOf"]
      end

      def build_atom(data)
        atom = {
          "description" => "",
          "type" => guess_type(data),
          "example" => data
        }

        if format = guess_format(data)
          atom["format"] = format
        end

        atom
      end

      def guess_type(data)
        in_type = data.class.to_s
        TYPE_MAP[in_type] || in_type.downcase
      end

      def guess_format(data)
        if data.kind_of?(Time)
          DATE_TIME_FORMAT
        elsif data.kind_of?(String)
          if data.start_with? "http://"
            URI_FORMAT
          elsif data.match(/\#[0-9a-fA-F]{3}(?:[0-9a-fA-F]{3})?\b/)
            COLOR_FORMAT
          else
            begin
              DATE_TIME_FORMAT if Time.iso8601(data)
            rescue
              nil
            end<commit_msg>handle uploaded files as strings<commit_after>module Lurker
  class SchemaModifier
    class Atom
      TYPE_MAP = {
        "ActionDispatch::Http::UploadedFile" => "string",
        "Fixnum" => "integer",
        "Float" => "number",
        "Hash" => "object",
        "Time" => "string",
        "TrueClass" => "boolean",
        "FalseClass" => "boolean",
        "NilClass" => "null"
      }.freeze

      DATE_TIME_FORMAT = "date-time".freeze
      COLOR_FORMAT = "color".freeze
      URI_FORMAT = "uri".freeze

      def initialize(json_schema_hash)
        @atom = json_schema_hash
      end

      def merge!(data)
        atom = build_atom(data)
        combined? ? append_new(atom) : combine_with(atom)
      end

      private

      def combine_with(atom)
        return @atom.clear.merge!(atom) unless @atom.key?("type")

        if @atom["type"] != atom["type"]
          existing_atom = @atom.dup
          @atom.clear
          @atom["anyOf"] = [existing_atom, atom]
        end
      end

      def append_new(atom)
        # TODO : If example empty - fill from duplicate
        return if !!@atom["anyOf"].detect { |atm| atm["type"] == atom["type"] }

        @atom["anyOf"] << atom
      end

      def combined?
        !!@atom["anyOf"]
      end

      def build_atom(data)
        atom = {
          "description" => "",
          "type" => guess_type(data),
          "example" => serialize(data)
        }

        if format = guess_format(data)
          atom["format"] = format
        end

        atom
      end

      def serialize(data)
        if data.is_a?(ActionDispatch::Http::UploadedFile)
          data.open.read
        else
          data
        end
      end

      def guess_type(data)
        in_type = data.class.to_s
        TYPE_MAP[in_type] || in_type.downcase
      end

      def guess_format(data)
        if data.kind_of?(Time)
          DATE_TIME_FORMAT
        elsif data.kind_of?(String)
          if data.start_with? "http://"
            URI_FORMAT
          elsif data.match(/\#[0-9a-fA-F]{3}(?:[0-9a-fA-F]{3})?\b/)
            COLOR_FORMAT
          else
            begin
              DATE_TIME_FORMAT if Time.iso8601(data)
            rescue
              nil
            end
{'fraction_non_alphanumeric': 0.12659725508755323, 'fraction_numerical': 0.00473260766682442, 'mean_word_length': 1.680405833861763, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5917589', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1300, 'n_words': 266}