<commit_before>module SpatialFeatures
  module Validation
    # SHP file must come first
    REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS = %w[shp shx dbf prj].freeze

    # Check if a shapefile archive includes the required component files, otherwise
    # raise an exception.
    #
    # @param [Zip::File] zip_file                 A Zip::File object
    # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
    # @param [Boolean] allow_generic_zip_files    When true, we skip validation entirely if the archive does not contain a .SHP file
    def self.validate_shapefile_archive!(zip_file, default_proj4_projection: nil, allow_generic_zip_files: false)
      zip_file_entries = zip_file.entries.each_with_object({}) do |f, obj|
        ext = File.extname(f.name).downcase[1..-1]
        next unless ext

        # TODO: we can do better here
        # if ext.casecmp?("shp") && obj.key?(ext)
        #   raise ::SpatialFeatures::Importers::InvalidShapefileArchive, "Zip files that contain multiple Shapefiles are not supported. Please separate each Shapefile into its own zip file."
        # end

        obj[ext] = File.basename(f.name, '.*')
      end

      shapefile_basename = zip_file_entries["shp"]
      unless shapefile_basename
        # not a shapefile archive but we don't care
        return if allow_generic_zip_files

        raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
      end

      REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS[1..-1].each do |ext|
        ext_basename = zip_file_entries[ext]
        next if ext_basename&.casecmp?(shapefile_basename)

        case ext
        when "prj"
          # special case for missing projection files to allow using default_proj4_projection
          next if default_proj4_projection

          raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{expected_component_path(shapefile_basename, ext)}"
        else
          # for all un-handled cases of missing files raise the more generic error
          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{expected_component_path(shapefile_basename, ext)}"
        end
      end

      true
    end

    def self.expected_component_path(basename, ext)
      "#{basename}.#{ext}"
    end
  end
end<commit_msg>Refactor validation to support multiple SHP files per archive<commit_after>module SpatialFeatures
  module Validation
    REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS = %w[shp shx dbf prj].freeze

    class << self
      # Check if a shapefile archive includes the required component files, otherwise
      # raise an exception.
      #
      # @param [Zip::File] zip_file                 A Zip::File object
      # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
      # @param [Boolean] allow_generic_zip_files    When true, we skip validation entirely if the archive does not contain a .SHP file
      def validate_shapefile_archive!(zip_file, default_proj4_projection: nil, allow_generic_zip_files: false)
        contains_shp_file = false

        zip_file_basenames = shapefiles_with_components(zip_file)

        if zip_file_basenames.empty?
          return if allow_generic_zip_files
          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
        end

        zip_file_basenames.each do |basename, extensions|
          validate_components_for_basename(basename, extensions, default_proj4_projection)
        end

        true
      end

      def shapefiles_with_components(zip_file)
        zip_file.entries.each_with_object({}) do |f, obj|
          filename = f.name

          basename = File.basename(filename, '.*')
          ext = File.extname(filename).downcase[1..-1]
          next unless ext

          obj[basename] ||= []
          obj[basename] << ext
        end.delete_if { |basename, exts| !exts.include?("shp") }
      end

      def validate_components_for_basename(basename, extensions, has_default_proj4_projection)
        required_extensions = REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS
        required_extensions -= ['prj'] if has_default_proj4_projection
        missing_extensions = required_extensions - extensions

        missing_extensions.each do |ext|
          case ext
            when "prj"
              raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{expected_component_path(basename, ext)}"
            else
              raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{expected_component_path(basename, ext)}"
            end
        end
      end

      def expected_component_path(basename, ext)
        "#{basename}.#{ext}"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.0736196319018405, 'fraction_numerical': 0.002968533544429052, 'mean_word_length': 3.112286411716843, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2472041', 'n_tokens_mistral': 1523, 'n_tokens_neox': 1415, 'n_words': 399}