<commit_before># encoding: utf-8
require 'open3'
require_relative './exceptions'

module CartoDB
  module Importer2
    class ShpNormalizer
      NORMALIZER_RELATIVE_PATH = 
        "../../../../../lib/importer/misc/shp_normalizer.py"

      def self.supported?(extension)
        %w{ .shp .tab }.include?(extension)
      end #self.supported?

      def initialize(filepath, job)
        @job      = job
        @filepath = filepath
      end #initialize

      def encoding
        normalize
        encoding = normalizer_output.fetch(:encoding)
        return 'LATIN1' if encoding == 'None' 
        return codepage_for(encoding) if windows?(encoding)
        encoding
      end #encoding

      def normalize
        stdout, stderr, status  = Open3.capture3(normalizer_command)
        output                  = stdout.strip.split(/, */, 4)
        self.normalizer_output  = {
          projection:   output[0],
          encoding:     output[1],
          source:       output[2],
          destination:  output[3]
        }

        raise ShpNormalizationError unless status.to_i == 0 
        raise ShpNormalizationError unless !!normalizer_output
        self
      end #normalize

      def prj?<commit_msg>[importer] ShpNormalizer will raise on an SHP missing required files<commit_after># encoding: utf-8
require 'open3'
require_relative './exceptions'

module CartoDB
  module Importer2
    class ShpNormalizer
      NORMALIZER_RELATIVE_PATH = 
        "../../../../../lib/importer/misc/shp_normalizer.py"

      def self.supported?(extension)
        %w{ .shp .tab }.include?(extension)
      end #self.supported?

      def initialize(filepath, job)
        @job      = job
        @filepath = filepath
      end #initialize

      def encoding
        normalize
        encoding = normalizer_output.fetch(:encoding)
        return 'LATIN1' if encoding == 'None' 
        return codepage_for(encoding) if windows?(encoding)
        encoding
      end #encoding

      def normalize
        raise InvalidShpError         unless dbf? && shx?
        raise MissingProjectionError  unless prj?

        stdout, stderr, status  = Open3.capture3(normalizer_command)
        output                  = stdout.strip.split(/, */, 4)
        self.normalizer_output  = {
          projection:   output[0],
          encoding:     output[1],
          source:       output[2],
          destination:  output[3]
        }

        raise ShpNormalizationError unless status.to_i == 0 
        raise ShpNormalizationError unless !!normalizer_output
        self
      end #normalize

      def prj?
{'fraction_non_alphanumeric': 0.0928351309707242, 'fraction_numerical': 0.009244992295839754, 'mean_word_length': 2.0127610208816704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3624587', 'n_tokens_mistral': 753, 'n_tokens_neox': 731, 'n_words': 179}