<commit_before>class Repositext
  class Merge
    class AcceptedCorrectionsIntoContentAt

      # Auto-merges accepted_corrections into content_at.
      # @param[String] accepted_corrections
      # @param[String] content_at to merge corrections into
      # @param[String] content_at_filename
      # @return[Outcome] the merged document is returned as #result if successful.
      def self.merge_auto(accepted_corrections, content_at, content_at_filename)
        corrections = extract_corrections(accepted_corrections)
        validate_corrections(corrections)
        outcome = merge_corrections_into_content_at(:auto, corrections, content_at, content_at_filename)
      end

      # Manually merges accepted_corrections into content_at.
      # @param[String] accepted_corrections
      # @param[String] content_at to merge corrections into
      # @param[String] content_at_filename
      # @return[Outcome] the merged document is returned as #result if successful.
      def self.merge_manually(accepted_corrections, content_at, content_at_filename)
        corrections = extract_corrections(accepted_corrections)
        validate_corrections(corrections)
        outcome = merge_corrections_into_content_at(:manual, corrections, content_at, content_at_filename)
      end

    protected

      # @param[String] accepted_corrections
      # @return[Array<Hash>] a hash describing the corrections
      def self.extract_corrections(accepted_corrections)
        corrections = []
        s = StringScanner.new(accepted_corrections)
        while !s.eos? do
          ca = {}
          # Advance cursor up to and excluding next numbered correction
          s.skip_until(/\n(?=\d+\.)/)
          # capture correction number
          correction_number = s.scan(/\d+/)
          ca[:correction_number] = correction_number  if correction_number
          # capture remainder of numbered correction's first line
          correction_location = s.scan(/\.[^\n]+/)
          if correction_location
            # extract paragraph number
            ca[:paragraph_number] = correction_location.match(/paragraphs?\s+(\d+)/i)[1].to_s
            # extract line number
            if (match = correction_location.match(/lines?\s+([\d\-]+)/i))
              ca[:line_number] = match[1].to_s
            else
              raise "Could not parse line number for correction ##{ ca[:correction_number] }, para ##{ ca[:paragraph_number] }"
            end
            # extract reads
            s.skip_until(/\nreads:\s+/i) # advance up to and including reads: marker
            ca[:before] = s.scan(/.+?(?=\nbecomes:)/im).to_s.strip # fetch everything up to "Becomes:"
            # extract becomes
            s.skip_until(/\nbecomes:\s+/i) # advance up to and including becomes: marker
            ca[:after] = s.scan(/.+?(?=(\n{2,}\d+\.|\n+\z))/m).to_s.strip # fetch up to next correction number, or end of document
          else
            # No more corrections found
            s.terminate
          end
          corrections << ca  if !ca.empty?
        end
        corrections
      end

      # Validates the extracted corrections
      # @param[Array<Hash>]
      def self.validate_corrections(corrections)
        # Validate that each correction has the required attrs
        with_missing_attrs = []
        corrections.each { |e|
          has_missing_attrs = [
            :correction_number,
            :paragraph_number,
            :line_number,
            :before,
            :after,
          ].any? { |attr| e[attr].nil? }
          if has_missing_attrs
            with_missing_attrs << e
          end
        }
        if with_missing_attrs.any?
          raise "Not all attrs are present: #{ with_missing_attrs.inspect }"
        end

        # Validate that we get consecutive correction_numbers
        correction_numbers = corrections.map { |e| e[:correction_number].to_i }.sort
        correction_numbers.each_cons(2) { |x,y|
          if y != x + 1
            raise "Correction numbers are not consecutive: #{ [x,y].inspect }"
          end
        }

        # Validate that no invalid characters are in correction file
        invalid_chars_regex = /–/
        corrections.each { |e|
          if e[:before] =~ invalid_chars_regex
            raise "Invalid character in correction ##{ e[:correction_number] }: #{ e[:before] }"
          end
          if e[:after] =~ invalid_chars_regex
            raise "Invalid character in correction ##{ e[:correction_number] }: #{ e[:before] }"
          end
        }
      end

      # Merges corrections into content_at
      # @param[Symbol] strategy one of :auto, :manual
      # @param[Array<Hash>] corrections
      # @param[String] content_at
      # @param[String] content_at_filename
      # @return[Outcome] where result is corrected_at and messages contains the report_lines
      def self.merge_corrections_into_content_at(strategy, corrections, content_at, content_at_filename)
        report_lines = []
        corrected_at = content_at.dup<commit_msg>Improved validation of accepted_corrections files:<commit_after>class Repositext
  class Merge
    class AcceptedCorrectionsIntoContentAt

      # Auto-merges accepted_corrections into content_at.
      # @param[String] accepted_corrections
      # @param[String] content_at to merge corrections into
      # @param[String] content_at_filename
      # @return[Outcome] the merged document is returned as #result if successful.
      def self.merge_auto(accepted_corrections, content_at, content_at_filename)
        validate_accepted_corrections_file(accepted_corrections)
        corrections = extract_corrections(accepted_corrections)
        validate_corrections(corrections)
        outcome = merge_corrections_into_content_at(:auto, corrections, content_at, content_at_filename)
      end

      # Manually merges accepted_corrections into content_at.
      # @param[String] accepted_corrections
      # @param[String] content_at to merge corrections into
      # @param[String] content_at_filename
      # @return[Outcome] the merged document is returned as #result if successful.
      def self.merge_manually(accepted_corrections, content_at, content_at_filename)
        corrections = extract_corrections(accepted_corrections)
        validate_corrections(corrections)
        outcome = merge_corrections_into_content_at(:manual, corrections, content_at, content_at_filename)
      end

    protected

      # Validates the contents of the accepted_corrections file before it attempts
      # any parsing.
      # @param[String] accepted_corrections
      def self.validate_accepted_corrections_file(accepted_corrections)
        # Validate that no invalid characters are in correction file
        invalid_chars = []
        [
          [/–/, 'EN DASH'],
          [/"/, 'Straight double quote'],
          [/'/, 'Straight single quote'],
        ].each do |(regex, description)|
          s = StringScanner.new(accepted_corrections)
          while !s.eos? do
            inv_char = s.scan_until(/.{,5}#{ regex }/) # match up to 5 chars before invalid char for reporting context
            if inv_char
              previous_text = accepted_corrections[0..(s.pos - 1)]
              line_num = previous_text.count("\n") + 1
              context = s.matched[(-[10, s.matched.length].min)..-1] + s.rest[0,10]
              invalid_chars << " - #{ description } on line #{ line_num }: #{ context }"
            else
              s.terminate
            end
          end
        end
        if invalid_chars.any?
          msg = ["Invalid characters:"]
          msg += invalid_chars
          raise msg.join("\n")
        end
      end

      # @param[String] accepted_corrections
      # @return[Array<Hash>] a hash describing the corrections
      def self.extract_corrections(accepted_corrections)
        corrections = []
        s = StringScanner.new(accepted_corrections)
        while !s.eos? do
          ca = {}
          # Advance cursor up to and excluding next numbered correction
          s.skip_until(/\n(?=\d+\.)/)
          # capture correction number
          correction_number = s.scan(/\d+/)
          ca[:correction_number] = correction_number  if correction_number
          # capture remainder of numbered correction's first line
          correction_location = s.scan(/\.[^\n]+/)
          if correction_location
            # extract paragraph number
            ca[:paragraph_number] = correction_location.match(/paragraphs?\s+(\d+)/i)[1].to_s
            # extract line number
            if (match = correction_location.match(/lines?\s+([\d\-]+)/i))
              ca[:line_number] = match[1].to_s
            else
              raise "Could not parse line number for correction ##{ ca[:correction_number] }, para ##{ ca[:paragraph_number] }"
            end
            # extract reads
            s.skip_until(/\nreads:\s+/i) # advance up to and including reads: marker
            ca[:before] = s.scan(/.+?(?=\nbecomes:)/im).to_s.strip # fetch everything up to "Becomes:"
            # extract becomes
            s.skip_until(/\nbecomes:\s+/i) # advance up to and including becomes: marker
            ca[:after] = s.scan(/.+?(?=(\n{2,}\d+\.|\n+\z))/m).to_s.strip # fetch up to next correction number, or end of document
          else
            # No more corrections found
            s.terminate
          end
          corrections << ca  if !ca.empty?
        end
        corrections
      end

      # Validates the extracted corrections
      # @param[Array<Hash>]
      def self.validate_corrections(corrections)
        # Validate that each correction has the required attrs
        with_missing_attrs = []
        corrections.each { |e|
          has_missing_attrs = [
            :correction_number,
            :paragraph_number,
            :line_number,
            :before,
            :after,
          ].any? { |attr| e[attr].nil? }
          if has_missing_attrs
            with_missing_attrs << e
          end
        }
        if with_missing_attrs.any?
          raise "Not all attrs are present: #{ with_missing_attrs.inspect }"
        end

        # Validate that we get consecutive correction_numbers
        correction_numbers = corrections.map { |e| e[:correction_number].to_i }.sort
        correction_numbers.each_cons(2) { |x,y|
          if y != x + 1
            raise "Correction numbers are not consecutive: #{ [x,y].inspect }"
          end
        }
      end

      # Merges corrections into content_at
      # @param[Symbol] strategy one of :auto, :manual
      # @param[Array<Hash>] corrections
      # @param[String] content_at
      # @param[String] content_at_filename
      # @return[Outcome] where result is corrected_at and messages contains the report_lines
      def self.merge_corrections_into_content_at(strategy, corrections, content_at, content_at_filename)
        report_lines = []
        corrected_at = content_at.dup
{'fraction_non_alphanumeric': 0.09501642935377876, 'fraction_numerical': 0.0019167579408543264, 'mean_word_length': 2.3682754380571778, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '255367', 'n_tokens_mistral': 3079, 'n_tokens_neox': 2989, 'n_words': 751}