<commit_before>class Repositext
  class Validation
    class Validator

      # Validates consistency of exported pdf content with content AT.
      class PdfExportConsistency < Validator

        # Runs all validations for self
        def run
          outcome = pdf_export_consistent?(@file_to_validate)
          log_and_report_validation_step(outcome.errors, outcome.warnings)
        end

      protected

        # For some languages the PDF text extraction does not work, so we have
        # to skip this validation.
        # See [Issue #322](https://vgrtr.vgr.local/vgr-repositext/vgr-repositext/issues/322) for details.
        def language_codes_to_skip
          %i[]
        end

        # @param pdf_file_name [String] absolute path to the PDF file
        # @return [Outcome]
        def pdf_export_consistent?(pdf_file_name)
          repository = @options['repository']
          language = repository.language

          if language_codes_to_skip.include?(language.code_3_chars)
            # Skip certain languages
            @logger.warning(
              "Skipped validation PdfExportConsistency because language #{ language.name } is not supported"
            )
            return Outcome.new(true, nil, [], [], [])
          end

          pdf_file_stub = Repositext::RFile::Binary.new(
            '_', language, pdf_file_name, repository
          )
          corresponding_content_at_file = pdf_file_stub.corresponding_content_at_file

          pdf_raw_text = extract_pdf_raw_text(
            pdf_file_name,
            @options['extract_text_from_pdf_service']
          )
          content_at_plain_text = corresponding_content_at_file.plain_text_contents(
            convert_smcaps_to_upper_case: true
          )

          errors = []
          warnings = []

          validate_content_consistency(pdf_raw_text, content_at_plain_text, errors, warnings)
          Outcome.new(errors.empty?, nil, [], errors, warnings)
        end

        # Extracts raw text from pdf_file_name
        # @param pdf_file_name [String]
        # @param extract_text_from_pdf_service [Services::ExtractTextFromPdf]
        def extract_pdf_raw_text(pdf_file_name, extract_text_from_pdf_service)
          extract_text_from_pdf_service.extract(pdf_file_name)
        end

        # Validates that contents of pdf_raw_text and content_at are consistent.
        # Mutates `errors` and `warnings` in place.
        # @param pdf_raw_text [String]
        # @param content_at_plain_text [String]
        # @param [Array] errors collector for errors
        # @param [Array] warnings collector for warnings
        def validate_content_consistency(pdf_raw_text, content_at_plain_text, errors, warnings)
          pdf_plain_text = sanitize_pdf_raw_text(pdf_raw_text)
          c_at_pt = sanitize_content_at_plain_text(content_at_plain_text)

          all_diffs_with_context = Suspension::StringComparer.compare(
            c_at_pt,
            pdf_plain_text,
            true, # include context
            false # include all diffs
          )

          # find insert [1], delete [-1], and change [-1,1] groups
          diff_groups = []
          current_diff_group = nil
          all_diffs_with_context.each { |diff|
            ins_del, txt_diff, location, context = diff
            case ins_del
            when -1
              if current_diff_group.nil?
                # A -1 always starts a new group
                current_diff_group = [diff]
              else
                # We don't expect to see a -1 if we already have a current_diff_group
                raise "Unexpected -1: #{ diff.inspect }"
              end
            when 0
              # 0 closes any diff groups
              if current_diff_group
                diff_groups << current_diff_group
                current_diff_group = nil
              end
            when 1
              if current_diff_group.nil?
                # An insertion
                current_diff_group = [diff]
              else
                # This is a change. We already have a -1
                current_diff_group << diff
              end
            else
              raise "handle this: #{ ins_del.inspect }"
            end
            if current_diff_group && current_diff_group.length > 2
              # No diff group should have more than 2 elements
              raise "Handle this: #{ current_diff_group.inspect }"
            end
          }
          # Finalize any started diff group
          if current_diff_group
            diff_groups << current_diff_group
            current_diff_group = nil
          end

          # Process each diff_group
          diff_groups.each { |diff_group|

            ins_del_signature = diff_group.map { |e| e.first }
            description = nil
            text_difference = nil

            case ins_del_signature
            when [-1]
              # Deletion
              ins_del, txt_diff, location, context = diff_group.first

              # Ignore missing horizontal_rule in PDF
              next  if "* * * * * * *\n" == txt_diff

              # Prepare error reporting data
              description = "Missing "<commit_msg>PDF export validation: Address issue where space between first and second line goes missing with drop cap eagles.<commit_after>class Repositext
  class Validation
    class Validator

      # Validates consistency of exported pdf content with content AT.
      class PdfExportConsistency < Validator

        # Runs all validations for self
        def run
          outcome = pdf_export_consistent?(@file_to_validate)
          log_and_report_validation_step(outcome.errors, outcome.warnings)
        end

      protected

        # For some languages the PDF text extraction does not work, so we have
        # to skip this validation.
        # See [Issue #322](https://vgrtr.vgr.local/vgr-repositext/vgr-repositext/issues/322) for details.
        def language_codes_to_skip
          %i[]
        end

        # @param pdf_file_name [String] absolute path to the PDF file
        # @return [Outcome]
        def pdf_export_consistent?(pdf_file_name)
          repository = @options['repository']
          language = repository.language

          if language_codes_to_skip.include?(language.code_3_chars)
            # Skip certain languages
            @logger.warning(
              "Skipped validation PdfExportConsistency because language #{ language.name } is not supported"
            )
            return Outcome.new(true, nil, [], [], [])
          end

          pdf_file_stub = Repositext::RFile::Binary.new(
            '_', language, pdf_file_name, repository
          )
          corresponding_content_at_file = pdf_file_stub.corresponding_content_at_file

          pdf_raw_text = extract_pdf_raw_text(
            pdf_file_name,
            @options['extract_text_from_pdf_service']
          )
          content_at_plain_text = corresponding_content_at_file.plain_text_contents(
            convert_smcaps_to_upper_case: true
          )

          errors = []
          warnings = []

          validate_content_consistency(pdf_raw_text, content_at_plain_text, errors, warnings)
          Outcome.new(errors.empty?, nil, [], errors, warnings)
        end

        # Extracts raw text from pdf_file_name
        # @param pdf_file_name [String]
        # @param extract_text_from_pdf_service [Services::ExtractTextFromPdf]
        def extract_pdf_raw_text(pdf_file_name, extract_text_from_pdf_service)
          extract_text_from_pdf_service.extract(pdf_file_name)
        end

        # Validates that contents of pdf_raw_text and content_at are consistent.
        # Mutates `errors` and `warnings` in place.
        # @param pdf_raw_text [String]
        # @param content_at_plain_text [String]
        # @param [Array] errors collector for errors
        # @param [Array] warnings collector for warnings
        def validate_content_consistency(pdf_raw_text, content_at_plain_text, errors, warnings)
          pdf_plain_text = sanitize_pdf_raw_text(pdf_raw_text)
          c_at_pt = sanitize_content_at_plain_text(content_at_plain_text)

          all_diffs_with_context = Suspension::StringComparer.compare(
            c_at_pt,
            pdf_plain_text,
            true, # include context
            false # include all diffs
          )

          # find insert [1], delete [-1], and change [-1,1] groups
          diff_groups = []
          current_diff_group = nil
          all_diffs_with_context.each { |diff|
            ins_del, txt_diff, location, context = diff
            case ins_del
            when -1
              if current_diff_group.nil?
                # A -1 always starts a new group
                current_diff_group = [diff]
              else
                # We don't expect to see a -1 if we already have a current_diff_group
                raise "Unexpected -1: #{ diff.inspect }"
              end
            when 0
              # 0 closes any diff groups
              if current_diff_group
                diff_groups << current_diff_group
                current_diff_group = nil
              end
            when 1
              if current_diff_group.nil?
                # An insertion
                current_diff_group = [diff]
              else
                # This is a change. We already have a -1
                current_diff_group << diff
              end
            else
              raise "handle this: #{ ins_del.inspect }"
            end
            if current_diff_group && current_diff_group.length > 2
              # No diff group should have more than 2 elements
              raise "Handle this: #{ current_diff_group.inspect }"
            end
          }
          # Finalize any started diff group
          if current_diff_group
            diff_groups << current_diff_group
            current_diff_group = nil
          end

          # Process each diff_group
          diff_groups.each { |diff_group|

            ins_del_signature = diff_group.map { |e| e.first }
            description = nil
            text_difference = nil

            case ins_del_signature
            when [-1]
              # Deletion
              ins_del, txt_diff, location, context = diff_group.first

              # Ignore missing horizontal_rule in PDF
              next  if "* * * * * * *\n" == txt_diff

              # With drop cap first eagle, pdf extractor gets confused and
              # doesn't insert a space between the first and second line.
              # It's safe to ignore.
              next  if (' ' == txt_diff) && ('line 2' == location) && (content_at_plain_text.index(""))

              # Prepare error reporting data
              description = "Missing "
{'fraction_non_alphanumeric': 0.0638139534883721, 'fraction_numerical': 0.004186046511627907, 'mean_word_length': 1.854753053637812, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4152341', 'n_tokens_mistral': 2929, 'n_tokens_neox': 2823, 'n_words': 748}