<commit_before>class Repositext
  class Validation
    class Validator
      # Validates that two files' paragraphs have identical styles applied.
      class ParagraphStyleConsistency < Validator

        attr_accessor :distinguish_between_normal_and_normal_pn # for testing

        # Runs all validations for self
        def run
          foreign_file, primary_file = @file_to_validate
          outcome = paragraph_styles_consistent?(foreign_file.read, primary_file.read)
          log_and_report_validation_step(outcome.errors, outcome.warnings)
        end

        # Checks if foreign_doc and primary_doc's paragraphs have identical styles applied.
        # @param [String] foreign_doc
        # @param [String] primary_doc
        # @return [Outcome]
        def paragraph_styles_consistent?(foreign_doc, primary_doc)
          # If the foreign file has no paragraph numbers, then we make no distinction
          # between .normal and .normal_pn. This is true for some legacy files.
          # We detect the presence of paragraph numbers in the foreign file,
          # and if the foreign file appears to have paragraph numbers, then
          # we distinguish between .normal and .normal_pn.
          @distinguish_between_normal_and_normal_pn = foreign_has_paragraph_numbers?(foreign_doc)

          foreign_doc_paragraph_styles = extract_paragraph_styles(foreign_doc)
          primary_doc_paragraph_styles = extract_paragraph_styles(primary_doc)

          mismatching_paragraph_styles = compute_paragraph_style_diff(
            foreign_doc_paragraph_styles,
            primary_doc_paragraph_styles
          )

          if mismatching_paragraph_styles.empty?
            Outcome.new(true, nil)
          else
            Outcome.new(
              false, nil, [],
              mismatching_paragraph_styles.map { |diff|
                Reportable.error(
                  [@file_to_validate.first.path],
                  ['Difference in paragraph style between foreign and its corresponding primary file', diff]
                )
              }
            )
          end
        end

      private

        PARAGRAPH_STYLE_REGEX = /^\{:/

        # Extracts paragraph styles from doc.
        # @param doc [String] kramdown document with paragraph styles
        # @return [String] one line for each doc line. Lines with paragraph style
        # contain the style, all other lines are empty.
        def extract_paragraph_styles(doc)
          doc.strip.split(/\n/).map { |line|
            line =~ PARAGRAPH_STYLE_REGEX ? line : ''
          }.inject([]) { |m,e|
            # Remove .omit classes. They are expected to be different between
            # primary and foreign languages.
            r = e.gsub(/\s*\.omit\s*/, ' ')
            # Remove everything but paragraph class names
            r = r.scan(/\.\w+/)
            m << r.join(' ')
          }
        end

        # Computes diffs between foreign and primary paragraph styles.
        # @param styles_f [Array<String>] one entry per line in foreign file.
        #     Only paragraph style lines contain text. All others are empty.
        # @param styles_p [Array<String>] one entry per line in primary file.
        # @return [Array<Hash>] one entry for each diff.
        def compute_paragraph_style_diff(styles_f, styles_p)
          if !@distinguish_between_normal_and_normal_pn<commit_msg>Remove .indent_for_eagle and replace .song_break with .song for paragraph_style_consistency.<commit_after>class Repositext
  class Validation
    class Validator
      # Validates that two files' paragraphs have identical styles applied.
      class ParagraphStyleConsistency < Validator

        attr_accessor :distinguish_between_normal_and_normal_pn # for testing

        # Runs all validations for self
        def run
          foreign_file, primary_file = @file_to_validate
          outcome = paragraph_styles_consistent?(foreign_file.read, primary_file.read)
          log_and_report_validation_step(outcome.errors, outcome.warnings)
        end

        # Checks if foreign_doc and primary_doc's paragraphs have identical styles applied.
        # @param [String] foreign_doc
        # @param [String] primary_doc
        # @return [Outcome]
        def paragraph_styles_consistent?(foreign_doc, primary_doc)
          # If the foreign file has no paragraph numbers, then we make no distinction
          # between .normal and .normal_pn. This is true for some legacy files.
          # We detect the presence of paragraph numbers in the foreign file,
          # and if the foreign file appears to have paragraph numbers, then
          # we distinguish between .normal and .normal_pn.
          @distinguish_between_normal_and_normal_pn = foreign_has_paragraph_numbers?(foreign_doc)

          foreign_doc_paragraph_styles = extract_paragraph_styles(foreign_doc)
          primary_doc_paragraph_styles = extract_paragraph_styles(primary_doc)

          mismatching_paragraph_styles = compute_paragraph_style_diff(
            foreign_doc_paragraph_styles,
            primary_doc_paragraph_styles
          )

          if mismatching_paragraph_styles.empty?
            Outcome.new(true, nil)
          else
            Outcome.new(
              false, nil, [],
              mismatching_paragraph_styles.map { |diff|
                Reportable.error(
                  [@file_to_validate.first.path],
                  ['Difference in paragraph style between foreign and its corresponding primary file', diff]
                )
              }
            )
          end
        end

      private

        PARAGRAPH_STYLE_REGEX = /^\{:/

        # Extracts paragraph styles from doc.
        # @param doc [String] kramdown document with paragraph styles
        # @return [String] one line for each doc line. Lines with paragraph style
        # contain the style, all other lines are empty.
        def extract_paragraph_styles(doc)
          doc.strip.split(/\n/).map { |line|
            line =~ PARAGRAPH_STYLE_REGEX ? line : ''
          }.inject([]) { |m,e|
            # Remove .omit and .indent_for_eagle classes. They are expected to be different between
            # primary and foreign languages.
            r = e.gsub(/\s*\.indent_for_eagle\s*/, ' ')
            r.gsub!(/\s*\.omit\s*/, ' ')
            # Normalize .song_break to .song because we expect them to be differnt between English and foreign
            r.gsub!(/\s*\.song_break\s*/, ' .song ')
            # Remove everything but paragraph class names
            r = r.scan(/\.\w+/)
            m << r.join(' ')
          }
        end

        # Computes diffs between foreign and primary paragraph styles.
        # @param styles_f [Array<String>] one entry per line in foreign file.
        #     Only paragraph style lines contain text. All others are empty.
        # @param styles_p [Array<String>] one entry per line in primary file.
        # @return [Array<Hash>] one entry for each diff.
        def compute_paragraph_style_diff(styles_f, styles_p)
          if !@distinguish_between_normal_and_normal_pn
{'fraction_non_alphanumeric': 0.06945618484080023, 'fraction_numerical': 0.0, 'mean_word_length': 2.2356426618049223, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2496963', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1814, 'n_words': 563}