<commit_before>module Metacrunch
  module UBPB
    module Transformations
      module MAB2SNR
        class Description < Metacrunch::Transformer::Step

          def perform
            target.add("search",  "description", descriptions)
            target.add("display", "description", descriptions)
          end

        private

          def descriptions
            unless @descriptions
              @descriptions = []

              # 405 - Erscheinungsverlauf von Zeitschriften
              @descriptions += descriptions_for("405")
              # 522 - Teilungsvermerk bei fortlaufenden Sammelwerken
              @descriptions += descriptions_for("522")
              # 523 - Erscheinungsverlauf von Monos
              @descriptions += descriptions_for("523")
              # Sonstige 500er
              (501..519).each { |f| @descriptions += descriptions_for("#{f}") }
              # 536 - Voraussichtlicher Erscheinungstermin
              @descriptions += descriptions_for("536")
              # 537 - Redaktionelle Bemerkungen (bei Zeitschriften ignorieren)
              @descriptions += descriptions_for("537") unless helper.is_journal?
            end

            @descriptions
          end

        private

          def descriptions_for(field)
            descriptions = []

            source.datafields(field, ind2: "1").each do |datafield|
              a = datafield.subfields("a").first_value
              p = datafield.subfields("p").first_value

              descriptions << [p, a].compact.join(": ") if a || p
            end

            descriptions<commit_msg>Refactor.<commit_after>module Metacrunch
  module UBPB
    module Transformations
      module MAB2SNR
        class Description < Metacrunch::Transformer::Step

          def perform
            target.add("search",  "description", descriptions)
            target.add("display", "description", descriptions)
          end

        private

          def descriptions
            @descriptions ||= begin
              descriptions = []

              # 405 - Erscheinungsverlauf von Zeitschriften
              descriptions += descriptions_for("405")
              # 522 - Teilungsvermerk bei fortlaufenden Sammelwerken
              descriptions += descriptions_for("522")
              # 523 - Erscheinungsverlauf von Monos
              descriptions += descriptions_for("523")
              # Sonstige 500er
              (501..519).each { |f| descriptions += descriptions_for("#{f}") }
              # 536 - Voraussichtlicher Erscheinungstermin
              descriptions += descriptions_for("536")
              # 537 - Redaktionelle Bemerkungen (bei Zeitschriften ignorieren)
              descriptions += descriptions_for("537") unless helper.is_journal?

              descriptions
            end
          end

        private

          def descriptions_for(field)
            descriptions = []

            source.datafields(field, ind2: "1").each do |datafield|
              a = datafield.subfields("a").first_value
              p = datafield.subfields("p").first_value

              descriptions << [p, a].compact.join(": ") if a || p
            end

            descriptions
{'fraction_non_alphanumeric': 0.08888888888888889, 'fraction_numerical': 0.02629107981220657, 'mean_word_length': 1.810905892700088, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4937393', 'n_tokens_mistral': 856, 'n_tokens_neox': 798, 'n_words': 179}