<commit_before>require 'pry'

module Hdo
  module Transcript
    module TextUtils

      #
      # https://en.wikipedia.org/wiki/LIX
      #

      SENTENCE_EXP = /[.:!?]/

      def self.lix(str)
        str = str.to_s
        words = words(str)

        a = words.size
        b = str.scan(SENTENCE_EXP).size
        b = 1 if b.zero?
        c = words.map { |e| e.gsub(/[.:;,]/, '') }.select { |e| e.size > 6 }.size

        score = (a / b.to_f) + ((c * 100) / a.to_f)

        classification = case score
                         when -Float::INFINITY..30
                           'Veldig lettlest / barnebøker'
                         when 30..40
                           'Lettlest, skjønnlitteratur'
                         when 40..50
                            'Middels vanskelig, avistekst'
                         when 50..60
                            'Vanskelig, offisielle tekster'
                         when 60..Float::INFINITY
                            'Meget vanskelig, byråkratnorsk'
                         end

        {
          score: score,
          classification: classification
        }
      end

      def self.words(str)
        str.to_s.scan(/\S+/)
      end

      def self.word_count(str)
        words(str).size
      end
    end
  end
end<commit_msg>LIX: Handle some common abbreviations<commit_after>require 'pry'

module Hdo
  module Transcript
    module TextUtils

      #
      # https://en.wikipedia.org/wiki/LIX
      #

      SENTENCE_EXP = /[.:!?]/

      def self.lix(str)
        str = str.to_s.gsub(/f\.\s*eks\s*\./, 'for eksempel').gsub(/bl\.\s*a\s*./, 'blant annet')
        words = words(str)

        a = words.size
        b = str.scan(SENTENCE_EXP).size
        b = 1 if b.zero?
        c = words.map { |e| e.gsub(/[.:;,]/, '') }.select { |e| e.size > 6 }.size

        score = (a / b.to_f) + ((c * 100) / a.to_f)

        classification = case score
                         when -Float::INFINITY..30
                           'Veldig lettlest / barnebøker'
                         when 30..40
                           'Lettlest, skjønnlitteratur'
                         when 40..50
                            'Middels vanskelig, avistekst'
                         when 50..60
                            'Vanskelig, offisielle tekster'
                         when 60..Float::INFINITY
                            'Meget vanskelig, byråkratnorsk'
                         end

        {
          score: score,
          classification: classification
        }
      end

      def self.words(str)
        str.to_s.scan(/\S+/)
      end

      def self.word_count(str)
        words(str).size
      end
    end
  end
end

if __FILE__ == $0
  p Hdo::Transcript::TextUtils.lix(STDIN.read)
end
{'fraction_non_alphanumeric': 0.11375947995666305, 'fraction_numerical': 0.015529071867100036, 'mean_word_length': 1.1048632218844985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5537889', 'n_tokens_mistral': 935, 'n_tokens_neox': 859, 'n_words': 179}