<commit_before>module SmartExcerpt
  class Helper
    include ActionView::Helpers::TextHelper

    def smart_truncate(s, opts = {})
      return '' if s.blank?
      opts = {words: 25}.merge(opts)
      if opts[:sentences]
        return s.split(/\.(\s|$)+/)[0, opts[:sentences]].map{|s| s.strip}.join('. ') + '.'
      end
      if opts[:letters]
        return truncate(s, length: opts[:letters], separator: ' ', omission: '...')
      end
      n = opts[:words]
      if n === Float::INFINITY
        return s
      end
      a = s.split(/\s/) # or /[ ]+/ to only split on spaces
      r = a[0...n].join(' ') + (a.size > n ? '...' : '')

      # replace &nbsp; with regular spaces
      r.gsub!(' ', ' ')

      # strip newlines
      r = r.strip.gsub("\r", '').gsub("\n", ' ')

      r.gsub(/\s+/, ' ')
    end<commit_msg>Add removing empty string<commit_after>module SmartExcerpt
  class Helper
    include ActionView::Helpers::TextHelper

    def smart_truncate(s, opts = {})
      return '' if s.blank?
      opts = {words: 25}.merge(opts)
      if opts[:sentences]
        return s.split(/\.(\s|$)+/).delete_if {|s| s == ' '}[0, opts[:sentences]].map{|s| s.strip}.join('. ') + '.'
      end
      if opts[:letters]
        return truncate(s, length: opts[:letters], separator: ' ', omission: '...')
      end
      n = opts[:words]
      if n === Float::INFINITY
        return s
      end
      a = s.split(/\s/) # or /[ ]+/ to only split on spaces
      r = a[0...n].join(' ') + (a.size > n ? '...' : '')

      # replace &nbsp; with regular spaces
      r.gsub!(' ', ' ')

      # strip newlines
      r = r.strip.gsub("\r", '').gsub("\n", ' ')

      r.gsub(/\s+/, ' ')
    end
{'fraction_non_alphanumeric': 0.21018354055654234, 'fraction_numerical': 0.004736530491415038, 'mean_word_length': 2.017857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802482', 'n_tokens_mistral': 606, 'n_tokens_neox': 579, 'n_words': 133}