<commit_before>module LinkThumbnailer
  class Model

    def to_json(*args)
      as_json.to_json(*args)
    end

    private

    def sanitize(str)
      return unless str
      str.strip.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/[\r\n\f]+/, "\n")
    end

  end
end<commit_msg>can't call string functions until encoding's clean<commit_after>module LinkThumbnailer
  class Model

    def to_json(*args)
      as_json.to_json(*args)
    end

    private

    def sanitize(str)
      return unless str
      str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '').strip.gsub(/[\r\n\f]+/, "\n")
    end

  end
end
{'fraction_non_alphanumeric': 0.1426448736998514, 'fraction_numerical': 0.0029717682020802376, 'mean_word_length': 2.9415204678362574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6061145', 'n_tokens_mistral': 239, 'n_tokens_neox': 226, 'n_words': 57}