<commit_before>require "docsplit"

module Docsplit
  class InfoExtractor
    # Pull out a single datum from a pdf.
    def extract(key, pdfs, opts)
      pdf = [pdfs].flatten.first
      cmd = "pdfinfo #{ESCAPE[pdf]} 2>&1"
      result = `#{cmd}`.chomp

      # FIX encoding issues
      result.encode('UTF-8', :invalid => :replace, :replace => '').encode('UTF-8')

      raise ExtractionFailed, result if $? != 0 
      match = result.match(MATCHERS[key])
      answer = match && match[1]
      answer = answer.to_i if answer && key == :length
      answer
    end 
  end
end<commit_msg>Tell pdfinfo to encode strings to UTF-8 when extracting metadata (#45)<commit_after>require "docsplit"

module Docsplit
  class InfoExtractor
    # Pull out a single datum from a pdf.
    def extract(key, pdfs, opts)
      pdf = [pdfs].flatten.first
      cmd = "pdfinfo -enc UTF-8 #{ESCAPE[pdf]} 2>&1"
      result = `#{cmd}`.chomp
      raise ExtractionFailed, result if $? != 0 
      match = result.match(MATCHERS[key])
      answer = match && match[1]
      answer = answer.to_i if answer && key == :length
      answer
    end 
  end
end
{'fraction_non_alphanumeric': 0.125552608311229, 'fraction_numerical': 0.01237842617152962, 'mean_word_length': 2.7114754098360656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4479756', 'n_tokens_mistral': 379, 'n_tokens_neox': 360, 'n_words': 111}