<commit_before>require 'rbconfig'

module Docsplit
  class PdfExtractor
    @@executable = nil

    HOST_OS = (defined?("RbConfig") ? RbConfig : Config)::CONFIG['host_os']
    def windows?
      !!HOST_OS.match(/mswin|windows|cygwin/i)
    end
    def osx?
      !!HOST_OS.match(/darwin/i)
    end
    def linux?
      !!HOST_OS.match(/linux/i)
    end
    
    def version_string
      @@help ||= `#{office_executable} -h 2>&1`.split("\n").first
    end
    
    def libre_office?
      !!version_string.match(/^LibreOffice/)
    end

    def open_office?
      !!version_string.match(/^OpenOffice.org/)
    end
    
    def office_search_paths
      if windows?
        office_names       = ["LibreOffice 3", "LibreOffice 4", "OpenOffice.org 3"]
        program_files_path = ENV["CommonProgramFiles"]
        search_paths       = office_name.map{ |program| File.join(program_files_path, program) }
      elsif osx?
        search_paths = %w(
          /Applications/LibreOffice.app/Contents
          /Applications/OpenOffice.org.app/Contents
        )
      else # probably linux/unix
        search_paths = %w(
          /usr/lib/libreoffice
          /opt/libreoffice
          /usr/lib/openoffice
          /opt/openoffice.org3
        )
      end
      search_paths
    end
    
    def office_executable
      paths = office_search_paths

      if ENV['OFFICE_PATH']
        raise ArgumentError, "No such file or directory #{ENV['OFFICE_PATH']}" unless File.exists? ENV['OFFICE_PATH']
        paths.unshift(ENV['OFFICE_PATH'])
      end
      
      path_pieces = ["soffice"]
      if windows?
        path_pieces += [["program", "soffice.bin"]]
      elsif osx?
        path_pieces += [["MacOS", "soffice"], ["Contents", "MacOS", "soffice"]]
      else
        path_pieces += [["program", "soffice"]]
      end
      
      paths.each do |path|
        if File.exists? path
          @@executable ||= path unless File.directory? path
          path_pieces.each do |pieces|
            check_path = File.join(path, pieces)
            @@executable ||= check_path if File.exists? check_path
          end
        end
        break if @@executable
      end
      raise OfficeNotFound, "No office software found" unless @@executable
      @@executable
    end
    
    def office_path
      File.dirname(File.dirname(office_executable))
    end
    
    def extract(docs, opts)
      out = opts[:output] || '.'
      FileUtils.mkdir_p out unless File.exists?(out)
      [docs].flatten.each do |doc|
        ext = File.extname(doc)
        basename = File.basename(doc, ext)
        escaped_doc, escaped_out, escaped_basename = [doc, out, basename].map(&ESCAPE)

        if GM_FORMATS.include?(`file -b --mime #{ESCAPE[doc]}`.strip.split(/[:;]\s+/)[0])
          `gm convert #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf`
        else
          if libre_office?
            options = "--headless --convert-to pdf --outdir #{escaped_out} #{escaped_doc}"
            cmd = "#{office_executable} #{options} 2>&1"
            result = `#{cmd}`.chomp
            raise ExtractionFailed, result if $? != 0
            true
          else # open office presumably
            options = "-jar #{ESCAPED_ROOT}/vendor/jodconverter/jodconverter-core-3.0-beta-4.jar -r #{ESCAPED_ROOT}/vendor/conf/document-formats.js"
            run_jod "#{options} #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf", [], {}
          end
        end
      end
    end

    CLASSPATH     = "#{ESCAPED_ROOT}/build#{File::PATH_SEPARATOR}#{ESCAPED_ROOT}/vendor/'*'"

    LOGGING       = "-Djava.util.logging.config.file=#{ESCAPED_ROOT}/vendor/logging.properties"

    HEADLESS      = "-Djava.awt.headless=true"
    
    private
    
    # Runs a Java command, with quieted logging, and the classpath set properly.
    def run_jod(command, pdfs, opts, return_output=false)

      pdfs   = [pdfs].flatten.map{|pdf| "\"#{pdf}\""}.join(' ')
      office = osx? ? "-Doffice.home=#{office_path}" : office_path
      cmd    = "java #{HEADLESS} #{LOGGING} #{office} -cp #{CLASSPATH} #{command} #{pdfs} 2>&1"
      result = `#{cmd}`.chomp
      raise ExtractionFailed, result if $? != 0<commit_msg>Add config for user profile.<commit_after>require 'rbconfig'

module Docsplit
  class PdfExtractor
    @@executable = nil

    HOST_OS = (defined?("RbConfig") ? RbConfig : Config)::CONFIG['host_os']
    def windows?
      !!HOST_OS.match(/mswin|windows|cygwin/i)
    end
    def osx?
      !!HOST_OS.match(/darwin/i)
    end
    def linux?
      !!HOST_OS.match(/linux/i)
    end
    
    def version_string
      @@help ||= `#{office_executable} -h 2>&1`.split("\n").first
    end
    
    def libre_office?
      !!version_string.match(/^LibreOffice/)
    end

    def open_office?
      !!version_string.match(/^OpenOffice.org/)
    end
    
    def office_search_paths
      if windows?
        office_names       = ["LibreOffice 3", "LibreOffice 4", "OpenOffice.org 3"]
        program_files_path = ENV["CommonProgramFiles"]
        search_paths       = office_name.map{ |program| File.join(program_files_path, program) }
      elsif osx?
        search_paths = %w(
          /Applications/LibreOffice.app/Contents
          /Applications/OpenOffice.org.app/Contents
        )
      else # probably linux/unix
        search_paths = %w(
          /usr/lib/libreoffice
          /opt/libreoffice
          /usr/lib/openoffice
          /opt/openoffice.org3
        )
      end
      search_paths
    end
    
    def office_executable
      paths = office_search_paths

      if ENV['OFFICE_PATH']
        raise ArgumentError, "No such file or directory #{ENV['OFFICE_PATH']}" unless File.exists? ENV['OFFICE_PATH']
        paths.unshift(ENV['OFFICE_PATH'])
      end
      
      path_pieces = ["soffice"]
      if windows?
        path_pieces += [["program", "soffice.bin"]]
      elsif osx?
        path_pieces += [["MacOS", "soffice"], ["Contents", "MacOS", "soffice"]]
      else
        path_pieces += [["program", "soffice"]]
      end
      
      paths.each do |path|
        if File.exists? path
          @@executable ||= path unless File.directory? path
          path_pieces.each do |pieces|
            check_path = File.join(path, pieces)
            @@executable ||= check_path if File.exists? check_path
          end
        end
        break if @@executable
      end
      raise OfficeNotFound, "No office software found" unless @@executable
      @@executable
    end
    
    def office_path
      File.dirname(File.dirname(office_executable))
    end
    
    def extract(docs, opts)
      out = opts[:output] || '.'
      FileUtils.mkdir_p out unless File.exists?(out)
      [docs].flatten.each do |doc|
        ext = File.extname(doc)
        basename = File.basename(doc, ext)
        escaped_doc, escaped_out, escaped_basename = [doc, out, basename].map(&ESCAPE)

        if GM_FORMATS.include?(`file -b --mime #{ESCAPE[doc]}`.strip.split(/[:;]\s+/)[0])
          `gm convert #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf`
        else
          if libre_office?
            # Set the LibreOffice user profile, so 
            ENV['SYSUSERCONFIG']="file://#{File.expand_path(escaped_out)}"

            options = "--headless --invisible  --norestore --nolockcheck --convert-to pdf --outdir #{escaped_out} #{escaped_doc}"
            cmd = "#{office_executable} #{options} 2>&1"
            result = `#{cmd}`.chomp
            raise ExtractionFailed, result if $? != 0
            true
          else # open office presumably, rely on JODConverter to figure it out.
            options = "-jar #{ESCAPED_ROOT}/vendor/jodconverter/jodconverter-core-3.0-beta-4.jar -r #{ESCAPED_ROOT}/vendor/conf/document-formats.js"
            run_jod "#{options} #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf", [], {}
          end
        end
      end
    end

    CLASSPATH     = "#{ESCAPED_ROOT}/build#{File::PATH_SEPARATOR}#{ESCAPED_ROOT}/vendor/'*'"

    LOGGING       = "-Djava.util.logging.config.file=#{ESCAPED_ROOT}/vendor/logging.properties"

    HEADLESS      = "-Djava.awt.headless=true"
    
    private
    
    # Runs a Java command, with quieted logging, and the classpath set properly.
    def run_jod(command, pdfs, opts, return_output=false)

      pdfs   = [pdfs].flatten.map{|pdf| "\"#{pdf}\""}.join(' ')
      office = osx? ? "-Doffice.home=#{office_path}" : office_path
      cmd    = "java #{HEADLESS} #{LOGGING} #{office} -cp #{CLASSPATH} #{command} #{pdfs} 2>&1"
      result = `#{cmd}`.chomp
      raise ExtractionFailed, result if $? != 0
{'fraction_non_alphanumeric': 0.13497433504433037, 'fraction_numerical': 0.003733084461035931, 'mean_word_length': 2.319008904374758, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3283463', 'n_tokens_mistral': 2890, 'n_tokens_neox': 2761, 'n_words': 570}