<commit_before>require 'iso-639'
module Heathen
  class Processor
    # Converts office documents to their counterpart (e.g. MS Word -> LibreOffice word,
    # or MS Excel -> LibreOffice Sheet) or to PDF. Calls the external 'libreoffice' utility
    # to achieve this.
    # @param: format [String] output format. Must be one of:
    #    pdf - convert to PDF (any libre-office format)
    #    ms  - corresponding Microsoft format
    #    oo  - corresponding LibreOffice format
    def libreoffice( format: )
      suffixes = {
        'pdf' => {
          '.*' => 'pdf',
        },
        'msoffice' => {
          'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'docx',
          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'xlsx',
          'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'pptx',
          'application/vnd.oasis.opendocument.text' => 'docx',
          'application/vnd.oasis.opendocument.spreadsheet' => 'xlsx',
          'application/vnd.oasis.opendocument.presentation' => 'pptx',
          'application/zip' => 'docx',
        },
        'ooffice' => {
          'application/msword' => 'odt',
          'application/vnd.ms-word' => 'odt',
          'application/vnd.ms-excel' => 'ods',
          'application/vnd.ms-office' => 'odt',
          'application/vnd.ms-powerpoint' => 'odp',
          'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'odt',
          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'ods',
          'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'odp',
        },
        'txt' => {
          '.*' => 'txt'
        }
      }

      raise InvalidParameterInStep.new('format', format) unless suffixes[format.to_s]
      to_suffix = nil
      suffixes[format.to_s].each do |k,v|
        to_suffix = v if job.mime_type =~ /#{k}/
      end
      raise InvalidMimeTypeInStep.new('(various document formats)', job.mime_type) unless to_suffix

      output = nil

      if to_suffix == 'txt'
        executioner.execute(
          Colore::C_.tika_path || 'tika',
          '--text',
          job.content_file,
          binary: true
        )

        output = executioner.stdout
      else
        target_file = "#{job.content_file}.#{to_suffix}"
        executioner.execute(
          Colore::C_.libreoffice_path || 'libreoffice',
          '--safe-mode',
          '--convert-to', to_suffix,
          '--outdir', sandbox_dir,
          job.content_file,<commit_msg>Allow multiple libreoffice instances in safe mode<commit_after>require 'iso-639'
module Heathen
  class Processor
    # Converts office documents to their counterpart (e.g. MS Word -> LibreOffice word,
    # or MS Excel -> LibreOffice Sheet) or to PDF. Calls the external 'libreoffice' utility
    # to achieve this.
    # @param: format [String] output format. Must be one of:
    #    pdf - convert to PDF (any libre-office format)
    #    ms  - corresponding Microsoft format
    #    oo  - corresponding LibreOffice format
    def libreoffice( format: )
      suffixes = {
        'pdf' => {
          '.*' => 'pdf',
        },
        'msoffice' => {
          'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'docx',
          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'xlsx',
          'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'pptx',
          'application/vnd.oasis.opendocument.text' => 'docx',
          'application/vnd.oasis.opendocument.spreadsheet' => 'xlsx',
          'application/vnd.oasis.opendocument.presentation' => 'pptx',
          'application/zip' => 'docx',
        },
        'ooffice' => {
          'application/msword' => 'odt',
          'application/vnd.ms-word' => 'odt',
          'application/vnd.ms-excel' => 'ods',
          'application/vnd.ms-office' => 'odt',
          'application/vnd.ms-powerpoint' => 'odp',
          'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'odt',
          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'ods',
          'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'odp',
        },
        'txt' => {
          '.*' => 'txt'
        }
      }

      raise InvalidParameterInStep.new('format', format) unless suffixes[format.to_s]
      to_suffix = nil
      suffixes[format.to_s].each do |k,v|
        to_suffix = v if job.mime_type =~ /#{k}/
      end
      raise InvalidMimeTypeInStep.new('(various document formats)', job.mime_type) unless to_suffix

      output = nil

      if to_suffix == 'txt'
        executioner.execute(
          Colore::C_.tika_path || 'tika',
          '--text',
          job.content_file,
          binary: true
        )

        output = executioner.stdout
      else
        target_file = "#{job.content_file}.#{to_suffix}"
        executioner.execute(
          Colore::C_.libreoffice_path || 'libreoffice',
          '-env:SingleAppInstance=false',
          '--safe-mode',
          '--convert-to', to_suffix,
          '--outdir', sandbox_dir,
          job.content_file,
{'fraction_non_alphanumeric': 0.13513513513513514, 'fraction_numerical': 0.0011419870574800152, 'mean_word_length': 2.3238456672991776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '814936', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1593, 'n_words': 298}