<commit_before>require 'open3'

ENV['FREELINGSHARE'] ||= '/usr/local/share/freeling'


module FreeLing
  class Analyzer
    attr_reader :document, :last_error

    DEFAULT_ANALYZE_PATH = '/usr/local/bin/analyzer'
    DEFAULT_CONFIG_PATH  = File.join(Padrino.root, 'config', 'freeling_es.cfg')

    NotRunningError     = Class.new(StandardError)
    AnalyzerError       = Class.new(StandardError)


    def initialize(document, opts={})
      @document = document

      @options = {
        :config_path => DEFAULT_CONFIG_PATH,
        :analyze_path => DEFAULT_ANALYZE_PATH,
        :input_format => :plain,
        :output_format => :tagged,
        :memoize => true,
      }.merge(opts)

      unless File.exists?(@options[:analyze_path])
        raise "#{@options[:analyze_path]} not found"
      end

      unless File.exists?(@options[:config_path])
        raise "#{@options[:config_path]} not found"
      end

      @last_error_mutex = Mutex.new
    end

    def sentences(run_again=false)
      if @options[:output_format] == :plain
        raise "Sentence splitter is not available with output format set to 'plain'"
      end

      if not run_again and @sentences
        return @sentences.to_enum
      end

      Enumerator.new do |yielder|
        tokens = []
        read_tokens.each do |token|
          if token
            tokens << token
          else
            yielder << tokens
            if @options[:memoize]
              @sentences ||= []
              @sentences << tokens
            end
            tokens = []
          end
        end<commit_msg>fix typo on FreeLing::Analyzer#sentences when output format is :token<commit_after>require 'open3'

ENV['FREELINGSHARE'] ||= '/usr/local/share/freeling'


module FreeLing
  class Analyzer
    attr_reader :document, :last_error

    DEFAULT_ANALYZE_PATH = '/usr/local/bin/analyzer'
    DEFAULT_CONFIG_PATH  = File.join(Padrino.root, 'config', 'freeling_es.cfg')

    NotRunningError     = Class.new(StandardError)
    AnalyzerError       = Class.new(StandardError)


    def initialize(document, opts={})
      @document = document

      @options = {
        :config_path => DEFAULT_CONFIG_PATH,
        :analyze_path => DEFAULT_ANALYZE_PATH,
        :input_format => :plain,
        :output_format => :tagged,
        :memoize => true,
      }.merge(opts)

      unless File.exists?(@options[:analyze_path])
        raise "#{@options[:analyze_path]} not found"
      end

      unless File.exists?(@options[:config_path])
        raise "#{@options[:config_path]} not found"
      end

      @last_error_mutex = Mutex.new
    end

    def sentences(run_again=false)
      if @options[:output_format] == :token
        raise "Sentence splitter is not available with output format set to 'token'"
      end

      if not run_again and @sentences
        return @sentences.to_enum
      end

      Enumerator.new do |yielder|
        tokens = []
        read_tokens.each do |token|
          if token
            tokens << token
          else
            yielder << tokens
            if @options[:memoize]
              @sentences ||= []
              @sentences << tokens
            end
            tokens = []
          end
        end
{'fraction_non_alphanumeric': 0.10565568676196395, 'fraction_numerical': 0.0006215040397762585, 'mean_word_length': 2.2029850746268655, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205145', 'n_tokens_mistral': 981, 'n_tokens_neox': 940, 'n_words': 215}