<commit_before># encoding: utf-8
# Define ENV variable for customizations to FreeLing default configurations
ENV['FREELING_CUSTOM'] ||= File.join(PADRINO_ROOT, "config", "freeling")

module Analyzer
  LETTERS = 'áéíóúñüça-z'
  LETTERS_UPCASE = 'ÁÉÍÓÚÑÜA-Z'

  NP_RE = "(?:[#{LETTERS_UPCASE}][#{LETTERS}]+(?:[ ,](?:[#{LETTERS_UPCASE}][#{LETTERS_UPCASE}#{LETTERS}]+|(?:(?:de|la|del)(?= ))))*)"
  ADDRESS_RE = Regexp.new("(?<!^)((?:Av.? )?#{NP_RE}+ [0-9]{1,5}(?![0-9\/])(,? )?#{NP_RE}*)")

  ##
  # Return an enumerator of tokens
  #
  # All tokens have a reference to its position in the string
  # for locating it easily.
  #
  def self.extract_tokens(content, lang = :es)
    require "freeling/analyzer"

    Enumerator.new do |yielder|
      pos = 0
      analyzer = FreeLing::Analyzer.new(content, {
        :share_path => File.join(Padrino.root, "config", "freeling"),
        :output_format => :token,
        :memoize => false,
        :language => lang
      })
      analyzer.tokens.each do |token|
        token_pos = content.index(token[:form], pos)
        yielder << token.merge(:pos => token_pos)
        pos = token_pos + token[:form].size
      end
    end
  end

  ##
  # Return an enumerator of tokens with PoS tags
  #
  # NOTE This function works correctly *only if* the following FreeLing
  # config options are reset:
  #
  #   RetokContractions=no
  #   TaggerRetokenize=no
  #
  # When any of these are set, Dictionary or Tagger modules can retokenize
  # contracted words (e.g. "he's" => "he is"), changing the original text.
  # An exception is raised if this happens.
  #
  def self.extract_tagged_tokens(content, lang = :es)
    require "freeling/analyzer"

    Enumerator.new do |yielder|
      no_tokens = false
      total_size = content.size
      sentence_pos = 0

      st = self.extract_tokens(content, lang)
      begin
        cur_st = st.next
      rescue StopIteration
        no_tokens = true
      end

      unless no_tokens
        analyzer = FreeLing::Analyzer.new(content,
          :share_path => File.join(Padrino.root, "config", "freeling"),
          :output_format => :tagged,
          :memoize => false,
          :language => lang)
        analyzer.sentences.each do |sentence|
          sentence.each do |token|
            logger.debug "Token (#{cur_st[:pos]}/#{total_size}): #{token}"
            new_token = nil

            # exact match
            if token[:form] == cur_st[:form]
              new_token = token.merge(:pos => cur_st[:pos], :sentence_pos => sentence_pos)
              if NamedEntity::CLASSES_PER_TAG[token[:tag]]
                new_token[:tokens] = [{ :pos => cur_st[:pos], :form => cur_st[:form] }]
              end<commit_msg>freeling-analyzer: Correct use of Analyzer in ExtractorTask<commit_after># encoding: utf-8

# Define ENV variable for customizations to FreeLing default configurations
ENV['FREELING_CUSTOM'] ||= File.join(PADRINO_ROOT, "config", "freeling")

module Analyzer
  LETTERS = 'áéíóúñüça-z'
  LETTERS_UPCASE = 'ÁÉÍÓÚÑÜA-Z'

  NP_RE = "(?:[#{LETTERS_UPCASE}][#{LETTERS}]+(?:[ ,](?:[#{LETTERS_UPCASE}][#{LETTERS_UPCASE}#{LETTERS}]+|(?:(?:de|la|del)(?= ))))*)"
  ADDRESS_RE = Regexp.new("(?<!^)((?:Av.? )?#{NP_RE}+ [0-9]{1,5}(?![0-9\/])(,? )?#{NP_RE}*)")

  ##
  # Return an enumerator of tokens
  #
  # All tokens have a reference to its position in the string
  # for locating it easily.
  #
  def self.extract_tokens(content, lang = :es)
    require "freeling/analyzer"

    Enumerator.new do |yielder|
      pos = 0
      analyzer = FreeLing::Analyzer.new(content, {
        :config_path => File.join(Padrino.root, "config", "freeling", "config", "#{lang}.cfg"),
        :output_format => :token,
        :memoize => false,
      })
      analyzer.tokens.each do |token|
        token_pos = content.index(token[:form], pos)
        yielder << token.merge(:pos => token_pos)
        pos = token_pos + token[:form].size
      end
    end
  end

  ##
  # Return an enumerator of tokens with PoS tags
  #
  # NOTE This function works correctly *only if* the following FreeLing
  # config options are reset:
  #
  #   RetokContractions=no
  #   TaggerRetokenize=no
  #
  # When any of these are set, Dictionary or Tagger modules can retokenize
  # contracted words (e.g. "he's" => "he is"), changing the original text.
  # An exception is raised if this happens.
  #
  def self.extract_tagged_tokens(content, lang = :es)
    require "freeling/analyzer"

    Enumerator.new do |yielder|
      no_tokens = false
      total_size = content.size
      sentence_pos = 0

      st = self.extract_tokens(content, lang)
      begin
        cur_st = st.next
      rescue StopIteration
        no_tokens = true
      end

      unless no_tokens
        analyzer = FreeLing::Analyzer.new(content, {
          :config_path => File.join(Padrino.root, "config", "freeling", "config", "#{lang}.cfg"),
          :output_format => :tagged,
          :memoize => false
        })
        analyzer.sentences.each do |sentence|
          sentence.each do |token|
            logger.debug "Token (#{cur_st[:pos]}/#{total_size}): #{token}"
            new_token = nil

            # exact match
            if token[:form] == cur_st[:form]
              new_token = token.merge(:pos => cur_st[:pos], :sentence_pos => sentence_pos)
              if NamedEntity::CLASSES_PER_TAG[token[:tag]]
                new_token[:tokens] = [{ :pos => cur_st[:pos], :form => cur_st[:form] }]
              end
{'fraction_non_alphanumeric': 0.14575713240673008, 'fraction_numerical': 0.0032918800292611556, 'mean_word_length': 2.7847750865051903, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897806', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1855, 'n_words': 427}