<commit_before>require "llt/constants"
require "llt/core"
require "llt/logger"
require "llt/sentence"

module LLT
  class Segmenter
    include Constants::Abbreviations
    include Core::Serviceable

    uses_logger { Logger.new('Segmenter', default: :debug) }

    def self.default_options
      { indexing: true }
    end

    # Abbreviations with boundary e.g. \bA
    #
    # This doesn't work in jruby (opened an issue at jruby/jruby#1269 ),
    # so we have to change things as long as this is not fixed.
    #
    # (?<=\s|^) can be just \b in MRI 2.0 and upwards
    AWB = ALL_ABBRS_PIPED.split('|').map { |abbr| "(?<=\\s|^)#{abbr}" }.join('|')
    SENTENCE_CLOSER = /(?<!#{AWB})\.(?!\.)|[;\?!:]|\n{2}/
    DIRECT_SPEECH_DELIMITER = /['"”]/
    TRAILERS = /\)|<\/.*?>/

    def segment(string, add_to: nil, **options)
      # dump whitespace at the beginning and end!
      @indexing = parse_option(:indexing, options)
      @id = 0 if @indexing
      string.strip!
      sentences = scan_through_string(StringScanner.new(string))
      add_to << sentences if add_to.respond_to?(:<<)
      sentences
    end

    private

    def scan_through_string(scanner, sentences = [])
      while scanner.rest?
        sentence = scanner.scan_until(SENTENCE_CLOSER) ||
          handle_broken_off_texts_or_raise(sentences, scanner)
        sentence << trailing_delimiters(scanner)

        sentence.strip!
        @logger.log("#{'Segmented '.green} #{sentences.size.to_s.cyan} #{sentence}")
        sentences << Sentence.new(sentence, id) unless sentence.empty?
      end
      sentences
    end

    def id
      if @indexing
        @id += 1
      end<commit_msg>Less colors for logger - better performance.<commit_after>require "llt/constants"
require "llt/core"
require "llt/logger"
require "llt/sentence"

module LLT
  class Segmenter
    include Constants::Abbreviations
    include Core::Serviceable

    uses_logger { Logger.new('Segmenter', default: :debug) }

    def self.default_options
      { indexing: true }
    end

    # Abbreviations with boundary e.g. \bA
    #
    # This doesn't work in jruby (opened an issue at jruby/jruby#1269 ),
    # so we have to change things as long as this is not fixed.
    #
    # (?<=\s|^) can be just \b in MRI 2.0 and upwards
    AWB = ALL_ABBRS_PIPED.split('|').map { |abbr| "(?<=\\s|^)#{abbr}" }.join('|')
    SENTENCE_CLOSER = /(?<!#{AWB})\.(?!\.)|[;\?!:]|\n{2}/
    DIRECT_SPEECH_DELIMITER = /['"”]/
    TRAILERS = /\)|<\/.*?>/

    def segment(string, add_to: nil, **options)
      # dump whitespace at the beginning and end!
      @indexing = parse_option(:indexing, options)
      @id = 0 if @indexing
      string.strip!
      sentences = scan_through_string(StringScanner.new(string))
      add_to << sentences if add_to.respond_to?(:<<)
      sentences
    end

    private

    def scan_through_string(scanner, sentences = [])
      while scanner.rest?
        sentence = scanner.scan_until(SENTENCE_CLOSER) ||
          handle_broken_off_texts_or_raise(sentences, scanner)
        sentence << trailing_delimiters(scanner)

        sentence.strip!
        unless sentence.empty?
          @logger.log("Segmented #{id} #{sentence}")
          sentences << Sentence.new(sentence, id)
        end
      end
      sentences
    end

    def id
      if @indexing
        @id += 1
      end
{'fraction_non_alphanumeric': 0.1377352853301464, 'fraction_numerical': 0.00537795040334628, 'mean_word_length': 2.6994475138121548, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '727617', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1111, 'n_words': 268}