<commit_before># -*- coding: utf-8 -*-
require 'csv'

module KokugoTagger
	module_function
	def annotate(file)
		file.each_line do |line|
			data = CabochaParser.parse(line)
			method_name = data[:type].downcase.to_sym
			method(method_name).call(data) if methods.include?(method_name)
			puts line
		end
	end
	def chunk(data)
		@chunks ||= []
		@chunks << @chunk = data
		@lpos ||= 0
		@chunk.update start:@lpos, end:@lpos, text:'', pos:nil, pred:nil, conj:nil
	end
	def token(data)
		@lpos += data[:text].size
		@chunk[:end] = @lpos
		@chunk[:text] += data[:text]
		pos data
		cform data
	end<commit_msg>debug.<commit_after># -*- coding: utf-8 -*-
require 'csv'

module KokugoTagger
	module_function
	def annotate(file)
		file.each_line do |line|
			next unless data = CabochaParser.parse(line)
			method_name = data[:type].downcase.to_sym
			method(method_name).call(data) if methods.include?(method_name)
			puts line
		end
	end
	def chunk(data)
		@chunks ||= []
		@chunks << @chunk = data
		@lpos ||= 0
		@chunk.update start:@lpos, end:@lpos, text:'', pos:nil, pred:nil, conj:nil
	end
	def token(data)
		@lpos += data[:text].size
		@chunk[:end] = @lpos
		@chunk[:text] += data[:text]
		pos data
		cform data
	end
{'fraction_non_alphanumeric': 0.15858668857847166, 'fraction_numerical': 0.003286770747740345, 'mean_word_length': 4.033057851239669, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6858294', 'n_tokens_mistral': 517, 'n_tokens_neox': 472, 'n_words': 107}