<commit_before>package org.onetwo.app.task;

import java.util.List;

import org.onetwo.common.log.MyLoggerFactory;
import org.slf4j.Logger;

public class TaskConsumer {
	private static final Logger logger = MyLoggerFactory.getLogger(TaskConsumer.class);

	private TaskQueue taskQueue;
	private TaskListenerRegistry taskListenerRegistry;
	

	public TaskConsumer(TaskQueue taskQueue, TaskListenerRegistry taskListenerRegistry) {
		super();
		this.taskQueue = taskQueue;
		this.taskListenerRegistry = taskListenerRegistry;
	}

	public void doTask() throws InterruptedException {
		TaskData task = null;
		for(;;){
			task = taskQueue.take();
			logger.info("thread[{}] take task: {}", Thread.currentThread().getId(), task.getName());
			doProcess(task);
		}
	}
	
	protected void doProcess(TaskData task){
		List<TaskListener> listeners = this.taskListenerRegistry.getTaskListenerGroup(task.getType()).getListeners();
		for(TaskListener taskListener : listeners){
			logger.info("execute task listenr[{}] for task {}", taskListener.getClass(), task.getName());<commit_msg>task..<commit_after>package org.onetwo.app.task;

import java.util.List;

import org.onetwo.common.log.MyLoggerFactory;
import org.slf4j.Logger;

public class TaskConsumer {
	private static final Logger logger = MyLoggerFactory.getLogger(TaskConsumer.class);

	private TaskQueue taskQueue;
	private TaskListenerRegistry taskListenerRegistry;
	

	public TaskConsumer(TaskQueue taskQueue, TaskListenerRegistry taskListenerRegistry) {
		super();
		this.taskQueue = taskQueue;
		this.taskListenerRegistry = taskListenerRegistry;
	}

	public void doTask() throws InterruptedException {
		TaskData task = null;
		for(;;){
			task = taskQueue.take();
			logger.info("task consumer thread[{}] take task: {}", Thread.currentThread().getId(), task.getName());
			doProcess(task);
		}
	}
	
	protected void doProcess(TaskData task){
		List<TaskListener> listeners = this.taskListenerRegistry.getTaskListenerGroup(task.getType()).getListeners();
		for(TaskListener taskListener : listeners){
			logger.info("task consumer execute task listenr[{}] for task {}", taskListener.getClass(), task.getName());
{'fraction_non_alphanumeric': 0.1177561427909133, 'fraction_numerical': 0.0009272137227630969, 'mean_word_length': 5.85079365079365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1464098', 'n_tokens_mistral': 644, 'n_tokens_neox': 603, 'n_words': 137}