<commit_before>package ca.ipredict.predictor.TDAG;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;

import ca.ipredict.database.Item;
import ca.ipredict.database.Sequence;
import ca.ipredict.predictor.Predictor;

/**
 * TDAG predictor is based on Markov Trees
 * Its fast but it has drawbacks in terms 
 * It takes a lot of space!
 * This implementation has a few of the optimization presented in the original paper below.
 * 
 * Optimization#1: max tree height as described in original papel -> reduce spacial size without noticeable effect on accuracy and coverage (to be double checked)
 * 
 * Source: <NAME>, and <NAME>. "Discrete sequence prediction and its applications." Machine learning 15.1 (1994): 43-68.
 */
public class TDAGPredictor extends Predictor {

	/**
	 * FIFO used during training to remember the last Node inserted in the tree
	 */
	private List<Node> state;
	
	/**
	 * Root of the tree
	 */
	private Node root;
	
	/**
	 * Number of nodes in the tree
	 */
	private Integer size;
	
	/**
	 * Max tree height, forbid creating branch with a length higher
	 * than this parameter
	 */
	private final Integer maxTreeHeight = 6;
	
	/**
	 * Map a list of symbol to a specific node in the tree
	 * It is used to lookup specific nodes in the prediction method.
	 */
	private HashMap<List<Integer>, Node> dictionnary;
	
	
	public TDAGPredictor() {
		root = new Node(0, new ArrayList<Integer>());
		size = 1;
		state = new ArrayList<Node>();
		dictionnary = new HashMap<List<Integer>, Node>();
		
		TAG = "TDAG";
	}

	public TDAGPredictor(String tag) {
		this();
		TAG = tag;
	}

	@Override
	public Boolean Train(List<Sequence> trainingSequences) {
		
		
		//for each training sequence
		for(Sequence seq : trainingSequences) {
			
			//resetting the states
			state.clear();
			state.add(root);
			
			//For each item in the current sequence
			for(Item item : seq.getItems()) {
				
				//Initiating the newState<commit_msg>Fixed a bug in TDAG where it would not reset when re-trained.<commit_after>package ca.ipredict.predictor.TDAG;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;

import ca.ipredict.database.Item;
import ca.ipredict.database.Sequence;
import ca.ipredict.predictor.Predictor;

/**
 * TDAG predictor is based on Markov Trees
 * Its fast but it has drawbacks in terms 
 * It takes a lot of space!
 * This implementation has a few of the optimization presented in the original paper below.
 * 
 * Optimization#1: max tree height as described in original papel -> reduce spacial size without noticeable effect on accuracy and coverage (to be double checked)
 * 
 * Source: Laird, Philip, and <NAME>. "Discrete sequence prediction and its applications." Machine learning 15.1 (1994): 43-68.
 */
public class TDAGPredictor extends Predictor {

	/**
	 * FIFO used during training to remember the last Node inserted in the tree
	 */
	private List<Node> state;
	
	/**
	 * Root of the tree
	 */
	private Node root;
	
	/**
	 * Number of nodes in the tree
	 */
	private Integer size;
	
	/**
	 * Max tree height, forbid creating branch with a length higher
	 * than this parameter
	 */
	private final Integer maxTreeHeight = 6;
	
	/**
	 * Map a list of symbol to a specific node in the tree
	 * It is used to lookup specific nodes in the prediction method.
	 */
	private HashMap<List<Integer>, Node> dictionnary;
	
	
	public TDAGPredictor() {
		root = new Node(0, new ArrayList<Integer>());
		size = 1;
		state = new ArrayList<Node>();
		dictionnary = new HashMap<List<Integer>, Node>();
		
		TAG = "TDAG";
	}

	public TDAGPredictor(String tag) {
		this();
		TAG = tag;
	}

	@Override
	public Boolean Train(List<Sequence> trainingSequences) {
		
		//reset
		root = new Node(0, new ArrayList<Integer>());
		size = 1;
		state = new ArrayList<Node>();
		dictionnary = new HashMap<List<Integer>, Node>();
		
		
		//for each training sequence
		for(Sequence seq : trainingSequences) {
			
			//resetting the states
			state.clear();
			state.add(root);
			
			//For each item in the current sequence
			for(Item item : seq.getItems()) {
				
				//Initiating the newState
{'fraction_non_alphanumeric': 0.09878600333253987, 'fraction_numerical': 0.007617233991906689, 'mean_word_length': 3.42781875658588, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '972531', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1194, 'n_words': 476}