<commit_before>package ca.uwaterloo.joos;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import ca.uwaterloo.joos.scanner.Token;
import ca.uwaterloo.joos.scanner.Scanner.ScanException;

public class Preprocessor {

	private final Set<String> undesiredTokenKinds = new HashSet<String>();

	public Preprocessor() {
		this.undesiredTokenKinds.add("COMMENT");
		this.undesiredTokenKinds.add("BLOCKCOMMENT");
		this.undesiredTokenKinds.add("JAVADOC");
	}

	public List<Token> processTokens(List<Token> rawTokens) throws ScanException {
		List<Token> tokens = new ArrayList<Token>();
		for(int i = 0;i<rawTokens.size();i++){
			Token token = rawTokens.get(i);
			
			if(!this.undesiredTokenKinds.contains(token.getKind())) tokens.add(token);
			if(token.getKind().equals("INTLIT"))
			{ 
				String intPositiveThreshold = "2147483647";
				String intNegThreshold = "2147483648";
				Set<String> expressionTokenKinds = new HashSet<String>();
				expressionTokenKinds.add("INTLIT");
				expressionTokenKinds.add("RPAREN");
				if(rawTokens.get(i-1).getKind().equals("MINUS")&&!expressionTokenKinds.contains(rawTokens.get(i-2).getKind()))
				{
					checkIntRange(token.getLexeme(),intNegThreshold);
				}
				else
			   {<commit_msg>modify comment before intliteral<commit_after>package ca.uwaterloo.joos;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import ca.uwaterloo.joos.scanner.Token;
import ca.uwaterloo.joos.scanner.Scanner.ScanException;

public class Preprocessor {

	private final Set<String> undesiredTokenKinds = new HashSet<String>();

	public Preprocessor() {
		this.undesiredTokenKinds.add("COMMENT");
		this.undesiredTokenKinds.add("BLOCKCOMMENT");
		this.undesiredTokenKinds.add("JAVADOC");
	}

	public List<Token> processTokens(List<Token> rawTokens) throws ScanException {
		List<Token> tokens = new ArrayList<Token>();
		
		for(Token token:rawTokens){
			if(!this.undesiredTokenKinds.contains(token.getKind())) tokens.add(token);
			
		}
		for(int i = 0;i<tokens.size();i++){
			Token token = tokens.get(i);
			if(token.getKind().equals("INTLIT"))
			{ 
				String intPositiveThreshold = "2147483647";
				String intNegThreshold = "2147483648";
				Set<String> expressionTokenKinds = new HashSet<String>();
				expressionTokenKinds.add("INTLIT");
				expressionTokenKinds.add("RPAREN");
				if(tokens.get(i-1).getKind().equals("MINUS")&&!expressionTokenKinds.contains(tokens.get(i-2).getKind()))
				{
					checkIntRange(token.getLexeme(),intNegThreshold);
				}
				else
			   {
{'fraction_non_alphanumeric': 0.14411990776325903, 'fraction_numerical': 0.017678708685626442, 'mean_word_length': 5.168246445497631, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5381842', 'n_tokens_mistral': 1018, 'n_tokens_neox': 879, 'n_words': 129}