<commit_before>/*
 * $Id$
 *
 * Copyright (c) 2004-2005 by the TeXlapse Team.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 */
package net.sourceforge.texlipse.editor;

import java.util.ArrayList;
import java.util.List;

import net.sourceforge.texlipse.editor.scanner.TexEnvironmentRule;

import org.eclipse.jface.text.rules.EndOfLineRule;
import org.eclipse.jface.text.rules.IPredicateRule;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.MultiLineRule;
import org.eclipse.jface.text.rules.RuleBasedPartitionScanner;
import org.eclipse.jface.text.rules.SingleLineRule;
import org.eclipse.jface.text.rules.Token;

/**
 * 
 * @author <NAME>
 * @author <NAME>
 */
public class TexPartitionScanner extends RuleBasedPartitionScanner {
	
	public final static String TEX_DEFAULT = "__tex_default";
	public final static String TEX_COMMENT = "__tex_commentPartition";
	public static final String TEX_MATH = "__tex_mathPartition"; 
	public static final String TEX_CURLY_BRACKETS = "__tex_curlyBracketPartition";
	public static final String TEX_SQUARE_BRACKETS = "__tex_squareBracketPartition";
	public static final String[] TEX_PARTITION_TYPES = new String[] {
			TEX_DEFAULT, 
			TEX_COMMENT,
			TEX_MATH,
			TEX_CURLY_BRACKETS,
			TEX_SQUARE_BRACKETS };
	
	public TexPartitionScanner() {
		IToken math 			= new Token(TEX_MATH);
		IToken texComment 		= new Token(TEX_COMMENT);
		//IToken curly_bracket 	= new Token(TEX_CURLY_BRACKETS);
		//IToken square_bracket	= new Token(TEX_SQUARE_BRACKETS);
		
		List rules = new ArrayList();
				
		//TODO: mark \\ Token.Undefined
		rules.add(new SingleLineRule("\\%"," ", Token.UNDEFINED)); //no comment when using "\%" in LaTeX 
		rules.add(new EndOfLineRule("%", texComment));
        rules.add(new TexEnvironmentRule("comment", texComment));
		rules.add(new SingleLineRule("\\\\[","]", Token.UNDEFINED));  //no math when using "\\[]" line breaks
		rules.add(new MultiLineRule("\\[","\\]", math)); 
		
		rules.add(new SingleLineRule("\\$", " ", Token.UNDEFINED)); // not a math equation \$

        rules.add(new TexEnvironmentRule("equation", math));
        rules.add(new TexEnvironmentRule("equation*", math));
        rules.add(new MultiLineRule("$$", "$$", math));
        rules.add(new MultiLineRule("$", "$", math));
        rules.add(new TexEnvironmentRule("eqnarray", math));
        rules.add(new TexEnvironmentRule("eqnarray*", math));
        rules.add(new TexEnvironmentRule("align", math));
        rules.add(new TexEnvironmentRule("align*", math));
        rules.add(new TexEnvironmentRule("alignat", math));
        rules.add(new TexEnvironmentRule("alignat*", math));
        rules.add(new TexEnvironmentRule("math", math));
        rules.add(new TexEnvironmentRule("displaymath", math));

		IPredicateRule[] result = new IPredicateRule[rules.size()];
		rules.toArray(result);
		setPredicateRules(result);
	}
}
<commit_msg>Added some more math environments, now all amsmath environments are supported!<commit_after>/*
 * $Id$
 *
 * Copyright (c) 2004-2005 by the TeXlapse Team.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 */
package net.sourceforge.texlipse.editor;

import java.util.ArrayList;
import java.util.List;

import net.sourceforge.texlipse.editor.scanner.TexEnvironmentRule;

import org.eclipse.jface.text.rules.EndOfLineRule;
import org.eclipse.jface.text.rules.IPredicateRule;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.MultiLineRule;
import org.eclipse.jface.text.rules.RuleBasedPartitionScanner;
import org.eclipse.jface.text.rules.SingleLineRule;
import org.eclipse.jface.text.rules.Token;

/**
 * 
 * @author <NAME>
 * @author <NAME>
 */
public class TexPartitionScanner extends RuleBasedPartitionScanner {
	
	public final static String TEX_DEFAULT = "__tex_default";
	public final static String TEX_COMMENT = "__tex_commentPartition";
	public static final String TEX_MATH = "__tex_mathPartition"; 
	public static final String TEX_CURLY_BRACKETS = "__tex_curlyBracketPartition";
	public static final String TEX_SQUARE_BRACKETS = "__tex_squareBracketPartition";
	public static final String[] TEX_PARTITION_TYPES = new String[] {
			TEX_DEFAULT, 
			TEX_COMMENT,
			TEX_MATH,
			TEX_CURLY_BRACKETS,
			TEX_SQUARE_BRACKETS };
	
	public TexPartitionScanner() {
		IToken math 			= new Token(TEX_MATH);
		IToken texComment 		= new Token(TEX_COMMENT);
		//IToken curly_bracket 	= new Token(TEX_CURLY_BRACKETS);
		//IToken square_bracket	= new Token(TEX_SQUARE_BRACKETS);
		
		List rules = new ArrayList();
				
		//TODO: mark \\ Token.Undefined
		rules.add(new SingleLineRule("\\%"," ", Token.UNDEFINED)); //no comment when using "\%" in LaTeX 
		rules.add(new EndOfLineRule("%", texComment));
        rules.add(new TexEnvironmentRule("comment", texComment));
		rules.add(new SingleLineRule("\\\\[","]", Token.UNDEFINED));  //no math when using "\\[]" line breaks
		rules.add(new MultiLineRule("\\[","\\]", math)); 
		
		rules.add(new SingleLineRule("\\$", " ", Token.UNDEFINED)); // not a math equation \$

        rules.add(new TexEnvironmentRule("equation", math));
        rules.add(new TexEnvironmentRule("equation*", math));
        rules.add(new MultiLineRule("$$", "$$", math));
        rules.add(new MultiLineRule("$", "$", math));
        rules.add(new TexEnvironmentRule("eqnarray", math));
        rules.add(new TexEnvironmentRule("eqnarray*", math));
        rules.add(new TexEnvironmentRule("math", math));
        rules.add(new TexEnvironmentRule("displaymath", math));
        //AMSMath environments
        rules.add(new TexEnvironmentRule("align", math));
        rules.add(new TexEnvironmentRule("align*", math));
        rules.add(new TexEnvironmentRule("alignat", math));
        rules.add(new TexEnvironmentRule("alignat*", math));
        rules.add(new TexEnvironmentRule("flalign", math));
        rules.add(new TexEnvironmentRule("flalign*", math));
        rules.add(new TexEnvironmentRule("multline", math));
        rules.add(new TexEnvironmentRule("multline*", math));
        rules.add(new TexEnvironmentRule("gather", math));
        rules.add(new TexEnvironmentRule("gather*", math));

		IPredicateRule[] result = new IPredicateRule[rules.size()];
		rules.toArray(result);
		setPredicateRules(result);
	}
}

{'fraction_non_alphanumeric': 0.1344222988161247, 'fraction_numerical': 0.0035965832459163793, 'mean_word_length': 4.395311236863379, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '936794', 'n_tokens_mistral': 2207, 'n_tokens_neox': 2058, 'n_words': 456}