<commit_before>/*
 Copyright (c) 2006 <NAME>, Jr, <NAME>, and
 Centre National de la Recherche Scientifique (CNRS).

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.

 $HeadURL: https://svn.lal.in2p3.fr/LCG/QWG/panc/trunk/src/org/quattor/ant/PanCompilerTask.java $
 $Id: PanCompilerTask.java 4004 2008-12-01 14:18:54Z loomis $
 */

package org.quattor.ant;

import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.DirSet;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.ant.types.Path;
import org.quattor.pan.Compiler;
import org.quattor.pan.CompilerLogging;
import org.quattor.pan.CompilerOptions;
import org.quattor.pan.CompilerResults;
import org.quattor.pan.output.Formatter;
import org.quattor.pan.output.FormatterUtils;

/**
 * An ant task which permits calling the pan compiler from an ant build file.
 * This task allows all of the compiler parameters to be accessed and will
 * optionally check dependency files before starting a build. See individual
 * setter methods for the parameters which can be used in the build file.
 * 
 * @author loomis
 * 
 */
public class PanCompilerTask extends Task {

	/* The pattern in the dependency files. (Two double-quoted strings). */
	private static Pattern depline = Pattern.compile("\"(.*)\"\\s+\"(.*)\"");

	/* A pattern that includes everything. For debugging includes/excludes. */
	private static Pattern allPattern = Pattern.compile(".*");

	/* List of files to actually compile and process. */
	private LinkedList<File> files = new LinkedList<File>();

	/* The root directory for the includes. */
	private File includeroot = null;

	/* A comma- or space-separated list of file globs. */
	private List<DirSet> includes = new LinkedList<DirSet>();

	/* The list of directories to include in search path. */
	private LinkedList<File> includeDirectories = new LinkedList<File>();

	private boolean debugTask = false;

	/* Produce very verbose output */
	private boolean debugVerbose = false;

	private List<Pattern> debugIncludePatterns = new LinkedList<Pattern>();

	private List<Pattern> debugExcludePatterns = new LinkedList<Pattern>();

	private boolean xmlWriteEnabled = true;

	private boolean depWriteEnabled = true;

	private int iterationLimit = 5000;

	private int callDepthLimit = 50;

	private Formatter formatter;

	private File outputDirectory = null;

	private File sessionDirectory = null;

	private boolean checkDependencies = true;

	private int nthread = 0;

	private boolean verbose = false;

	private boolean gzipOutput = false;

	private int deprecationLevel = 0;

	private boolean forceBuild = false;

	final protected static String debugIdent = "    ";

	private String loggingFlags = "none";

	private File logfile = null;

	private boolean dumpAnnotations = false;

	private FileStatCache statCache = null;

	/**
	 * Constructor
	 */
	public PanCompilerTask() {
		// Set profile output format
		setFormatter("xmldb");

		// Create a cache for the modification times of the templates. In the
		// case where all of the files are up to date, this will save repeated
		// disk reads to determine the state of files as dependencies are in
		// common within a cluster.
		statCache = new FileStatCache();
	}

	/**
	 * Launch a build with the pan compiler using the parameters and list of
	 * file set via ant.
	 * 
	 * @throws BuildException
	 *             if the 'includeroot' parameter isn't specified when using the
	 *             'includes' parameter
	 */
	@Override
	public void execute() throws BuildException {

		// If some include globs were specified, then check that the
		// includeroot was specified. Add the necessary paths.
		if (includes.size() > 0) {
			if (includeroot == null) {
				throw new BuildException(
						"includeroot must be specified to use 'includes' parameter");
			}

			Path antpath = new Path(getProject());<commit_msg>move file cache initialization to static initializer<commit_after>/*
 Copyright (c) 2006 <NAME>, Jr, <NAME>, and
 Centre National de la Recherche Scientifique (CNRS).

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.

 $HeadURL: https://svn.lal.in2p3.fr/LCG/QWG/panc/trunk/src/org/quattor/ant/PanCompilerTask.java $
 $Id: PanCompilerTask.java 4004 2008-12-01 14:18:54Z loomis $
 */

package org.quattor.ant;

import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.DirSet;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.ant.types.Path;
import org.quattor.pan.Compiler;
import org.quattor.pan.CompilerLogging;
import org.quattor.pan.CompilerOptions;
import org.quattor.pan.CompilerResults;
import org.quattor.pan.output.Formatter;
import org.quattor.pan.output.FormatterUtils;

/**
 * An ant task which permits calling the pan compiler from an ant build file.
 * This task allows all of the compiler parameters to be accessed and will
 * optionally check dependency files before starting a build. See individual
 * setter methods for the parameters which can be used in the build file.
 * 
 * @author loomis
 * 
 */
public class PanCompilerTask extends Task {

	/* The pattern in the dependency files. (Two double-quoted strings). */
	private static Pattern depline = Pattern.compile("\"(.*)\"\\s+\"(.*)\"");

	/* A pattern that includes everything. For debugging includes/excludes. */
	private static Pattern allPattern = Pattern.compile(".*");

	/* List of files to actually compile and process. */
	private LinkedList<File> files = new LinkedList<File>();

	/* The root directory for the includes. */
	private File includeroot = null;

	/* A comma- or space-separated list of file globs. */
	private List<DirSet> includes = new LinkedList<DirSet>();

	/* The list of directories to include in search path. */
	private LinkedList<File> includeDirectories = new LinkedList<File>();

	private boolean debugTask = false;

	/* Produce very verbose output */
	private boolean debugVerbose = false;

	private List<Pattern> debugIncludePatterns = new LinkedList<Pattern>();

	private List<Pattern> debugExcludePatterns = new LinkedList<Pattern>();

	private boolean xmlWriteEnabled = true;

	private boolean depWriteEnabled = true;

	private int iterationLimit = 5000;

	private int callDepthLimit = 50;

	private Formatter formatter;

	private File outputDirectory = null;

	private File sessionDirectory = null;

	private boolean checkDependencies = true;

	private int nthread = 0;

	private boolean verbose = false;

	private boolean gzipOutput = false;

	private int deprecationLevel = 0;

	private boolean forceBuild = false;

	final protected static String debugIdent = "    ";

	private String loggingFlags = "none";

	private File logfile = null;

	private boolean dumpAnnotations = false;

	// Create a cache for the modification times of the templates. In the
	// case where all of the files are up to date, this will save repeated
	// disk reads to determine the state of files as dependencies are in
	// common within a cluster.
	private FileStatCache statCache = new FileStatCache();

	/**
	 * Constructor
	 */
	public PanCompilerTask() {
		setFormatter("xmldb");
	}

	/**
	 * Launch a build with the pan compiler using the parameters and list of
	 * file set via ant.
	 * 
	 * @throws BuildException
	 *             if the 'includeroot' parameter isn't specified when using the
	 *             'includes' parameter
	 */
	@Override
	public void execute() throws BuildException {

		// If some include globs were specified, then check that the
		// includeroot was specified. Add the necessary paths.
		if (includes.size() > 0) {
			if (includeroot == null) {
				throw new BuildException(
						"includeroot must be specified to use 'includes' parameter");
			}

			Path antpath = new Path(getProject());
{'fraction_non_alphanumeric': 0.09078227277604893, 'fraction_numerical': 0.007940766176628394, 'mean_word_length': 3.994640943193998, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3008618', 'n_tokens_mistral': 2753, 'n_tokens_neox': 2527, 'n_words': 995}