<commit_before>package com.seleniumtests.util.squashta;


import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.filter.ElementFilter;
import org.jdom2.input.SAXBuilder;

import com.seleniumtests.reporter.TestLogging;

public class TaScriptGenerator {
	
	private static final Logger logger = TestLogging.getLogger(TaScriptGenerator.class);
	
	private String srcPath;
	private String dstPath;
	private String application;
	
	// patterns for exclusion from .ta script generation
	private static final String XML_EXCLUDE = "EXCLUDE_FROM_SQUASH_TA";
	private static final String FEATURE_EXCLUDE = "@EXCLUDE_FROM_SQUASH_TA";	

	/**
	 * 
	 * @param srcPath		Path to root of the test application. it's the location of the "data" folder
	 * @param dstPath		Path to root where files will be written
	 * @param application	name of the application
	 */
	public TaScriptGenerator(String application, String srcPath, String dstPath) {
		this.srcPath = srcPath.replace("\\", "/");
		this.dstPath = dstPath.replace("\\", "/");
		this.application = application;
	}
	
	/**
	 * Search for scenarios in feature files
	 * @param path
	 * @param application
	 * @return
	 */
	public List<String> parseFeatures() {
		
		Pattern scenarioPattern = Pattern.compile("^\\s*Scenario:(.*)");
		Pattern scenarioOutlinePattern = Pattern.compile("^\\s*Scenario Outline:(.*)");
		
		// look for feature file into data folder
		File dir = Paths.get(srcPath, "data", application, "features").toFile();
		if (!dir.exists()) {
			return new ArrayList<String>();
		}

    	File[] featureFiles = dir.listFiles(new FilenameFilter() {
    		public boolean accept(File dir, String filename) { 
    			return filename.endsWith(".feature"); 
    		}
    	});
    	
    	List<String> scenarios = new ArrayList<String>();
    	for (File featureFile: featureFiles) {
    		try {
    			boolean exclude = false;
				for (String line: FileUtils.readLines(featureFile, "UTF-8")) {
					
					// exclude scenarios using tag @EXCLUDE_FROM_SQUASH_TA 
					if (line.contains(FEATURE_EXCLUDE)) {
						exclude = true;
					}
					
					Matcher matcher = scenarioPattern.matcher(line);
					if (matcher.matches()) {
						if (!exclude) {
							scenarios.add(matcher.group(1).trim());
						}
						exclude = false;
					}
					Matcher matcherOutline = scenarioOutlinePattern.matcher(line);
					if (matcherOutline.matches()) {
						if (!exclude) {
							scenarios.add(matcherOutline.group(1).trim());
						}
						exclude = false;
					}
				}
			} catch (IOException e) {
			}
    	}
    	return scenarios;
    	
	}
	
	/**
	 * Search for tests in TestNG files
	 * @param path
	 * @param application
	 * @return
	 */
	public List<SquashTaTestDef> parseTestNgXml() {
		
		// look for feature file into data folder
		File dir = Paths.get(srcPath, "data", application, "testng").toFile();
		if (!dir.exists()) {
			return new ArrayList<SquashTaTestDef>();
		}

    	File[] testngFiles = dir.listFiles(new FilenameFilter() {
    		public boolean accept(File dir, String filename) { 
    			return filename.endsWith(".xml"); 
    		}
    	});
    	
    	List<SquashTaTestDef> tests = new ArrayList<SquashTaTestDef>();
    	
    	for (File testngFile: testngFiles) {
    		
    		Document doc;
    		SAXBuilder sxb = new SAXBuilder();
    	    try {
    	        
    	        doc = sxb.build(testngFile);
    	    }
    	    catch(Exception e){
    	    	logger.error(String.format("Fichier %s illisible: %s", testngFile, e.getMessage()));
    	    	return tests;
    	    }
    	    
    	    Element suite = doc.getRootElement();
    	    if (!suite.getName().equalsIgnoreCase("suite")) {
    	    	continue;
    	    }
    	    
    	    for (Element test: suite.getChildren("test")) {
    	    	
    	    	boolean cucumberTest = false;
    	    	String cucumberNamedTest = "";
    	    	boolean exclude = false;
    	    	
    	    	// search cucumber parameters among test parameters
    	    	// does test specifies precise cucumber properties (cucumberTests / cucumberTags)
    	    	for (Element param: test.getChildren("parameter")) {<commit_msg>disable DTD check in TA parser<commit_after>package com.seleniumtests.util.squashta;


import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.filter.ElementFilter;
import org.jdom2.input.SAXBuilder;

import com.seleniumtests.reporter.TestLogging;

public class TaScriptGenerator {
	
	private static final Logger logger = TestLogging.getLogger(TaScriptGenerator.class);
	
	private String srcPath;
	private String dstPath;
	private String application;
	
	// patterns for exclusion from .ta script generation
	private static final String XML_EXCLUDE = "EXCLUDE_FROM_SQUASH_TA";
	private static final String FEATURE_EXCLUDE = "@EXCLUDE_FROM_SQUASH_TA";	

	/**
	 * 
	 * @param srcPath		Path to root of the test application. it's the location of the "data" folder
	 * @param dstPath		Path to root where files will be written
	 * @param application	name of the application
	 */
	public TaScriptGenerator(String application, String srcPath, String dstPath) {
		this.srcPath = srcPath.replace("\\", "/");
		this.dstPath = dstPath.replace("\\", "/");
		this.application = application;
	}
	
	/**
	 * Search for scenarios in feature files
	 * @param path
	 * @param application
	 * @return
	 */
	public List<String> parseFeatures() {
		
		Pattern scenarioPattern = Pattern.compile("^\\s*Scenario:(.*)");
		Pattern scenarioOutlinePattern = Pattern.compile("^\\s*Scenario Outline:(.*)");
		
		// look for feature file into data folder
		File dir = Paths.get(srcPath, "data", application, "features").toFile();
		if (!dir.exists()) {
			return new ArrayList<String>();
		}

    	File[] featureFiles = dir.listFiles(new FilenameFilter() {
    		public boolean accept(File dir, String filename) { 
    			return filename.endsWith(".feature"); 
    		}
    	});
    	
    	List<String> scenarios = new ArrayList<String>();
    	for (File featureFile: featureFiles) {
    		try {
    			boolean exclude = false;
				for (String line: FileUtils.readLines(featureFile, "UTF-8")) {
					
					// exclude scenarios using tag @EXCLUDE_FROM_SQUASH_TA 
					if (line.contains(FEATURE_EXCLUDE)) {
						exclude = true;
					}
					
					Matcher matcher = scenarioPattern.matcher(line);
					if (matcher.matches()) {
						if (!exclude) {
							scenarios.add(matcher.group(1).trim());
						}
						exclude = false;
					}
					Matcher matcherOutline = scenarioOutlinePattern.matcher(line);
					if (matcherOutline.matches()) {
						if (!exclude) {
							scenarios.add(matcherOutline.group(1).trim());
						}
						exclude = false;
					}
				}
			} catch (IOException e) {
			}
    	}
    	return scenarios;
    	
	}
	
	/**
	 * Search for tests in TestNG files
	 * @param path
	 * @param application
	 * @return
	 */
	public List<SquashTaTestDef> parseTestNgXml() {
		
		// look for feature file into data folder
		File dir = Paths.get(srcPath, "data", application, "testng").toFile();
		if (!dir.exists()) {
			return new ArrayList<SquashTaTestDef>();
		}

    	File[] testngFiles = dir.listFiles(new FilenameFilter() {
    		public boolean accept(File dir, String filename) { 
    			return filename.endsWith(".xml"); 
    		}
    	});
    	
    	List<SquashTaTestDef> tests = new ArrayList<SquashTaTestDef>();
    	
    	for (File testngFile: testngFiles) {
    		
    		Document doc;
    		SAXBuilder sxb = new SAXBuilder();
    		sxb.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
    	    try {
    	        
    	        doc = sxb.build(testngFile);
    	    }
    	    catch(Exception e){
    	    	logger.error(String.format("Fichier %s illisible: %s", testngFile, e.getMessage()));
    	    	return tests;
    	    }
    	    
    	    Element suite = doc.getRootElement();
    	    if (!suite.getName().equalsIgnoreCase("suite")) {
    	    	continue;
    	    }
    	    
    	    for (Element test: suite.getChildren("test")) {
    	    	
    	    	boolean cucumberTest = false;
    	    	String cucumberNamedTest = "";
    	    	boolean exclude = false;
    	    	
    	    	// search cucumber parameters among test parameters
    	    	// does test specifies precise cucumber properties (cucumberTests / cucumberTags)
    	    	for (Element param: test.getChildren("parameter")) {
{'fraction_non_alphanumeric': 0.11578604044357468, 'fraction_numerical': 0.0017395085888236572, 'mean_word_length': 2.8879966187658495, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3634121', 'n_tokens_mistral': 3223, 'n_tokens_neox': 2824, 'n_words': 669}