<commit_before>package ee.stacc.productivity.edsl.main;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
import java.util.Map;
import java.util.Scanner;

import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jdt.core.IJavaElement;
import org.eclipse.jdt.core.IJavaProject;
import org.eclipse.jdt.core.JavaCore;
import org.junit.Test;

import ee.stacc.productivity.edsl.checkers.INodeDescriptor;
import ee.stacc.productivity.edsl.checkers.IStringNodeDescriptor;
import ee.stacc.productivity.edsl.crawler.PositionUtil;
import ee.stacc.productivity.edsl.crawler.UnsupportedNodeDescriptor;

public class JavaElementCheckerTest {
	JavaElementChecker checker = new JavaElementChecker();
	IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
	
	public JavaElementCheckerTest()  {
		
	}
	
	@Test
	public void testEDSLTestProjectStrings() throws IOException, CoreException {
		testJavaElementAbstractStrings("EDSLTestProject", "");
	}
	
	@Test
	public void testEArvedStrings() throws IOException, CoreException {
		testJavaElementAbstractStrings("earved", "src");
	}
	
	/*@Test
	public void testCompiereString() throws IOException, CoreException {
		testJavaElementAbstractStrings("compiere", "");
	}
	*/
	
	private void testJavaElementAbstractStrings(String projectName, 
			String packageFragmentRoot) throws IOException, CoreException {
		
		IProject project = root.getProject(projectName);
		
		if (!project.isOpen()) {
			project.open(null);
		}
		
		IJavaProject javaProject = (IJavaProject)project.getNature(JavaCore.NATURE_ID);
		
		if (packageFragmentRoot.isEmpty()) {
			testJavaElementAbstractStrings(javaProject);
		}
		else {
			testJavaElementAbstractStrings(javaProject.findPackageFragmentRoot
					(javaProject.getPath().append(packageFragmentRoot)));
		}
	}
	
	private void testJavaElementAbstractStrings(IJavaElement element) throws IOException {
		Map<String, Object> options = OptionLoader.getElementSqlCheckerProperties(element);
		List<INodeDescriptor> hotspots = checker.findHotspots(new IJavaElement[] {element}, options);
		
		File outputFile = element.getResource().getLocation().append("FoundAbstractStrings.txt").toFile();
		if (outputFile.exists()) {
			outputFile.delete();
		}
		
		PrintStream output = new PrintStream(outputFile);
		
		
		for (INodeDescriptor desc : hotspots) {
			output.print(PositionUtil.getLineString(desc.getPosition()) + ", ");
			if (desc instanceof IStringNodeDescriptor) {
				output.println(((IStringNodeDescriptor)desc).getAbstractValue());
			}
			else if (desc instanceof UnsupportedNodeDescriptor) {
				output.println("unsupported: " 
						+ ((UnsupportedNodeDescriptor)desc).getProblemMessage());
			}
			else {
				output.println();
			}
		}
		
		File expectedFile = element.getResource().getLocation()
			.append("ExpectedAbstractStrings.txt").toFile();
		
		assertTrue("Expected abstract strings != found abstract strings: " + expectedFile.getParent(), 
					filesAreEqual(outputFile, expectedFile));

	}
	
	boolean filesAreEqual(File a, File b) throws FileNotFoundException {
		assert a.exists() && b.exists();
		
		Scanner scA = new Scanner(a);
		Scanner scB = new Scanner(b);
		
		while (scA.hasNextLine()) {
			if (!scB.hasNextLine() || ! scA.nextLine().equals(scB.nextLine())) {
				return false;
			}
		}
		
		return true;
	}
}
<commit_msg>sorting<commit_after>package ee.stacc.productivity.edsl.main;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Scanner;

import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jdt.core.IJavaElement;
import org.eclipse.jdt.core.IJavaProject;
import org.eclipse.jdt.core.JavaCore;
import org.junit.Test;

import ee.stacc.productivity.edsl.checkers.INodeDescriptor;
import ee.stacc.productivity.edsl.checkers.IStringNodeDescriptor;
import ee.stacc.productivity.edsl.crawler.PositionUtil;
import ee.stacc.productivity.edsl.crawler.UnsupportedNodeDescriptor;

public class JavaElementCheckerTest {
	JavaElementChecker checker = new JavaElementChecker();
	IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
	
	public JavaElementCheckerTest()  {
		
	}
	
	@Test
	public void testEDSLTestProjectStrings() throws IOException, CoreException {
		testJavaElementAbstractStrings("EDSLTestProject", "");
	}
	
	@Test
	public void testEArvedStrings() throws IOException, CoreException {
		testJavaElementAbstractStrings("earved", "src");
	}
	
	/*@Test
	public void testCompiereString() throws IOException, CoreException {
		testJavaElementAbstractStrings("compiere", "");
	}
	*/
	
	private void testJavaElementAbstractStrings(String projectName, 
			String packageFragmentRoot) throws IOException, CoreException {
		
		IProject project = root.getProject(projectName);
		
		if (!project.isOpen()) {
			project.open(null);
		}
		
		IJavaProject javaProject = (IJavaProject)project.getNature(JavaCore.NATURE_ID);
		
		if (packageFragmentRoot.isEmpty()) {
			testJavaElementAbstractStrings(javaProject);
		}
		else {
			testJavaElementAbstractStrings(javaProject.findPackageFragmentRoot
					(javaProject.getPath().append(packageFragmentRoot)));
		}
	}
	
	private void testJavaElementAbstractStrings(IJavaElement element) throws IOException {
		Map<String, Object> options = OptionLoader.getElementSqlCheckerProperties(element);
		List<INodeDescriptor> hotspots = checker.findHotspots(new IJavaElement[] {element}, options);
		
		File outputFile = element.getResource().getLocation().append("FoundAbstractStrings.txt").toFile();
		if (outputFile.exists()) {
			outputFile.delete();
		}
		
		List<String> lines = new ArrayList<String>();
		
		for (INodeDescriptor desc : hotspots) {
			String start = PositionUtil.getLineString(desc.getPosition()) + ", ";
			
			if (desc instanceof IStringNodeDescriptor) {
				lines.add(start + 
						((IStringNodeDescriptor)desc).getAbstractValue().toString());
			}
			else if (desc instanceof UnsupportedNodeDescriptor) {
				lines.add(start + "unsupported: " 
						+ ((UnsupportedNodeDescriptor)desc).getProblemMessage());
			}
			else {
				lines.add("???");
			}
		}
		
		//Collections.sort(lines);
		PrintStream output = new PrintStream(outputFile);
		
		for (String s : lines) {
			output.println(s);
		}
		
		File expectedFile = element.getResource().getLocation()
			.append("ExpectedAbstractStrings.txt").toFile();
		
		assertTrue("Expected abstract strings != found abstract strings: " + expectedFile.getParent(), 
					filesAreEqual(outputFile, expectedFile));

	}
	
	boolean filesAreEqual(File a, File b) throws FileNotFoundException {
		assert a.exists() && b.exists();
		
		Scanner scA = new Scanner(a);
		Scanner scB = new Scanner(b);
		
		while (scA.hasNextLine()) {
			if (!scB.hasNextLine() || ! scA.nextLine().equals(scB.nextLine())) {
				return false;
			}
		}
		
		return true;
	}
}

{'fraction_non_alphanumeric': 0.11240913811007269, 'fraction_numerical': 0.0, 'mean_word_length': 4.220189701897019, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2659619', 'n_tokens_mistral': 2541, 'n_tokens_neox': 2268, 'n_words': 398}