<commit_before>/***************************************************************************
 *
 *   JavaAnalyser.java
 *   -------------------
 *   copyright            : (c) 2009 by PureSol-Technologies
 *   author               : <NAME>
 *   email                : <EMAIL>
 *
 ***************************************************************************/

package com.puresol.coding.lang.test;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.log4j.Logger;

import com.puresol.coding.CodeRange;
import com.puresol.coding.CodeRangeType;
import com.puresol.coding.ProgrammingLanguage;
import com.puresol.coding.analysis.AnalyzerException;
import com.puresol.coding.analysis.Analyzer;
import com.puresol.coding.lang.test.grammar.TestLanguageGrammar;
import com.puresol.uhura.lexer.Lexer;
import com.puresol.uhura.lexer.LexerException;
import com.puresol.uhura.lexer.LexerResult;
import com.puresol.uhura.parser.Parser;
import com.puresol.uhura.parser.ParserException;
import com.puresol.uhura.parser.ParserTree;
import com.puresol.utils.Persistence;
import com.puresol.utils.PersistenceException;

/**
 * 
 * @author <NAME>
 * 
 */
public class TestLanguageAnalyser implements Analyzer {

	private static final long serialVersionUID = -3601131473616977648L;

	private static final Logger logger = Logger
			.getLogger(TestLanguageAnalyser.class);

	private final File file;
	private final transient TestLanguageGrammar grammar;
	private Date date = new Date();
	private ParserTree parserTree = null;

	public TestLanguageAnalyser(File file) {
		super();
		this.file = file;
		grammar = TestLanguageGrammar.getInstance();
	}

	@Override
	public void parse() throws AnalyzerException {
		try {
			date = new Date();
			Lexer lexer = grammar.getLexer();
			LexerResult lexerResult = lexer.lex(new FileReader(file),
					file.toString());
			Parser parser = grammar.getParser();
			parserTree = parser.parse(lexerResult);
		} catch (ParserException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);
		} catch (IOException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);
		} catch (LexerException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);<commit_msg>- introduce new stage to lexing procedure: source code reading into a SourceCode   object with SourceCodeLine as list to support pre processing of different files   into a single source code - adopted all related classes to the new stage<commit_after>/***************************************************************************
 *
 *   JavaAnalyser.java
 *   -------------------
 *   copyright            : (c) 2009 by PureSol-Technologies
 *   author               : <NAME>
 *   email                : <EMAIL>
 *
 ***************************************************************************/

package com.puresol.coding.lang.test;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.log4j.Logger;

import com.puresol.coding.CodeRange;
import com.puresol.coding.CodeRangeType;
import com.puresol.coding.ProgrammingLanguage;
import com.puresol.coding.analysis.AnalyzerException;
import com.puresol.coding.analysis.Analyzer;
import com.puresol.coding.lang.test.grammar.TestLanguageGrammar;
import com.puresol.uhura.lexer.Lexer;
import com.puresol.uhura.lexer.LexerException;
import com.puresol.uhura.lexer.LexerResult;
import com.puresol.uhura.lexer.SourceCode;
import com.puresol.uhura.parser.Parser;
import com.puresol.uhura.parser.ParserException;
import com.puresol.uhura.parser.ParserTree;
import com.puresol.utils.Persistence;
import com.puresol.utils.PersistenceException;

/**
 * 
 * @author <NAME>
 * 
 */
public class TestLanguageAnalyser implements Analyzer {

	private static final long serialVersionUID = -3601131473616977648L;

	private static final Logger logger = Logger
			.getLogger(TestLanguageAnalyser.class);

	private final File file;
	private final transient TestLanguageGrammar grammar;
	private Date date = new Date();
	private ParserTree parserTree = null;

	public TestLanguageAnalyser(File file) {
		super();
		this.file = file;
		grammar = TestLanguageGrammar.getInstance();
	}

	@Override
	public void parse() throws AnalyzerException {
		try {
			date = new Date();
			Lexer lexer = grammar.getLexer();
			LexerResult lexerResult = lexer.lex(SourceCode.read(file),
					file.toString());
			Parser parser = grammar.getParser();
			parserTree = parser.parse(lexerResult);
		} catch (ParserException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);
		} catch (IOException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);
		} catch (LexerException e) {
			logger.error(e.getMessage(), e);
			throw new AnalyzerException(this);
{'fraction_non_alphanumeric': 0.16643159379407615, 'fraction_numerical': 0.009671569615152125, 'mean_word_length': 4.378114842903575, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4335141', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1467, 'n_words': 331}