<commit_before>package info.iconmaster.typhon;

import java.io.File;
import java.io.IOException;
import java.util.Scanner;
import java.util.regex.Pattern;

import info.iconmaster.typhon.compiler.TyphonSourceReader;
import info.iconmaster.typhon.errors.TyphonError;
import info.iconmaster.typhon.language.Package;
import info.iconmaster.typhon.util.CommandLineHelper.Result;
import info.iconmaster.typhon.util.CommandLineHelper.UnknownOptionException;
import info.iconmaster.typhon.util.FileUtils;

public class Typhon {
	public static final String VERSION = "@TYPHON_VERSION@";
	
	public static void main(String[] args) {
		try {
			Result options = TyphonCommandLine.CMD_PARSER.parseCommandLine(args);
			
			if (options.optionalArguments.containsKey(TyphonCommandLine.OPTION_HELP)) {
				TyphonCommandLine.CMD_PARSER.printUsage(System.out);
				return;
			}
			
			if (options.optionalArguments.containsKey(TyphonCommandLine.OPTION_VERSION)) {
				System.out.println(VERSION);
				return;
			}
			
			if (options.positionalArguments.size() == 0) {
				System.err.println("error: no input files specified");
				System.err.println();
				TyphonCommandLine.CMD_PARSER.printUsage(System.err);
				return;
			}
			
			TyphonInput tni = new TyphonInput();
			for (String fileName : options.positionalArguments) {
				File file = new File(fileName);
				if (!file.exists()) {
					System.err.println("error: file or folder '"+fileName+"' does not exist");
					return;
				}
				tni.inputFiles.addAll(FileUtils.getAllFiles(file, FileUtils.FILTER_TYPHON_FILES));
			}
			
			for (File file : tni.inputFiles) {
				Package p;
				try {
					p = TyphonSourceReader.parseFile(tni, file);
					tni.inputPackages.add(p);
				} catch (IOException e) {
					System.err.println("error: cannot read input file '"+file.getName()+"': "+e.getMessage());
					return;
				}
			}
			
			// check for errors
			
			if (!tni.errors.isEmpty()) {
				for (TyphonError error : tni.errors) {
					System.err.println(error);
					
					File file = new File(error.source.file);
					if (error.source.file != null && file.exists()) {
						try {
							//String text = new String(Files.readAllBytes(file.toPath()));
							//System.out.println(text.substring(40,42));
							
							Scanner scanner = new Scanner(file);
							scanner.useDelimiter("[\r\n]");
							Pattern pattern = Pattern.compile("[^\r\n]*[\r\n]*", Pattern.DOTALL | Pattern.MULTILINE);
							int offset = 0;
							
							while (scanner.hasNext()) {
								String line = scanner.next();
								offset += line.length() + 1;
								
								if (offset > error.source.begin) {
									System.err.println("\t"+line.replace('\t', ' '));
									
									int curPos = offset-error.source.begin;
									System.err.print('\t');
									for (int i = 0; i < curPos; i++) {
										System.err.print(' ');
									}
									System.err.println('^');
									
									break;
								}
							}
						} catch (IOException e) {
							// ignore; we don't need location information THAT badly
						}<commit_msg>Small alteration<commit_after>package info.iconmaster.typhon;

import java.io.File;
import java.io.IOException;
import java.util.Scanner;
import java.util.regex.Pattern;

import info.iconmaster.typhon.compiler.TyphonSourceReader;
import info.iconmaster.typhon.errors.TyphonError;
import info.iconmaster.typhon.language.Package;
import info.iconmaster.typhon.util.CommandLineHelper.Result;
import info.iconmaster.typhon.util.CommandLineHelper.UnknownOptionException;
import info.iconmaster.typhon.util.FileUtils;

public class Typhon {
	public static final String VERSION = "@TYPHON_VERSION@";
	
	public static void main(String[] args) {
		try {
			Result options = TyphonCommandLine.CMD_PARSER.parseCommandLine(args);
			
			if (options.optionalArguments.containsKey(TyphonCommandLine.OPTION_HELP)) {
				TyphonCommandLine.CMD_PARSER.printUsage(System.out);
				return;
			}
			
			if (options.optionalArguments.containsKey(TyphonCommandLine.OPTION_VERSION)) {
				System.out.println(VERSION);
				return;
			}
			
			if (options.positionalArguments.size() == 0) {
				System.err.println("error: no input files specified");
				System.err.println();
				TyphonCommandLine.CMD_PARSER.printUsage(System.err);
				return;
			}
			
			TyphonInput tni = new TyphonInput();
			for (String fileName : options.positionalArguments) {
				File file = new File(fileName);
				if (!file.exists()) {
					System.err.println("error: file or folder '"+fileName+"' does not exist");
					return;
				}
				tni.inputFiles.addAll(FileUtils.getAllFiles(file, FileUtils.FILTER_TYPHON_FILES));
			}
			
			for (File file : tni.inputFiles) {
				Package p;
				try {
					p = TyphonSourceReader.parseFile(tni, file);
					tni.inputPackages.add(p);
				} catch (IOException e) {
					System.err.println("error: cannot read input file '"+file.getName()+"': "+e.getMessage());
					return;
				}
			}
			
			// check for errors
			
			if (!tni.errors.isEmpty()) {
				for (TyphonError error : tni.errors) {
					System.err.println(error);
					
					File file = new File(error.source.file);
					if (error.source.file != null && file.exists()) {
						try {
							//String text = new String(Files.readAllBytes(file.toPath()));
							//System.out.println(text.substring(40,42));
							
							Scanner scanner = new Scanner(file);
							scanner.useDelimiter("[\r\n]");
							int offset = 0;
							
							while (scanner.hasNext()) {
								String line = scanner.next();
								offset += line.length() + 1;
								
								if (offset > error.source.begin) {
									System.err.println("\t"+line.replace('\t', ' '));
									
									int curPos = offset-error.source.begin;
									System.err.print('\t');
									for (int i = 0; i < curPos; i++) {
										System.err.print(' ');
									}
									System.err.println('^');
									
									break;
								}
							}
						} catch (IOException e) {
							// ignore; we don't need location information THAT badly
						}
{'fraction_non_alphanumeric': 0.135018290655138, 'fraction_numerical': 0.002660458929165281, 'mean_word_length': 2.8190476190476192, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6030939', 'n_tokens_mistral': 2579, 'n_tokens_neox': 1859, 'n_words': 358}