<commit_before>package dk.aau.cs.qweb.airbase;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

import dk.aau.cs.qweb.airbase.database.Database;
import dk.aau.cs.qweb.airbase.input.FileStructure;
import dk.aau.cs.qweb.airbase.types.CubeStructure;
import dk.aau.cs.qweb.airbase.types.TripleContainer;
import dk.aau.cs.qweb.airbase.types.Tuple;

public class App {

	public static void main(String[] args) {
		
		CommandLineParser parser = new DefaultParser();
		CubeStructure structure = null;
		List<String> files = new ArrayList<String>();
		// create the Options
		Options options = new Options();
		options.addOption("h", "help", false, "Display this message." );
		options.addOption("c", "config", true, "path to config file");
				
		try {
		    CommandLine line = parser.parse( options, args );
				    
		    if (line.hasOption( "help" )) {
		    	printHelp(null,options);
		    	System.exit(0);
			} 
			
		    if (line.hasOption("config")) {
		    	try (BufferedReader br = new BufferedReader(new FileReader(line.getOptionValue("config")))) {

					String fileLine;
					while ((fileLine = br.readLine()) != null) {
						if (fileLine.startsWith("cube-structure")) {
							Config.setCubeStructurePath(fileLine.split(" ")[1]);
						}
						else if (fileLine.startsWith("input-file")) {
							files.add(fileLine.split(" ")[1]);
						}
					}
				}
		    }
		}
		catch( ParseException exp ) {
			printHelp(exp, options);
		} 
		catch (Exception exp) {
			exp.printStackTrace();
		}
		
		Database dbConnection = new Database();
		
		for (String file : files) {
			
			FileStructure fileStructure;
			try {
				fileStructure = new FileStructure(file);
				while (fileStructure.hasNext()) {
					Tuple tuple = (Tuple) fileStructure.next();
					System.out.println(tuple);
					TripleContainer triples = new TripleContainer(tuple);
					
					
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	
	private static void printHelp(ParseException exp, Options options) {
		String header = "";
		HelpFormatter formatter = new HelpFormatter();
		if (exp != null) {
			header = "Unexpected exception:" + exp.getMessage();
		}
		formatter.printHelp("Provenance Enabled Cubes App", header, options, null, true);
	}

}
<commit_msg>clean up and call the db connection to write triples to disk. the db connection is still not implemented<commit_after>package dk.aau.cs.qweb.airbase;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

import dk.aau.cs.qweb.airbase.database.Database;
import dk.aau.cs.qweb.airbase.input.FileStructure;
import dk.aau.cs.qweb.airbase.types.TripleContainer;
import dk.aau.cs.qweb.airbase.types.Tuple;

public class App {

	public static void main(String[] args) {
		
		CommandLineParser parser = new DefaultParser();
		List<String> files = new ArrayList<String>();
		// create the Options
		Options options = new Options();
		options.addOption("h", "help", false, "Display this message." );
		options.addOption("c", "config", true, "path to config file");
				
		try {
		    CommandLine line = parser.parse( options, args );
				    
		    if (line.hasOption( "help" )) {
		    	printHelp(null,options);
		    	System.exit(0);
			} 
			
		    if (line.hasOption("config")) {
		    	try (BufferedReader br = new BufferedReader(new FileReader(line.getOptionValue("config")))) {

					String fileLine;
					while ((fileLine = br.readLine()) != null) {
						if (fileLine.startsWith("cube-structure")) {
							Config.setCubeStructurePath(fileLine.split(" ")[1]);
						}
						else if (fileLine.startsWith("input-file")) {
							files.add(fileLine.split(" ")[1]);
						}
					}
				}
		    }
		}
		catch( ParseException exp ) {
			printHelp(exp, options);
		} 
		catch (Exception exp) {
			exp.printStackTrace();
		}
		
		Database dbConnection = new Database();
		
		for (String file : files) {
			System.out.println(file);
			Config.setCurrentInputFilePath(file);
			FileStructure fileStructure;
			try {
				fileStructure = new FileStructure(file);
				while (fileStructure.hasNext()) {
					Tuple tuple = (Tuple) fileStructure.next();
					TripleContainer triples = new TripleContainer(tuple);
					
					dbConnection.writeToDisk(triples);
					
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	
	private static void printHelp(ParseException exp, Options options) {
		String header = "";
		HelpFormatter formatter = new HelpFormatter();
		if (exp != null) {
			header = "Unexpected exception:" + exp.getMessage();
		}
		formatter.printHelp("Provenance Enabled Cubes App", header, options, null, true);
	}

}

{'fraction_non_alphanumeric': 0.1311413454270597, 'fraction_numerical': 0.0011337868480725624, 'mean_word_length': 3.49320882852292, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '519715', 'n_tokens_mistral': 1954, 'n_tokens_neox': 1635, 'n_words': 361}