<commit_before>package gr.ntua.cslab.asap.workflow;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
import java.util.logging.Logger;

//import net.sourceforge.jeval.EvaluationException;
//import net.sourceforge.jeval.Evaluator;
import gr.ntua.cslab.asap.staticLibraries.AbstractOperatorLibrary;
import gr.ntua.cslab.asap.staticLibraries.DatasetLibrary;
import gr.ntua.cslab.asap.staticLibraries.MaterializedWorkflowLibrary;
import gr.ntua.cslab.asap.staticLibraries.OperatorLibrary;
import gr.ntua.cslab.asap.operators.AbstractOperator;
import gr.ntua.cslab.asap.operators.Dataset;
import gr.ntua.cslab.asap.operators.MaterializedOperators;
import gr.ntua.cslab.asap.operators.NodeName;
import gr.ntua.cslab.asap.operators.Operator;
import gr.ntua.cslab.asap.rest.beans.OperatorDictionary;
import gr.ntua.cslab.asap.rest.beans.WorkflowDictionary;

public class AbstractWorkflow1 {
	private List<WorkflowNode> targets;
	private List<WorkflowNode> abstractInputs;
	private HashMap<String,WorkflowNode> workflowNodes;
	public String name, directory;
	private static Logger logger = Logger.getLogger(AbstractWorkflow1.class.getName());

	private HashMap<String, WorkflowNode> materilizedDatasets;
	public HashMap<String,String> groupInputs;
	public String optimizationFunction;
	public String functionTarget;
	private String policy;

	@Override
	public String toString() {
		return targets.toString();
	}

	public AbstractWorkflow1(String name) {
		this.directory = "/tmp";
		this.name=name;
		targets = new ArrayList<WorkflowNode>();
		workflowNodes = new HashMap<String, WorkflowNode>();
		materilizedDatasets = new HashMap<String, WorkflowNode>();
	}

	public AbstractWorkflow1(String name, String directory) {
		this.directory = directory;
		this.name=name;
		targets = new ArrayList<WorkflowNode>();
		workflowNodes = new HashMap<String, WorkflowNode>();
	}

	public void addTarget(WorkflowNode target) {
		targets.add(target);
	}

	public void addTargets(List<WorkflowNode> targets) {
		this.targets.addAll(targets);
	}

	public List<WorkflowNode> getTargets() {
		return targets;
	}

	public MaterializedWorkflow1 materialize(String metric, String nameExtention, String policy) throws Exception {
		OperatorLibrary.moveid=0;
		parsePolicy(policy);
		String fullName=name+"_"+nameExtention;
		MaterializedWorkflow1 materializedWorkflow = new MaterializedWorkflow1(fullName, MaterializedWorkflowLibrary.getWorkflowDirectory()+"/"+fullName);
		if(materilizedDatasets!=null)
			materializedWorkflow.materilizedDatasets=materilizedDatasets;
		else
			materializedWorkflow.materilizedDatasets=new HashMap<>();
		materializedWorkflow.setAbstractWorkflow(this);
		materializedWorkflow.setPolicy(groupInputs, optimizationFunction, functionTarget);
		Workflow1DPTable dpTable = new Workflow1DPTable();
		WorkflowNode temp = null;
		Double bestCost = 0.0;
		Double tempCost = 0.0;
		List<WorkflowNode> bestPlan=null;

		for(WorkflowNode t : targets){
			logger.info( "Materializing workflow node: " + t.toStringNorecursive());
			List<WorkflowNode> l = t.materialize(metric, materializedWorkflow,dpTable,t.getName());
			/* vpapa: assert that WorkflowNode.materialize() returned something
				valid
			*/
			if( l != null && !l.isEmpty()){
				if(functionTarget.contains("min")){
					bestCost=Double.MAX_VALUE;
					for(WorkflowNode r : l){
						tempCost = dpTable.getCost(r.dataset);
						if(tempCost<bestCost){
							bestCost=tempCost;
							bestPlan=dpTable.getPlan(r.dataset);
						}
					}
				}
				else if(functionTarget.contains("max")){
					bestCost = -Double.MAX_VALUE;
					for(WorkflowNode r : l){
						tempCost = dpTable.getCost(r.dataset);
						if(tempCost>bestCost){
							bestCost=tempCost;
							bestPlan=dpTable.getPlan(r.dataset);
						}
					}
				}
				/* vpapa: target may have or may have not a dataset
				*/
				if( t.dataset != null){
					/* vpapa: temp below is going to be the next WorkflowNode
						having as materialized dataset the dataset of the current
						target WorkflowNode and as input the current target itself
					*/
					temp = new WorkflowNode(false, false, t.getName());
					temp.setDataset( t.dataset);
					//System.out.println(l);
					temp.addInputs(l);
					materializedWorkflow.addTarget(temp);
				}
				else{
					/* vpapa: in the absence of a dataset still the operator should
						be added to the workflow
					*/
					temp = l.get( 0).inputs.get( 0);
					temp.setDataset( l.get( 0).dataset);
					materializedWorkflow.addTarget(temp);
				}
				bestPlan.add(t);
				materializedWorkflow.setBestPlan(t.toStringNorecursive(), bestPlan);
				logger.info("Optimal cost: "+bestCost);
				materializedWorkflow.optimalCost=bestCost;
			}
		}
		return materializedWorkflow;
	}// end of AbstractWorkflow1 materialize


	public MaterializedWorkflow1 replan(
			HashMap<String, WorkflowNode> materilizedDatasets) throws Exception {
		this.materilizedDatasets=materilizedDatasets;
		return materialize("execTime", "", policy);
	}



	public String parsePolicy(String policy) {
		this.policy=policy;
		groupInputs = new HashMap<String, String>();
		String[] p = policy.split("\n");
		for (int i = 0; i < p.length; i++) {
			String[] p1 = p[i].split(",");
			if(p1[0].equals("groupInputs")){
				groupInputs.put(p1[1], p1[2]);
			}
			else if(p1[0].equals("function")){
				optimizationFunction=p1[1];
				functionTarget=p1[2];
			}
		}
		return optimizationFunction;
		//System.out.println(functionTarget);
	}

	public static String getPolicyFromString(String policy) {
		String[] p = policy.split("\n");
		for (int i = 0; i < p.length; i++) {
			String[] p1 = p[i].split(",");
			if(p1[0].equals("function")){
				return p1[1];
			}
		}
		return "";
		//System.out.println(functionTarget);
	}

	public void writeToDir(String directory) throws Exception {

		for(WorkflowNode t : targets){
			t.setAllNotVisited();
		}

        File workflowDir = new File(directory);
        if (!workflowDir.exists()) {
        	workflowDir.mkdirs();
        }
        File opDir = new File(directory+"/operators");
        if (!opDir.exists()) {
        	opDir.mkdirs();
        }
        File datasetDir = new File(directory+"/datasets");
        if (!datasetDir.exists()) {
        	datasetDir.mkdirs();
        }
        File edgeGraph = new File(directory+"/graph");
    	FileOutputStream fos = new FileOutputStream(edgeGraph);

    	BufferedWriter graphWritter = new BufferedWriter(new OutputStreamWriter(fos));


		for(WorkflowNode t : targets){
			t.writeToDir(directory+"/operators",directory+"/datasets",graphWritter);
			graphWritter.write(t.toStringNorecursive() +",$$target");
			graphWritter.newLine();
		}

		graphWritter.close();
	}


	public void readFromDir(String directory) throws IOException {
		HashMap<String,WorkflowNode> nodes = new HashMap<String, WorkflowNode>();
		File folder = new File(directory+"/operators");
		File[] files = folder.listFiles();

		for (int i = 0; i < files.length; i++) {
			if (files[i].isFile() && !files[i].isHidden()) {
				WorkflowNode n = new WorkflowNode(true, true,"");
				AbstractOperator temp = new AbstractOperator(files[i].getName());
				temp.readPropertiesFromFile(files[i]);
				n.setAbstractOperator(temp);
				nodes.put(temp.opName, n);
			}
		}
		folder = new File(directory+"/datasets");
		/* vpapa: read only if datasets folder exists and it has content */
		if( folder.exists()){
			files = folder.listFiles();
			if( files != null && files.length > 0){
				for (int i = 0; i < files.length; i++) {
					if (files[i].isFile() && !files[i].isHidden()) {
						WorkflowNode n =null;
						Dataset temp = new Dataset(files[i].getName());
						temp.readPropertiesFromFile(files[i]);
						int metadatsize =temp.datasetTree.tree.size();
						if(metadatsize==0){
							n = new WorkflowNode(false, true,"");
						}
						else{
							n = new WorkflowNode(false, false, "");
						}
						n.setDataset(temp);
						nodes.put(temp.datasetName, n);
					}
				}
			}
		}
		//putting nodes into workflowNodes make them available for printing at IReS WUI
		workflowNodes.putAll(nodes);
		File edgeGraph = new File(directory+"/graph");
		FileInputStream fis = new FileInputStream(edgeGraph);
		BufferedReader br = new BufferedReader(new InputStreamReader(fis));

		String line = null;
		String[] e	= null;
		WorkflowNode src = null;
		WorkflowNode dest = null;
		/* vpapa: operators or datasets defined in graph file may be missing from
			the corresponding folders or misswritten into abstract workflow's graph
			file
		*/
		try{
			while ((line = br.readLine()) != null) {
				e = line.split(",");
				if(e[1].equals("$$target")){
					this.targets.add(nodes.get(e[0]));
				}
				else if(e.length==2){
					src = nodes.get(e[0]);
					dest = nodes.get(e[1]);
					dest.addInput(src);
					src.addOutput(dest);
				}
				else if(e.length==3){
					src = nodes.get(e[0]);
					dest = nodes.get(e[1]);
					if(dest.isOperator){
						dest.addInput(Integer.parseInt(e[2]), src);
						src.addOutput(dest);
					}
					else{
						dest.addInput(src);
						src.addOutput(Integer.parseInt(e[2]), dest);
					}
				}
			}
		}
		catch( NullPointerException npe){
			System.out.println( "ERROR: The corresponding files of operators or datasets"
								+ " " + e[ 0] + " and " + e[ 1] + " it looks like"
								+ " that they are missing from the relative folders"
								+ "or miswritten into abstract workflow's graph file.");
			logger.info( "ERROR: The corresponding files of operators or datasets"
								+ " " + e[ 0] + " and " + e[ 1] + " it looks like"
								+ " that they are missing from the relative folders"
								+ "or miswritten into abstract workflow's graph file.");
		}
        catch( IndexOutOfBoundsException iobe){
             System.out.println( "ERROR: Destination: " + dest + " has " + src + "as source"
             					 + " at its input with number " + e[ 2] + ". However, " + dest
             					 + " has " + inputs.size() + " inputs which is less than " + e[ 2] + "."
             					 + " Make sure that the graph file is set appropriately of the"
             					 + " corresponding workflow for this operator i.e. " + dest + ".");
             logger.info( "ERROR: Destination: " + dest + " has " + src + "as source"
 					 + " at its input with number " + e[ 2] + ". However, " + dest
 					 + " has " + inputs.size() + " inputs which is less than " + e[ 2] + "."
 					 + " Make sure that the graph file is set appropriately of the"
 					 + " corresponding workflow for this operator i.e. " + dest + ".");
        }
		br.close();
	}


	public void refresh() {
		for(Entry<String, WorkflowNode> e : workflowNodes.entrySet()){
			if(e.getValue().isOperator){
				AbstractOperator abstractOp = AbstractOperatorLibrary.getOperator(e.getKey());
				workflowNodes.get(e.getKey()).abstractOperator=abstractOp;
			}
			else if(!e.getValue().isAbstract){
				Dataset d = DatasetLibrary.getDataset(e.getKey());
				workflowNodes.get(e.getKey()).dataset=d;
			}
		}

	}


	public String graphToString() throws IOException {
		ByteArrayOutputStream bs = new ByteArrayOutputStream();
		BufferedWriter graphWritter = new BufferedWriter(new OutputStreamWriter(bs));
<commit_msg>Update big workflows for wind<commit_after>package gr.ntua.cslab.asap.workflow;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
import java.util.logging.Logger;

//import net.sourceforge.jeval.EvaluationException;
//import net.sourceforge.jeval.Evaluator;
import gr.ntua.cslab.asap.staticLibraries.AbstractOperatorLibrary;
import gr.ntua.cslab.asap.staticLibraries.DatasetLibrary;
import gr.ntua.cslab.asap.staticLibraries.MaterializedWorkflowLibrary;
import gr.ntua.cslab.asap.staticLibraries.OperatorLibrary;
import gr.ntua.cslab.asap.operators.AbstractOperator;
import gr.ntua.cslab.asap.operators.Dataset;
import gr.ntua.cslab.asap.operators.MaterializedOperators;
import gr.ntua.cslab.asap.operators.NodeName;
import gr.ntua.cslab.asap.operators.Operator;
import gr.ntua.cslab.asap.rest.beans.OperatorDictionary;
import gr.ntua.cslab.asap.rest.beans.WorkflowDictionary;

public class AbstractWorkflow1 {
	private List<WorkflowNode> targets;
	private List<WorkflowNode> abstractInputs;
	private HashMap<String,WorkflowNode> workflowNodes;
	public String name, directory;
	private static Logger logger = Logger.getLogger(AbstractWorkflow1.class.getName());

	private HashMap<String, WorkflowNode> materilizedDatasets;
	public HashMap<String,String> groupInputs;
	public String optimizationFunction;
	public String functionTarget;
	private String policy;

	@Override
	public String toString() {
		return targets.toString();
	}

	public AbstractWorkflow1(String name) {
		this.directory = "/tmp";
		this.name=name;
		targets = new ArrayList<WorkflowNode>();
		workflowNodes = new HashMap<String, WorkflowNode>();
		materilizedDatasets = new HashMap<String, WorkflowNode>();
	}

	public AbstractWorkflow1(String name, String directory) {
		this.directory = directory;
		this.name=name;
		targets = new ArrayList<WorkflowNode>();
		workflowNodes = new HashMap<String, WorkflowNode>();
	}

	public void addTarget(WorkflowNode target) {
		targets.add(target);
	}

	public void addTargets(List<WorkflowNode> targets) {
		this.targets.addAll(targets);
	}

	public List<WorkflowNode> getTargets() {
		return targets;
	}

	public MaterializedWorkflow1 materialize(String metric, String nameExtention, String policy) throws Exception {
		OperatorLibrary.moveid=0;
		parsePolicy(policy);
		String fullName=name+"_"+nameExtention;
		MaterializedWorkflow1 materializedWorkflow = new MaterializedWorkflow1(fullName, MaterializedWorkflowLibrary.getWorkflowDirectory()+"/"+fullName);
		if(materilizedDatasets!=null)
			materializedWorkflow.materilizedDatasets=materilizedDatasets;
		else
			materializedWorkflow.materilizedDatasets=new HashMap<>();
		materializedWorkflow.setAbstractWorkflow(this);
		materializedWorkflow.setPolicy(groupInputs, optimizationFunction, functionTarget);
		Workflow1DPTable dpTable = new Workflow1DPTable();
		WorkflowNode temp = null;
		Double bestCost = 0.0;
		Double tempCost = 0.0;
		List<WorkflowNode> bestPlan=null;

		for(WorkflowNode t : targets){
			logger.info( "Materializing workflow node: " + t.toStringNorecursive());
			List<WorkflowNode> l = t.materialize(metric, materializedWorkflow,dpTable,t.getName());
			/* vpapa: assert that WorkflowNode.materialize() returned something
				valid
			*/
			if( l != null && !l.isEmpty()){
				if(functionTarget.contains("min")){
					bestCost=Double.MAX_VALUE;
					for(WorkflowNode r : l){
						tempCost = dpTable.getCost(r.dataset);
						if(tempCost<bestCost){
							bestCost=tempCost;
							bestPlan=dpTable.getPlan(r.dataset);
						}
					}
				}
				else if(functionTarget.contains("max")){
					bestCost = -Double.MAX_VALUE;
					for(WorkflowNode r : l){
						tempCost = dpTable.getCost(r.dataset);
						if(tempCost>bestCost){
							bestCost=tempCost;
							bestPlan=dpTable.getPlan(r.dataset);
						}
					}
				}
				/* vpapa: target may have or may have not a dataset
				*/
				if( t.dataset != null){
					/* vpapa: temp below is going to be the next WorkflowNode
						having as materialized dataset the dataset of the current
						target WorkflowNode and as input the current target itself
					*/
					temp = new WorkflowNode(false, false, t.getName());
					temp.setDataset( t.dataset);
					//System.out.println(l);
					temp.addInputs(l);
					materializedWorkflow.addTarget(temp);
				}
				else{
					/* vpapa: in the absence of a dataset still the operator should
						be added to the workflow
					*/
					temp = l.get( 0).inputs.get( 0);
					temp.setDataset( l.get( 0).dataset);
					materializedWorkflow.addTarget(temp);
				}
				bestPlan.add(t);
				materializedWorkflow.setBestPlan(t.toStringNorecursive(), bestPlan);
				logger.info("Optimal cost: "+bestCost);
				materializedWorkflow.optimalCost=bestCost;
			}
		}
		return materializedWorkflow;
	}// end of AbstractWorkflow1 materialize


	public MaterializedWorkflow1 replan(
			HashMap<String, WorkflowNode> materilizedDatasets) throws Exception {
		this.materilizedDatasets=materilizedDatasets;
		return materialize("execTime", "", policy);
	}



	public String parsePolicy(String policy) {
		this.policy=policy;
		groupInputs = new HashMap<String, String>();
		String[] p = policy.split("\n");
		for (int i = 0; i < p.length; i++) {
			String[] p1 = p[i].split(",");
			if(p1[0].equals("groupInputs")){
				groupInputs.put(p1[1], p1[2]);
			}
			else if(p1[0].equals("function")){
				optimizationFunction=p1[1];
				functionTarget=p1[2];
			}
		}
		return optimizationFunction;
		//System.out.println(functionTarget);
	}

	public static String getPolicyFromString(String policy) {
		String[] p = policy.split("\n");
		for (int i = 0; i < p.length; i++) {
			String[] p1 = p[i].split(",");
			if(p1[0].equals("function")){
				return p1[1];
			}
		}
		return "";
		//System.out.println(functionTarget);
	}

	public void writeToDir(String directory) throws Exception {

		for(WorkflowNode t : targets){
			t.setAllNotVisited();
		}

        File workflowDir = new File(directory);
        if (!workflowDir.exists()) {
        	workflowDir.mkdirs();
        }
        File opDir = new File(directory+"/operators");
        if (!opDir.exists()) {
        	opDir.mkdirs();
        }
        File datasetDir = new File(directory+"/datasets");
        if (!datasetDir.exists()) {
        	datasetDir.mkdirs();
        }
        File edgeGraph = new File(directory+"/graph");
    	FileOutputStream fos = new FileOutputStream(edgeGraph);

    	BufferedWriter graphWritter = new BufferedWriter(new OutputStreamWriter(fos));


		for(WorkflowNode t : targets){
			t.writeToDir(directory+"/operators",directory+"/datasets",graphWritter);
			graphWritter.write(t.toStringNorecursive() +",$$target");
			graphWritter.newLine();
		}

		graphWritter.close();
	}


	public void readFromDir(String directory) throws IOException {
		HashMap<String,WorkflowNode> nodes = new HashMap<String, WorkflowNode>();
		File folder = new File(directory+"/operators");
		File[] files = folder.listFiles();

		for (int i = 0; i < files.length; i++) {
			if (files[i].isFile() && !files[i].isHidden()) {
				WorkflowNode n = new WorkflowNode(true, true,"");
				AbstractOperator temp = new AbstractOperator(files[i].getName());
				temp.readPropertiesFromFile(files[i]);
				n.setAbstractOperator(temp);
				nodes.put(temp.opName, n);
			}
		}
		folder = new File(directory+"/datasets");
		/* vpapa: read only if datasets folder exists and it has content */
		if( folder.exists()){
			files = folder.listFiles();
			if( files != null && files.length > 0){
				for (int i = 0; i < files.length; i++) {
					if (files[i].isFile() && !files[i].isHidden()) {
						WorkflowNode n =null;
						Dataset temp = new Dataset(files[i].getName());
						temp.readPropertiesFromFile(files[i]);
						int metadatsize =temp.datasetTree.tree.size();
						if(metadatsize==0){
							n = new WorkflowNode(false, true,"");
						}
						else{
							n = new WorkflowNode(false, false, "");
						}
						n.setDataset(temp);
						nodes.put(temp.datasetName, n);
					}
				}
			}
		}
		//putting nodes into workflowNodes make them available for printing at IReS WUI
		workflowNodes.putAll(nodes);
		File edgeGraph = new File(directory+"/graph");
		FileInputStream fis = new FileInputStream(edgeGraph);
		BufferedReader br = new BufferedReader(new InputStreamReader(fis));

		String line = null;
		String[] e	= null;
		WorkflowNode src = null;
		WorkflowNode dest = null;
		/* vpapa: operators or datasets defined in graph file may be missing from
			the corresponding folders or misswritten into abstract workflow's graph
			file
		*/
		try{
			while ((line = br.readLine()) != null) {
				e = line.split(",");
				if(e[1].equals("$$target")){
					this.targets.add(nodes.get(e[0]));
				}
				else if(e.length==2){
					src = nodes.get(e[0]);
					dest = nodes.get(e[1]);
					dest.addInput(src);
					src.addOutput(dest);
				}
				else if(e.length==3){
					src = nodes.get(e[0]);
					dest = nodes.get(e[1]);
					if(dest.isOperator){
						dest.addInput(Integer.parseInt(e[2]), src);
						src.addOutput(dest);
					}
					else{
						dest.addInput(src);
						src.addOutput(Integer.parseInt(e[2]), dest);
					}
				}
			}
		}
		catch( NullPointerException npe){
			System.out.println( "ERROR: The corresponding files of operators or datasets"
								+ " " + e[ 0] + " and " + e[ 1] + " it looks like"
								+ " that they are missing from the relative folders"
								+ "or miswritten into abstract workflow's graph file.");
			logger.info( "ERROR: The corresponding files of operators or datasets"
								+ " " + e[ 0] + " and " + e[ 1] + " it looks like"
								+ " that they are missing from the relative folders"
								+ "or miswritten into abstract workflow's graph file.");
		}
        catch( IndexOutOfBoundsException iobe){
             System.out.println( "ERROR: Destination: " + dest + " has " + src + "as source"
             					 + " at its input with number " + e[ 2] + ". However, " + dest
             					 + " has " + dest.inputs.size() + " inputs which is less than " + e[ 2] + "."
             					 + " Make sure that the graph file is set appropriately of the"
             					 + " corresponding workflow for this operator i.e. " + dest + ".");
             logger.info( "ERROR: Destination: " + dest + " has " + src + "as source"
 					 + " at its input with number " + e[ 2] + ". However, " + dest
 					 + " has " + dest.inputs.size() + " inputs which is less than " + e[ 2] + "."
 					 + " Make sure that the graph file is set appropriately of the"
 					 + " corresponding workflow for this operator i.e. " + dest + ".");
        }
		br.close();
	}


	public void refresh() {
		for(Entry<String, WorkflowNode> e : workflowNodes.entrySet()){
			if(e.getValue().isOperator){
				AbstractOperator abstractOp = AbstractOperatorLibrary.getOperator(e.getKey());
				workflowNodes.get(e.getKey()).abstractOperator=abstractOp;
			}
			else if(!e.getValue().isAbstract){
				Dataset d = DatasetLibrary.getDataset(e.getKey());
				workflowNodes.get(e.getKey()).dataset=d;
			}
		}

	}


	public String graphToString() throws IOException {
		ByteArrayOutputStream bs = new ByteArrayOutputStream();
		BufferedWriter graphWritter = new BufferedWriter(new OutputStreamWriter(bs));

{'fraction_non_alphanumeric': 0.129776247848537, 'fraction_numerical': 0.005249569707401033, 'mean_word_length': 3.5921754593953765, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '144557', 'n_tokens_mistral': 8607, 'n_tokens_neox': 7269, 'n_words': 1656}