<commit_before>package com.splicemachine.derby.impl.ast;

/**
 * Created by yifuma on 6/12/15.
 */


import com.google.common.base.Function;
import com.google.common.base.Strings;

import com.google.common.collect.Lists;

import com.splicemachine.db.iapi.error.StandardException;
import com.splicemachine.db.iapi.sql.compile.CostEstimate;
import com.splicemachine.db.iapi.sql.compile.JoinStrategy;
import com.splicemachine.db.iapi.sql.compile.Visitable;
import com.splicemachine.db.iapi.sql.dictionary.ConglomerateDescriptor;
import com.splicemachine.db.iapi.sql.dictionary.TableDescriptor;
import com.splicemachine.db.iapi.types.DataValueDescriptor;
import com.splicemachine.db.impl.sql.compile.*;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import java.util.*;


/**
 * @author <NAME>
 */
public class PlanDebugger extends AbstractSpliceVisitor {

    public static Logger LOG = Logger.getLogger(PlanDebugger.class);

    public static final String spaces = "  ";
    private static final Logger COST_LOG=Logger.getLogger(PlanDebugger.class.getName()+".CostLog");

    private boolean explain = false;
    public static ThreadLocal<Map<String,ExplainTree>> planMap = new ThreadLocal<Map<String,ExplainTree>>(){
        @Override
        protected Map<String,ExplainTree> initialValue(){
            return new HashMap<>();
        }
    };

    // Only visit root node

    @Override
    public boolean isPostOrder() {
        return false;
    }

    @Override
    public boolean skipChildren(Visitable node) {
        return true;
    }

    @Override
    public Visitable visit(ExplainNode node) throws StandardException{
        explain = true;
        return visit(node.getPlanRoot());
    }

    @Override
    public Visitable defaultVisit(Visitable node) throws StandardException {

        ResultSetNode rsn;
        if ((explain || LOG.isInfoEnabled()) &&
                node instanceof DMLStatementNode &&
                (rsn = ((DMLStatementNode) node).getResultSetNode()) != null) {

            ExplainTree tree = buildExplainTree(rsn);
//            List<Pair<String,Integer>> plan = planToString(rsn);
            Map<String, ExplainTree> m=planMap.get();
            m.put(query, tree);

            if (LOG.isInfoEnabled()){
                String treeToString = treeToString(rsn);
                LOG.info(String.format("Plan nodes for query <<\n\t%s\n>>\n%s",
                        query, treeToString));
            }
//            if (PLAN_LOG.isTraceEnabled()){
//                Gson gson = new GsonBuilder().setPrettyPrinting().create();
//                PLAN_LOG.trace(gson.toJson(ImmutableMap.of("query", query, "plan", nodeInfo(rsn, 0))));
//            }
        }
        explain = false;
        return node;
    }



    public static Map without(Map m, Object... keys){
        for (Object k: keys){
            m.remove(k);
        }
        return m;
    }

    public static Map prune(Map m){
        List<Object> toPrune = new LinkedList<>();
        for (Map.Entry e: (Set<Map.Entry<Object,Object>>)m.entrySet()){
            Object val = e.getValue();
            if (val == null || (val instanceof List && ((List)val).size() == 0)){
                toPrune.add(e.getKey());
            }
        }
        for (Object k: toPrune){
            m.remove(k);
        }
        return m;
    }

    public static String simplifyString(String className, Map<String, Object> info){
        className = className.toLowerCase();
        if(className.equals("frombasetable")){
            return (String)info.get("table");
        }
        else if(className.equals("joinnode")){
            String join = ((String)(info.get("exe"))).toLowerCase();
            if(join.equals("nestedloop")){
                return "NLJ";
            }
            else if(join.equals("sortmerge")){
                return "MSJ";
            }
            else {
                return join;
            }
        }
        else if(className.equals("projectrestrictnode")){
            return "PRN";
        }
        else if(className.equals("indextobaserownode")){
            return "ITBRN";
        }
        else{
            return className.toUpperCase();
        }
    }

    public static String infoToString(Map<String,Object> info,boolean addSpaces) throws StandardException {
        Map<String,Object> copy = new HashMap<>(info);
        Object results = copy.get("results");
        Object clazz = copy.get("class");
        clazz = simplifyString((String)clazz, info);
        int level = (Integer)copy.get("level");
        String space;
        if(addSpaces)
            space = Strings.repeat(spaces,level);
        else
            space = "";
        return String.format("%s%s (%s) %s",
                space,
                clazz,
                prune(without(copy, "class", "results", "level", "subqueries", "children", "exe", "table")),
                results != null ? printResults(level+2, (List<Map<String, Object>>) results) : "");
    }

    private static String printResults(int nTimes, List<Map<String, Object>> results)  {
        String indent = Strings.repeat(spaces, nTimes);
        StringBuilder buf = new StringBuilder("\n");
        for (Map<String,Object> row : results) {
            buf.append(indent).append('{');
            for (Map.Entry<String,Object> entry : row.entrySet()) {
                String key = entry.getKey().toLowerCase();
                if(!key.equals("exe") && !key.equals("table")){
                    String value = (String)entry.getValue();
                    buf.append(value).append(", ");
                }

            }
            if (buf.length() > 2 && buf.charAt(buf.length()-2) == ',') {
                buf.setLength(buf.length() - 2);
                buf.append("}\n");
            }
        }
        if (buf.length() > 0) {
            // remove last '\n'
            buf.setLength(buf.length()-1);
        }
        return buf.toString();
    }

    private ExplainTree buildExplainTree(ResultSetNode rsn) throws StandardException{
        ExplainTree.Builder builder = new ExplainTree.Builder();
        pushExplain(rsn,builder);
        return builder.build();
    }

    private void pushExplain(ResultSetNode rsn,ExplainTree.Builder builder) throws StandardException{
        CostEstimate ce = rsn.getFinalCostEstimate().getBase();
        if(COST_LOG.isTraceEnabled()){
            COST_LOG.trace("RowOrdering for node "+ rsn.getClass().getSimpleName()+" is "+ ce.getRowOrdering());
        }
        int rsNum = rsn.getResultSetNumber();

        if(rsn instanceof FromBaseTable){
            FromBaseTable fbt=(FromBaseTable)rsn;
            if(COST_LOG.isTraceEnabled()){
                COST_LOG.trace(fbt.getTableName()+":"+fbt.getTableNumber());
            }
            TableDescriptor tableDescriptor=fbt.getTableDescriptor();
            String tableName = String.format("%s(%s)",tableDescriptor.getName(),tableDescriptor.getHeapConglomerateId());

            ConglomerateDescriptor cd = fbt.getTrulyTheBestAccessPath().getConglomerateDescriptor();
            String indexName = null;
            if (cd.isIndex()) {
                indexName = String.format("%s(%s)", cd.getConglomerateName(), cd.getConglomerateNumber());
            }
            List<String> qualifiers =  Lists.transform(preds(rsn), PredicateUtils.predToString);
            builder.addBaseTable(rsNum,ce,tableName,indexName,qualifiers,fbt.isMultiProbing());
        }else if(rsn instanceof RowResultSetNode){
            builder.pushValuesNode(rsNum,ce);
        } else if(rsn instanceof ProjectRestrictNode){
            pushExplain(((SingleChildResultSetNode)rsn).getChildResult(),builder);
            if(!((ProjectRestrictNode)rsn).nopProjectRestrict()){
                List<String> predicates=Lists.transform(preds(rsn),PredicateUtils.predToString);
                builder.pushProjection(rsNum,ce,predicates);
            }
        }else if(rsn instanceof IndexToBaseRowNode){
            pushExplain(((IndexToBaseRowNode)rsn).getSource(),builder);
            long heapTable=((IndexToBaseRowNode)rsn).getBaseConglomerateDescriptor().getConglomerateNumber();
            builder.pushIndexFetch(rsNum,ce,heapTable);
        }else if(rsn instanceof ScrollInsensitiveResultSetNode){
            pushExplain(((ScrollInsensitiveResultSetNode)rsn).getChildResult(),builder);
        }else if(rsn instanceof JoinNode){
            JoinNode j  = (JoinNode)rsn;
            JoinStrategy joinStrategy = RSUtils.ap(j).getJoinStrategy();
            List<String> joinPreds =  Lists.transform(PredicateUtils.PLtoList(j.joinPredicates),PredicateUtils.predToString);
            ExplainTree.Builder rightBuilder = new ExplainTree.Builder();
            pushExplain(j.getRightResultSet(),rightBuilder);
            //push the left side directly
            pushExplain(j.getLeftResultSet(),builder);

            builder.pushJoin(rsNum,ce,joinStrategy,joinPreds,rightBuilder);
        } else if(rsn instanceof SingleChildResultSetNode){

            pushExplain(((SingleChildResultSetNode)rsn).getChildResult(),builder);
            String nodeName=rsn.getClass().getSimpleName().replace("Node","");
            builder.pushNode(nodeName,rsNum,ce);
        } else if(rsn instanceof TableOperatorNode){
            //we have two tables to work with
            ExplainTree.Builder rightBuilder = new ExplainTree.Builder();
            pushExplain(((TableOperatorNode)rsn).getRightResultSet(),rightBuilder);
            pushExplain(((TableOperatorNode)rsn).getLeftResultSet(),builder);
            builder.pushTableOperator(rsn.getClass().getSimpleName().replace("Node",""),rsNum,ce,rightBuilder);
        } else if(rsn instanceof FromVTI){
            FromVTI vti = (FromVTI)rsn;
            String tableName = vti.getName();
            builder.addBaseTable(rsNum,ce,"VTI:"+tableName,null,null,false);
        }

        //collect subqueries
        List<SubqueryNode> subs = RSUtils.collectExpressionNodes(rsn,SubqueryNode.class);
        if(subs.size()>0){
            ExplainTree.Builder subqueryBuilder = new ExplainTree.Builder();
            for(SubqueryNode subqueryNode : subs){
                pushExplain(subqueryNode.getResultSet(),subqueryBuilder);
                boolean exprSub = subqueryNode.getSubqueryType()==SubqueryNode.EXPRESSION_SUBQUERY;
                boolean corrSub = subqueryNode.hasCorrelatedCRs();
                boolean invariant = subqueryNode.isInvariant();
                builder.pushSubquery(subqueryNode.getResultSet().getResultSetNumber(),exprSub,corrSub,invariant,subqueryBuilder);
                subqueryBuilder.reset(); //reset for the next subquery in the list
            }
        }
    }

    public static Map<String,Object> nodeInfo(final ResultSetNode rsn, final int level) throws StandardException {
        Map<String,Object> info = new HashMap<>();
        // CostEstimate co = rsn.getFinalCostEstimate().getBase();
        info.put("class", JoinInfo.className.apply(rsn));
        info.put("n", rsn.getResultSetNumber());
        info.put("level", level);
        //  info.put("cost", co.prettyString());
        if (Level.TRACE.equals(LOG.getLevel())) {
//        if(LOG.isTraceEnabled()){
            // FIXME: FIND OUT WHY LOG.isTraceEnabled() always returns false

            // we only want to see exec row info when THIS logger is set to trace:
            // com.splicemachine.derby.impl.ast.PlanPrinter=TRACE
            // or
            // call SYSCS_UTIL.SYSCS_SET_LOGGER_LEVEL('com.splicemachine.derby.impl.ast.PlanDebugger', 'TRACE');

            List<Map<String, Object>> columnInfo =  getResultColumnInfo(rsn);
            int count = 1;
            for(Map<String, Object> m : columnInfo){
                String colName = (String)m.get("column");
                if(colName != null && !colName.equals("") && !colName.equals("null")){
                    Object src = m.get("src");
                    if(src != null && !src.equals("") && !src.equals("null")){
                        info.put(Integer.toString(count), src);
                    }
                }
                count++;
            }
        }
        List<ResultSetNode> children = RSUtils.getChildren(rsn);
        info.put("children", Lists.reverse(Lists.transform(children, new Function<ResultSetNode, Map<String,Object>>() {
            @Override
            public Map<String,Object> apply(ResultSetNode child) {
                try {
                    return nodeInfo(child, level + 1);<commit_msg>Hopefully fixing case sensitivity file<commit_after>package com.splicemachine.derby.impl.ast;

/**
 * Created by yifuma on 6/12/15.
 */


import com.google.common.base.Function;
import com.google.common.base.Strings;

import com.google.common.collect.Lists;

import com.splicemachine.db.iapi.error.StandardException;
import com.splicemachine.db.iapi.sql.compile.CostEstimate;
import com.splicemachine.db.iapi.sql.compile.JoinStrategy;
import com.splicemachine.db.iapi.sql.compile.Visitable;
import com.splicemachine.db.iapi.sql.dictionary.ConglomerateDescriptor;
import com.splicemachine.db.iapi.sql.dictionary.TableDescriptor;
import com.splicemachine.db.iapi.types.DataValueDescriptor;
import com.splicemachine.db.impl.sql.compile.*;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import java.util.*;


/**
 * @author <NAME>
 */
<<<<<<< HEAD
public class PlanDebugger extends AbstractSpliceVisitor {

    public static Logger LOG = Logger.getLogger(PlanDebugger.class);

    public static final String spaces = "  ";
    private static final Logger COST_LOG=Logger.getLogger(PlanDebugger.class.getName()+".CostLog");
=======
public class planDebugger extends AbstractSpliceVisitor {

    public static Logger LOG = Logger.getLogger(planDebugger.class);

    public static final String spaces = "  ";
    private static final Logger COST_LOG=Logger.getLogger(planDebugger.class.getName()+".CostLog");
>>>>>>> Hopefully fixing case sensitivity file

    private boolean explain = false;
    public static ThreadLocal<Map<String,ExplainTree>> planMap = new ThreadLocal<Map<String,ExplainTree>>(){
        @Override
        protected Map<String,ExplainTree> initialValue(){
            return new HashMap<>();
        }
    };

    // Only visit root node

    @Override
    public boolean isPostOrder() {
        return false;
    }

    @Override
    public boolean skipChildren(Visitable node) {
        return true;
    }

    @Override
    public Visitable visit(ExplainNode node) throws StandardException{
        explain = true;
        return visit(node.getPlanRoot());
    }

    @Override
    public Visitable defaultVisit(Visitable node) throws StandardException {

        ResultSetNode rsn;
        if ((explain || LOG.isInfoEnabled()) &&
                node instanceof DMLStatementNode &&
                (rsn = ((DMLStatementNode) node).getResultSetNode()) != null) {

            ExplainTree tree = buildExplainTree(rsn);
//            List<Pair<String,Integer>> plan = planToString(rsn);
            Map<String, ExplainTree> m=planMap.get();
            m.put(query, tree);

            if (LOG.isInfoEnabled()){
                String treeToString = treeToString(rsn);
                LOG.info(String.format("Plan nodes for query <<\n\t%s\n>>\n%s",
                        query, treeToString));
            }
//            if (PLAN_LOG.isTraceEnabled()){
//                Gson gson = new GsonBuilder().setPrettyPrinting().create();
//                PLAN_LOG.trace(gson.toJson(ImmutableMap.of("query", query, "plan", nodeInfo(rsn, 0))));
//            }
        }
        explain = false;
        return node;
    }



    public static Map without(Map m, Object... keys){
        for (Object k: keys){
            m.remove(k);
        }
        return m;
    }

    public static Map prune(Map m){
        List<Object> toPrune = new LinkedList<>();
        for (Map.Entry e: (Set<Map.Entry<Object,Object>>)m.entrySet()){
            Object val = e.getValue();
            if (val == null || (val instanceof List && ((List)val).size() == 0)){
                toPrune.add(e.getKey());
            }
        }
        for (Object k: toPrune){
            m.remove(k);
        }
        return m;
    }

    public static String simplifyString(String className, Map<String, Object> info){
        className = className.toLowerCase();
        if(className.equals("frombasetable")){
            return (String)info.get("table");
        }
        else if(className.equals("joinnode")){
            String join = ((String)(info.get("exe"))).toLowerCase();
            if(join.equals("nestedloop")){
                return "NLJ";
            }
            else if(join.equals("sortmerge")){
                return "MSJ";
            }
            else {
                return join;
            }
        }
        else if(className.equals("projectrestrictnode")){
            return "PRN";
        }
        else if(className.equals("indextobaserownode")){
            return "ITBRN";
        }
        else{
            return className.toUpperCase();
        }
    }

    public static String infoToString(Map<String,Object> info,boolean addSpaces) throws StandardException {
        Map<String,Object> copy = new HashMap<>(info);
        Object results = copy.get("results");
        Object clazz = copy.get("class");
        clazz = simplifyString((String)clazz, info);
        int level = (Integer)copy.get("level");
        String space;
        if(addSpaces)
            space = Strings.repeat(spaces,level);
        else
            space = "";
        return String.format("%s%s (%s) %s",
                space,
                clazz,
                prune(without(copy, "class", "results", "level", "subqueries", "children", "exe", "table")),
                results != null ? printResults(level+2, (List<Map<String, Object>>) results) : "");
    }

    private static String printResults(int nTimes, List<Map<String, Object>> results)  {
        String indent = Strings.repeat(spaces, nTimes);
        StringBuilder buf = new StringBuilder("\n");
        for (Map<String,Object> row : results) {
            buf.append(indent).append('{');
            for (Map.Entry<String,Object> entry : row.entrySet()) {
                String key = entry.getKey().toLowerCase();
                if(!key.equals("exe") && !key.equals("table")){
                    String value = (String)entry.getValue();
                    buf.append(value).append(", ");
                }

            }
            if (buf.length() > 2 && buf.charAt(buf.length()-2) == ',') {
                buf.setLength(buf.length() - 2);
                buf.append("}\n");
            }
        }
        if (buf.length() > 0) {
            // remove last '\n'
            buf.setLength(buf.length()-1);
        }
        return buf.toString();
    }

    private ExplainTree buildExplainTree(ResultSetNode rsn) throws StandardException{
        ExplainTree.Builder builder = new ExplainTree.Builder();
        pushExplain(rsn,builder);
        return builder.build();
    }

    private void pushExplain(ResultSetNode rsn,ExplainTree.Builder builder) throws StandardException{
        CostEstimate ce = rsn.getFinalCostEstimate().getBase();
        if(COST_LOG.isTraceEnabled()){
            COST_LOG.trace("RowOrdering for node "+ rsn.getClass().getSimpleName()+" is "+ ce.getRowOrdering());
        }
        int rsNum = rsn.getResultSetNumber();

        if(rsn instanceof FromBaseTable){
            FromBaseTable fbt=(FromBaseTable)rsn;
            if(COST_LOG.isTraceEnabled()){
                COST_LOG.trace(fbt.getTableName()+":"+fbt.getTableNumber());
            }
            TableDescriptor tableDescriptor=fbt.getTableDescriptor();
            String tableName = String.format("%s(%s)",tableDescriptor.getName(),tableDescriptor.getHeapConglomerateId());

            ConglomerateDescriptor cd = fbt.getTrulyTheBestAccessPath().getConglomerateDescriptor();
            String indexName = null;
            if (cd.isIndex()) {
                indexName = String.format("%s(%s)", cd.getConglomerateName(), cd.getConglomerateNumber());
            }
            List<String> qualifiers =  Lists.transform(preds(rsn), PredicateUtils.predToString);
            builder.addBaseTable(rsNum,ce,tableName,indexName,qualifiers,fbt.isMultiProbing());
        }else if(rsn instanceof RowResultSetNode){
            builder.pushValuesNode(rsNum,ce);
        } else if(rsn instanceof ProjectRestrictNode){
            pushExplain(((SingleChildResultSetNode)rsn).getChildResult(),builder);
            if(!((ProjectRestrictNode)rsn).nopProjectRestrict()){
                List<String> predicates=Lists.transform(preds(rsn),PredicateUtils.predToString);
                builder.pushProjection(rsNum,ce,predicates);
            }
        }else if(rsn instanceof IndexToBaseRowNode){
            pushExplain(((IndexToBaseRowNode)rsn).getSource(),builder);
            long heapTable=((IndexToBaseRowNode)rsn).getBaseConglomerateDescriptor().getConglomerateNumber();
            builder.pushIndexFetch(rsNum,ce,heapTable);
        }else if(rsn instanceof ScrollInsensitiveResultSetNode){
            pushExplain(((ScrollInsensitiveResultSetNode)rsn).getChildResult(),builder);
        }else if(rsn instanceof JoinNode){
            JoinNode j  = (JoinNode)rsn;
            JoinStrategy joinStrategy = RSUtils.ap(j).getJoinStrategy();
            List<String> joinPreds =  Lists.transform(PredicateUtils.PLtoList(j.joinPredicates),PredicateUtils.predToString);
            ExplainTree.Builder rightBuilder = new ExplainTree.Builder();
            pushExplain(j.getRightResultSet(),rightBuilder);
            //push the left side directly
            pushExplain(j.getLeftResultSet(),builder);

            builder.pushJoin(rsNum,ce,joinStrategy,joinPreds,rightBuilder);
        } else if(rsn instanceof SingleChildResultSetNode){

            pushExplain(((SingleChildResultSetNode)rsn).getChildResult(),builder);
            String nodeName=rsn.getClass().getSimpleName().replace("Node","");
            builder.pushNode(nodeName,rsNum,ce);
        } else if(rsn instanceof TableOperatorNode){
            //we have two tables to work with
            ExplainTree.Builder rightBuilder = new ExplainTree.Builder();
            pushExplain(((TableOperatorNode)rsn).getRightResultSet(),rightBuilder);
            pushExplain(((TableOperatorNode)rsn).getLeftResultSet(),builder);
            builder.pushTableOperator(rsn.getClass().getSimpleName().replace("Node",""),rsNum,ce,rightBuilder);
        } else if(rsn instanceof FromVTI){
            FromVTI vti = (FromVTI)rsn;
            String tableName = vti.getName();
            builder.addBaseTable(rsNum,ce,"VTI:"+tableName,null,null,false);
        }

        //collect subqueries
        List<SubqueryNode> subs = RSUtils.collectExpressionNodes(rsn,SubqueryNode.class);
        if(subs.size()>0){
            ExplainTree.Builder subqueryBuilder = new ExplainTree.Builder();
            for(SubqueryNode subqueryNode : subs){
                pushExplain(subqueryNode.getResultSet(),subqueryBuilder);
                boolean exprSub = subqueryNode.getSubqueryType()==SubqueryNode.EXPRESSION_SUBQUERY;
                boolean corrSub = subqueryNode.hasCorrelatedCRs();
                boolean invariant = subqueryNode.isInvariant();
                builder.pushSubquery(subqueryNode.getResultSet().getResultSetNumber(),exprSub,corrSub,invariant,subqueryBuilder);
                subqueryBuilder.reset(); //reset for the next subquery in the list
            }
        }
    }

    public static Map<String,Object> nodeInfo(final ResultSetNode rsn, final int level) throws StandardException {
        Map<String,Object> info = new HashMap<>();
        // CostEstimate co = rsn.getFinalCostEstimate().getBase();
        info.put("class", JoinInfo.className.apply(rsn));
        info.put("n", rsn.getResultSetNumber());
        info.put("level", level);
        //  info.put("cost", co.prettyString());
        if (Level.TRACE.equals(LOG.getLevel())) {
//        if(LOG.isTraceEnabled()){
            // FIXME: FIND OUT WHY LOG.isTraceEnabled() always returns false

            // we only want to see exec row info when THIS logger is set to trace:
            // com.splicemachine.derby.impl.ast.PlanPrinter=TRACE
            // or
<<<<<<< HEAD
            // call SYSCS_UTIL.SYSCS_SET_LOGGER_LEVEL('com.splicemachine.derby.impl.ast.PlanDebugger', 'TRACE');
=======
            // call SYSCS_UTIL.SYSCS_SET_LOGGER_LEVEL('com.splicemachine.derby.impl.ast.planDebugger', 'TRACE');
>>>>>>> Hopefully fixing case sensitivity file

            List<Map<String, Object>> columnInfo =  getResultColumnInfo(rsn);
            int count = 1;
            for(Map<String, Object> m : columnInfo){
                String colName = (String)m.get("column");
                if(colName != null && !colName.equals("") && !colName.equals("null")){
                    Object src = m.get("src");
                    if(src != null && !src.equals("") && !src.equals("null")){
                        info.put(Integer.toString(count), src);
                    }
                }
                count++;
            }
        }
        List<ResultSetNode> children = RSUtils.getChildren(rsn);
        info.put("children", Lists.reverse(Lists.transform(children, new Function<ResultSetNode, Map<String,Object>>() {
            @Override
            public Map<String,Object> apply(ResultSetNode child) {
                try {
                    return nodeInfo(child, level + 1);
{'fraction_non_alphanumeric': 0.12707289603479746, 'fraction_numerical': 0.0013981125480601187, 'mean_word_length': 2.434707216219821, 'pattern_counts': {'":': 2, '<': 97, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4943360', 'n_tokens_mistral': 7309, 'n_tokens_neox': 7137, 'n_words': 1306}