<commit_before>/*
 * Copyright (c) 2012-2018 Snowflake Computing Inc. All rights reserved.
 */

package net.snowflake.client.core;

import com.fasterxml.jackson.databind.JsonNode;
import net.snowflake.client.jdbc.ErrorCode;
import net.snowflake.client.jdbc.SnowflakeChunkDownloader;
import net.snowflake.client.jdbc.SnowflakeColumnMetadata;
import net.snowflake.client.jdbc.SnowflakeSQLException;
import net.snowflake.client.jdbc.SnowflakeUtil;
import net.snowflake.common.core.SFBinaryFormat;
import net.snowflake.common.core.SFTime;
import net.snowflake.common.core.SFTimestamp;
import net.snowflake.common.core.SnowflakeDateTimeFormat;
import net.snowflake.common.util.TimeUtil;

import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.sql.Types;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import net.snowflake.client.log.SFLogger;
import net.snowflake.client.log.SFLoggerFactory;

import static net.snowflake.client.core.SessionUtil.CLIENT_MEMORY_LIMIT;
import static net.snowflake.client.core.SessionUtil.CLIENT_PREFETCH_THREADS;

/**
 * Created by jhuang on 2/1/16.
 */
public class ResultUtil
{
  static final SFLogger logger = SFLoggerFactory.getLogger(ResultUtil.class);

  // Construct a default UTC zone for TIMESTAMPNTZ
  private static TimeZone timeZoneUTC = TimeZone.getTimeZone("UTC");

  static public class ResultInput
  {
    JsonNode resultJSON;

    // These fields are used for building a result downloader which
    // uses http client
    int connectionTimeout;
    int socketTimeout;
    int networkTimeoutInMilli;

    public ResultInput setResultJSON(JsonNode resultJSON)
    {
      this.resultJSON = resultJSON;
      return this;
    }

    public ResultInput setConnectionTimeout(int connectionTimeout)
    {
      this.connectionTimeout = connectionTimeout;
      return this;
    }

    public ResultInput setSocketTimeout(int socketTimeout)
    {
      this.socketTimeout = socketTimeout;
      return this;
    }

    public ResultInput setNetworkTimeoutInMilli(int networkTimeoutInMilli)
    {
      this.networkTimeoutInMilli = networkTimeoutInMilli;
      return this;
    }
  }

  static public class ResultOutput
  {
    long chunkCount;
    String queryId;
    String finalDatabaseName;
    String finalSchemaName;
    String finalRoleName;
    String finalWarehouseName;
    SFStatementType statementType;
    boolean totalRowCountTruncated;
    Map<String, Object> parameters = new HashMap<>();
    int columnCount;
    private List<SnowflakeColumnMetadata> resultColumnMetadata =
        new ArrayList<SnowflakeColumnMetadata>();
    private JsonNode currentChunkRowset = null;
    int currentChunkRowCount;
    long resultVersion;
    int numberOfBinds;
    boolean arrayBindSupported;
    SnowflakeChunkDownloader chunkDownloader;
    SnowflakeDateTimeFormat timestampNTZFormatter;
    SnowflakeDateTimeFormat timestampLTZFormatter;
    SnowflakeDateTimeFormat timestampTZFormatter;
    SnowflakeDateTimeFormat dateFormatter;
    SnowflakeDateTimeFormat timeFormatter;
    TimeZone timeZone;
    boolean honorClientTZForTimestampNTZ;
    SFBinaryFormat binaryFormatter;
    long sendResultTime;

    public long getChunkCount()
    {
      return chunkCount;
    }

    public boolean isArrayBindSupported()
    {
      return arrayBindSupported;
    }

    public String getQueryId()
    {
      return queryId;
    }

    public String getFinalDatabaseName()
    {
      return finalDatabaseName;
    }

    public String getFinalSchemaName()
    {
      return finalSchemaName;
    }

    String getFinalRoleName()
    {
      return finalRoleName;
    }

    String getFinalWarehouseName()
    {
      return finalWarehouseName;
    }

    public SFStatementType getStatementType()
    {
      return statementType;
    }

    public boolean isTotalRowCountTruncated()
    {
      return totalRowCountTruncated;
    }

    public Map<String, Object> getParameters()
    {
      return parameters;
    }

    public int getColumnCount()
    {
      return columnCount;
    }

    public List<SnowflakeColumnMetadata> getResultColumnMetadata()
    {
      return resultColumnMetadata;
    }

    public JsonNode getAndClearCurrentChunkRowset()
    {
      JsonNode currentChunkRowset = this.currentChunkRowset;
      this.currentChunkRowset = null;
      return currentChunkRowset;
    }

    public int getCurrentChunkRowCount()
    {
      return currentChunkRowCount;
    }

    public long getResultVersion()
    {
      return resultVersion;
    }

    public int getNumberOfBinds()
    {
      return numberOfBinds;
    }

    public SnowflakeChunkDownloader getChunkDownloader()
    {
      return chunkDownloader;
    }

    public SnowflakeDateTimeFormat getTimestampNTZFormatter()
    {
      return timestampNTZFormatter;
    }

    public SnowflakeDateTimeFormat getTimestampLTZFormatter()
    {
      return timestampLTZFormatter;
    }

    public SnowflakeDateTimeFormat getTimestampTZFormatter()
    {
      return timestampTZFormatter;
    }

    public SnowflakeDateTimeFormat getDateFormatter()
    {
      return dateFormatter;
    }

    public SnowflakeDateTimeFormat getTimeFormatter()
    {
      return timeFormatter;
    }

    public TimeZone getTimeZone()
    {
      return timeZone;
    }

    public boolean isHonorClientTZForTimestampNTZ()
    {
      return honorClientTZForTimestampNTZ;
    }

    public SFBinaryFormat getBinaryFormatter()
    {
      return binaryFormatter;
    }

    public long getSendResultTime()
    {
      return sendResultTime;
    }
  }

  /**
   * A common helper to process result response
   *
   * @param resultData wrapper object over simple json result
   * @param sfSession the Snowflake session
   * @return processed result output
   * @throws SnowflakeSQLException if failed to get number of columns
   */
  static public ResultOutput processResult(ResultInput resultData,
                                           SFSession sfSession)
      throws SnowflakeSQLException
  {
    ResultOutput resultOutput = new ResultOutput();

    logger.debug("Entering processResult");
    JsonNode rootNode = resultData.resultJSON;

    SnowflakeUtil.checkErrorAndThrowException(rootNode);

    // get the query id
    resultOutput.queryId = rootNode.path("data").path("queryId").asText();

    JsonNode databaseNode = rootNode.path("data").path("finalDatabaseName");
    resultOutput.finalDatabaseName = databaseNode.isNull() ? null : databaseNode.asText();

    JsonNode schemaNode = rootNode.path("data").path("finalSchemaName");
    resultOutput.finalSchemaName = schemaNode.isNull() ? null : schemaNode.asText();

    JsonNode roleNode = rootNode.path("data").path("finalRoleName");
    resultOutput.finalRoleName = roleNode.isNull() ? null : roleNode.asText();

    JsonNode warehouseNode = rootNode.path("data").path("finalWarehouseName");
    resultOutput.finalWarehouseName = warehouseNode.isNull() ? null : warehouseNode.asText();

    resultOutput.statementType = SFStatementType.lookUpTypeById(
        rootNode.path("data").path("statementTypeId").asLong());

    resultOutput.totalRowCountTruncated
        = rootNode.path("data").path("totalTruncated").asBoolean();

    if (logger.isDebugEnabled())
      logger.debug("query id: {}", resultOutput.queryId);

    // extrace parameters
    resultOutput.parameters =
        SessionUtil.getCommonParams(rootNode.path("data").path("parameters"));

    // initialize column metadata
    resultOutput.columnCount = rootNode.path("data").path("rowtype").size();

    for (int i = 0; i < resultOutput.columnCount; i++)
    {
      JsonNode colNode = rootNode.path("data").path("rowtype").path(i);

      SnowflakeColumnMetadata columnMetadata
          = SnowflakeUtil.extractColumnMetadata(
              colNode, sfSession.isJdbcTreatDecimalAsInt());

      resultOutput.resultColumnMetadata.add(columnMetadata);

      if (logger.isDebugEnabled())
        logger.debug("Get column metadata: {}",
             columnMetadata.toString());
    }

    resultOutput.currentChunkRowset = rootNode.path("data").path("rowset");

    if (resultOutput.currentChunkRowset == null ||
        resultOutput.currentChunkRowset.isMissingNode())
    {
      resultOutput.currentChunkRowCount = 0;
    }
    else
    {
      resultOutput.currentChunkRowCount = resultOutput.currentChunkRowset.size();
    }

    logger.debug("First chunk row count: {}",
        resultOutput.currentChunkRowCount);

    JsonNode chunksNode = rootNode.path("data").path("chunks");

    if (!chunksNode.isMissingNode())
    {
      resultOutput.chunkCount = chunksNode.size();

      // Try to get the Query Result Master Key
      JsonNode qrmkNode = rootNode.path("data").path("qrmk");
      final String qrmk = qrmkNode.isMissingNode() ?
          null : qrmkNode.textValue();

      JsonNode chunkHeaders = rootNode.path("data").path("chunkHeaders");

      // the initialization of chunk downloader will start prefetching
      // first few chunks
      if (resultOutput.chunkCount > 0)
      {
        logger.debug("#chunks={}, initialize chunk downloader",
            resultOutput.chunkCount);

        // prefetch threads
        int resultPrefetchThreads = 4;
        if (resultOutput.parameters.get(CLIENT_PREFETCH_THREADS) != null)
        {
          resultPrefetchThreads =
              (int) resultOutput.parameters.get(CLIENT_PREFETCH_THREADS);
        }

        /*
         * Should JsonParser be used instead of the original Json deserializer.
         */
        boolean useJsonParser = true;
        if (resultOutput.parameters.get("JDBC_USE_JSON_PARSER") != null)
        {
          useJsonParser =
              (boolean) resultOutput.parameters.get("JDBC_USE_JSON_PARSER");
        }

        int memoryUsage = 1536;
        if (resultOutput.parameters.get(CLIENT_MEMORY_LIMIT) != null)
        {
          memoryUsage =
              (int) resultOutput.parameters.get(CLIENT_MEMORY_LIMIT);
        }

        boolean efficientChunkStorage = false;
        if (resultOutput.parameters.get("JDBC_EFFICIENT_CHUNK_STORAGE") != null)
        {
          efficientChunkStorage = (boolean)
              resultOutput.parameters.get("JDBC_EFFICIENT_CHUNK_STORAGE");
        }

        if (useJsonParser && efficientChunkStorage)
        {
          resultPrefetchThreads =
              Math.max(Math.min(resultPrefetchThreads,
                                Runtime.getRuntime().availableProcessors() / 2),
                       2);
        }
        else
        {
          resultPrefetchThreads = 1;
        }
        // initialize the chunk downloader
        resultOutput.chunkDownloader =
            new SnowflakeChunkDownloader(resultOutput.columnCount,
                                         chunksNode,
                                         resultPrefetchThreads,
                                         qrmk,
                                         chunkHeaders,
                                         resultData.networkTimeoutInMilli,
                                         useJsonParser,
                                         memoryUsage * 1024 * 1024,
                                         efficientChunkStorage);
      }
    }

    String sqlTimestampFormat = (String) effectiveParamValue(
        resultOutput.parameters, "TIMESTAMP_OUTPUT_FORMAT");

    // Special handling of specialized formatters, use a helper function
    resultOutput.timestampNTZFormatter = specializedFormatter(
        resultOutput.parameters,
        "timestamp_ntz",
        "TIMESTAMP_NTZ_OUTPUT_FORMAT",
        sqlTimestampFormat);

    resultOutput.timestampLTZFormatter = specializedFormatter(
        resultOutput.parameters,
        "timestamp_ltz",
        "TIMESTAMP_LTZ_OUTPUT_FORMAT",
        sqlTimestampFormat);

    resultOutput.timestampTZFormatter = specializedFormatter(
        resultOutput.parameters,<commit_msg> SNOW-56134 SNOW-61209 jdbc client memory setting debug<commit_after>/*
 * Copyright (c) 2012-2018 Snowflake Computing Inc. All rights reserved.
 */

package net.snowflake.client.core;

import com.fasterxml.jackson.databind.JsonNode;
import net.snowflake.client.jdbc.ErrorCode;
import net.snowflake.client.jdbc.SnowflakeChunkDownloader;
import net.snowflake.client.jdbc.SnowflakeColumnMetadata;
import net.snowflake.client.jdbc.SnowflakeSQLException;
import net.snowflake.client.jdbc.SnowflakeUtil;
import net.snowflake.common.core.SFBinaryFormat;
import net.snowflake.common.core.SFTime;
import net.snowflake.common.core.SFTimestamp;
import net.snowflake.common.core.SnowflakeDateTimeFormat;
import net.snowflake.common.util.TimeUtil;

import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.sql.Types;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import net.snowflake.client.log.SFLogger;
import net.snowflake.client.log.SFLoggerFactory;

import static net.snowflake.client.core.SessionUtil.CLIENT_MEMORY_LIMIT;
import static net.snowflake.client.core.SessionUtil.CLIENT_PREFETCH_THREADS;

/**
 * Created by jhuang on 2/1/16.
 */
public class ResultUtil
{
  static final SFLogger logger = SFLoggerFactory.getLogger(ResultUtil.class);

  // Construct a default UTC zone for TIMESTAMPNTZ
  private static TimeZone timeZoneUTC = TimeZone.getTimeZone("UTC");

  static public class ResultInput
  {
    JsonNode resultJSON;

    // These fields are used for building a result downloader which
    // uses http client
    int connectionTimeout;
    int socketTimeout;
    int networkTimeoutInMilli;

    public ResultInput setResultJSON(JsonNode resultJSON)
    {
      this.resultJSON = resultJSON;
      return this;
    }

    public ResultInput setConnectionTimeout(int connectionTimeout)
    {
      this.connectionTimeout = connectionTimeout;
      return this;
    }

    public ResultInput setSocketTimeout(int socketTimeout)
    {
      this.socketTimeout = socketTimeout;
      return this;
    }

    public ResultInput setNetworkTimeoutInMilli(int networkTimeoutInMilli)
    {
      this.networkTimeoutInMilli = networkTimeoutInMilli;
      return this;
    }
  }

  static public class ResultOutput
  {
    long chunkCount;
    String queryId;
    String finalDatabaseName;
    String finalSchemaName;
    String finalRoleName;
    String finalWarehouseName;
    SFStatementType statementType;
    boolean totalRowCountTruncated;
    Map<String, Object> parameters = new HashMap<>();
    int columnCount;
    private List<SnowflakeColumnMetadata> resultColumnMetadata =
        new ArrayList<SnowflakeColumnMetadata>();
    private JsonNode currentChunkRowset = null;
    int currentChunkRowCount;
    long resultVersion;
    int numberOfBinds;
    boolean arrayBindSupported;
    SnowflakeChunkDownloader chunkDownloader;
    SnowflakeDateTimeFormat timestampNTZFormatter;
    SnowflakeDateTimeFormat timestampLTZFormatter;
    SnowflakeDateTimeFormat timestampTZFormatter;
    SnowflakeDateTimeFormat dateFormatter;
    SnowflakeDateTimeFormat timeFormatter;
    TimeZone timeZone;
    boolean honorClientTZForTimestampNTZ;
    SFBinaryFormat binaryFormatter;
    long sendResultTime;

    public long getChunkCount()
    {
      return chunkCount;
    }

    public boolean isArrayBindSupported()
    {
      return arrayBindSupported;
    }

    public String getQueryId()
    {
      return queryId;
    }

    public String getFinalDatabaseName()
    {
      return finalDatabaseName;
    }

    public String getFinalSchemaName()
    {
      return finalSchemaName;
    }

    String getFinalRoleName()
    {
      return finalRoleName;
    }

    String getFinalWarehouseName()
    {
      return finalWarehouseName;
    }

    public SFStatementType getStatementType()
    {
      return statementType;
    }

    public boolean isTotalRowCountTruncated()
    {
      return totalRowCountTruncated;
    }

    public Map<String, Object> getParameters()
    {
      return parameters;
    }

    public int getColumnCount()
    {
      return columnCount;
    }

    public List<SnowflakeColumnMetadata> getResultColumnMetadata()
    {
      return resultColumnMetadata;
    }

    public JsonNode getAndClearCurrentChunkRowset()
    {
      JsonNode currentChunkRowset = this.currentChunkRowset;
      this.currentChunkRowset = null;
      return currentChunkRowset;
    }

    public int getCurrentChunkRowCount()
    {
      return currentChunkRowCount;
    }

    public long getResultVersion()
    {
      return resultVersion;
    }

    public int getNumberOfBinds()
    {
      return numberOfBinds;
    }

    public SnowflakeChunkDownloader getChunkDownloader()
    {
      return chunkDownloader;
    }

    public SnowflakeDateTimeFormat getTimestampNTZFormatter()
    {
      return timestampNTZFormatter;
    }

    public SnowflakeDateTimeFormat getTimestampLTZFormatter()
    {
      return timestampLTZFormatter;
    }

    public SnowflakeDateTimeFormat getTimestampTZFormatter()
    {
      return timestampTZFormatter;
    }

    public SnowflakeDateTimeFormat getDateFormatter()
    {
      return dateFormatter;
    }

    public SnowflakeDateTimeFormat getTimeFormatter()
    {
      return timeFormatter;
    }

    public TimeZone getTimeZone()
    {
      return timeZone;
    }

    public boolean isHonorClientTZForTimestampNTZ()
    {
      return honorClientTZForTimestampNTZ;
    }

    public SFBinaryFormat getBinaryFormatter()
    {
      return binaryFormatter;
    }

    public long getSendResultTime()
    {
      return sendResultTime;
    }
  }

  /**
   * A common helper to process result response
   *
   * @param resultData wrapper object over simple json result
   * @param sfSession the Snowflake session
   * @return processed result output
   * @throws SnowflakeSQLException if failed to get number of columns
   */
  static public ResultOutput processResult(ResultInput resultData,
                                           SFSession sfSession)
      throws SnowflakeSQLException
  {
    ResultOutput resultOutput = new ResultOutput();

    logger.debug("Entering processResult");
    JsonNode rootNode = resultData.resultJSON;

    SnowflakeUtil.checkErrorAndThrowException(rootNode);

    // get the query id
    resultOutput.queryId = rootNode.path("data").path("queryId").asText();

    JsonNode databaseNode = rootNode.path("data").path("finalDatabaseName");
    resultOutput.finalDatabaseName = databaseNode.isNull() ? null : databaseNode.asText();

    JsonNode schemaNode = rootNode.path("data").path("finalSchemaName");
    resultOutput.finalSchemaName = schemaNode.isNull() ? null : schemaNode.asText();

    JsonNode roleNode = rootNode.path("data").path("finalRoleName");
    resultOutput.finalRoleName = roleNode.isNull() ? null : roleNode.asText();

    JsonNode warehouseNode = rootNode.path("data").path("finalWarehouseName");
    resultOutput.finalWarehouseName = warehouseNode.isNull() ? null : warehouseNode.asText();

    resultOutput.statementType = SFStatementType.lookUpTypeById(
        rootNode.path("data").path("statementTypeId").asLong());

    resultOutput.totalRowCountTruncated
        = rootNode.path("data").path("totalTruncated").asBoolean();

    if (logger.isDebugEnabled())
      logger.debug("query id: {}", resultOutput.queryId);

    // extrace parameters
    resultOutput.parameters =
        SessionUtil.getCommonParams(rootNode.path("data").path("parameters"));

    // initialize column metadata
    resultOutput.columnCount = rootNode.path("data").path("rowtype").size();

    for (int i = 0; i < resultOutput.columnCount; i++)
    {
      JsonNode colNode = rootNode.path("data").path("rowtype").path(i);

      SnowflakeColumnMetadata columnMetadata
          = SnowflakeUtil.extractColumnMetadata(
              colNode, sfSession.isJdbcTreatDecimalAsInt());

      resultOutput.resultColumnMetadata.add(columnMetadata);

      if (logger.isDebugEnabled())
        logger.debug("Get column metadata: {}",
             columnMetadata.toString());
    }

    resultOutput.currentChunkRowset = rootNode.path("data").path("rowset");

    if (resultOutput.currentChunkRowset == null ||
        resultOutput.currentChunkRowset.isMissingNode())
    {
      resultOutput.currentChunkRowCount = 0;
    }
    else
    {
      resultOutput.currentChunkRowCount = resultOutput.currentChunkRowset.size();
    }

    logger.debug("First chunk row count: {}",
        resultOutput.currentChunkRowCount);

    JsonNode chunksNode = rootNode.path("data").path("chunks");

    if (!chunksNode.isMissingNode())
    {
      resultOutput.chunkCount = chunksNode.size();

      // Try to get the Query Result Master Key
      JsonNode qrmkNode = rootNode.path("data").path("qrmk");
      final String qrmk = qrmkNode.isMissingNode() ?
          null : qrmkNode.textValue();

      JsonNode chunkHeaders = rootNode.path("data").path("chunkHeaders");

      // the initialization of chunk downloader will start prefetching
      // first few chunks
      if (resultOutput.chunkCount > 0)
      {
        logger.debug("#chunks={}, initialize chunk downloader",
            resultOutput.chunkCount);

        // prefetch threads
        int resultPrefetchThreads = 4;
        if (resultOutput.parameters.get(CLIENT_PREFETCH_THREADS) != null)
        {
          resultPrefetchThreads =
              (int) resultOutput.parameters.get(CLIENT_PREFETCH_THREADS);
        }

        /*
         * Should JsonParser be used instead of the original Json deserializer.
         */
        boolean useJsonParser = true;
        if (resultOutput.parameters.get("JDBC_USE_JSON_PARSER") != null)
        {
          useJsonParser =
              (boolean) resultOutput.parameters.get("JDBC_USE_JSON_PARSER");
        }

        int memoryUsage = 1536;
        if (resultOutput.parameters.get(CLIENT_MEMORY_LIMIT) != null)
        {
          memoryUsage =
              (int) resultOutput.parameters.get(CLIENT_MEMORY_LIMIT);
        }

        boolean efficientChunkStorage = false;
        if (resultOutput.parameters.get("JDBC_EFFICIENT_CHUNK_STORAGE") != null)
        {
          efficientChunkStorage = (boolean)
              resultOutput.parameters.get("JDBC_EFFICIENT_CHUNK_STORAGE");
        }

        if (useJsonParser && efficientChunkStorage)
        {
          resultPrefetchThreads =
              Math.max(Math.min(resultPrefetchThreads,
                                Runtime.getRuntime().availableProcessors() / 2),
                       2);
        }
        else
        {
          resultPrefetchThreads = 1;
        }
        // initialize the chunk downloader
        resultOutput.chunkDownloader =
            new SnowflakeChunkDownloader(resultOutput.columnCount,
                                         chunksNode,
                                         resultPrefetchThreads,
                                         qrmk,
                                         chunkHeaders,
                                         resultData.networkTimeoutInMilli,
                                         useJsonParser,
                            (long) memoryUsage * 1024 * 1024,
                                         efficientChunkStorage);
      }
    }

    String sqlTimestampFormat = (String) effectiveParamValue(
        resultOutput.parameters, "TIMESTAMP_OUTPUT_FORMAT");

    // Special handling of specialized formatters, use a helper function
    resultOutput.timestampNTZFormatter = specializedFormatter(
        resultOutput.parameters,
        "timestamp_ntz",
        "TIMESTAMP_NTZ_OUTPUT_FORMAT",
        sqlTimestampFormat);

    resultOutput.timestampLTZFormatter = specializedFormatter(
        resultOutput.parameters,
        "timestamp_ltz",
        "TIMESTAMP_LTZ_OUTPUT_FORMAT",
        sqlTimestampFormat);

    resultOutput.timestampTZFormatter = specializedFormatter(
        resultOutput.parameters,
{'fraction_non_alphanumeric': 0.08218726838507782, 'fraction_numerical': 0.0029646710038705427, 'mean_word_length': 2.8127158555729985, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1213794', 'n_tokens_mistral': 6691, 'n_tokens_neox': 6474, 'n_words': 1297}