<commit_before>/*
 * Copyright 2013 The Chromium OS Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * SPI driver for Chrome EC.
 *
 * This uses DMA to handle transmission and reception.
 */

#include "chipset.h"
#include "clock.h"
#include "console.h"
#include "dma.h"
#include "gpio.h"
#include "hooks.h"
#include "link_defs.h"
#include "registers.h"
#include "spi.h"
#include "stm32-dma.h"
#include "system.h"
#include "timer.h"
#include "util.h"

/* Console output macros */
#define CPUTS(outstr) cputs(CC_SPI, outstr)
#define CPRINTS(format, args...) cprints(CC_SPI, format, ## args)
#define CPRINTF(format, args...) cprintf(CC_SPI, format, ## args)

/* SPI FIFO registers */
#ifdef CHIP_FAMILY_STM32H7
#define SPI_TXDR REG8(&STM32_SPI1_REGS->txdr)
#define SPI_RXDR REG8(&STM32_SPI1_REGS->rxdr)
#else
#define SPI_TXDR STM32_SPI1_REGS->dr
#define SPI_RXDR STM32_SPI1_REGS->dr
#endif

/* DMA channel option */
static const struct dma_option dma_tx_option = {
	STM32_DMAC_SPI1_TX, (void *)&SPI_TXDR,
	STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT
#ifdef CHIP_FAMILY_STM32F4
	| STM32_DMA_CCR_CHANNEL(STM32_SPI1_TX_REQ_CH)
#endif
};

static const struct dma_option dma_rx_option = {
	STM32_DMAC_SPI1_RX, (void *)&SPI_RXDR,
	STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT
#ifdef CHIP_FAMILY_STM32F4
	| STM32_DMA_CCR_CHANNEL(STM32_SPI1_RX_REQ_CH)
#endif
};

/*
 * Timeout to wait for SPI request packet
 *
 * This affects the slowest SPI clock we can support.  A delay of 8192 us
 * permits a 512-byte request at 500 KHz, assuming the master starts sending
 * bytes as soon as it asserts chip select.  That's as slow as we would
 * practically want to run the SPI interface, since running it slower
 * significantly impacts firmware update times.
 */
#define SPI_CMD_RX_TIMEOUT_US 8192

#ifdef CONFIG_SPI_PROTOCOL_V2
/*
 * Offset of output parameters needs to account for pad and framing bytes and
 * one last past-end byte at the end so any additional bytes clocked out by
 * the AP will have a known and identifiable value.
 */
#define SPI_PROTO2_OFFSET (EC_PROTO2_RESPONSE_HEADER_BYTES + 2)
#define SPI_PROTO2_OVERHEAD (SPI_PROTO2_OFFSET +		\
			     EC_PROTO2_RESPONSE_TRAILER_BYTES + 1)
#endif /* defined(CONFIG_SPI_PROTOCOL_V2) */
/*
 * Max data size for a version 3 request/response packet.  This is big enough
 * to handle a request/response header, flash write offset/size, and 512 bytes
 * of flash data.
 */
#define SPI_MAX_REQUEST_SIZE 0x220
#define SPI_MAX_RESPONSE_SIZE 0x220

/*
 * The AP blindly clocks back bytes over the SPI interface looking for a
 * framing byte.  So this preamble must always precede the actual response
 * packet.  Search for "spi-frame-header" in U-boot to see how that's
 * implemented.
 *
 * The preamble must be 32-bit aligned so that the response buffer is also
 * 32-bit aligned.
 */
static const uint8_t out_preamble[4] = {
	EC_SPI_PROCESSING,
	EC_SPI_PROCESSING,
	EC_SPI_PROCESSING,
	EC_SPI_FRAME_START,  /* This is the byte which matters */
};

/*
 * Space allocation of the past-end status byte (EC_SPI_PAST_END) in the out_msg
 * buffer. This seems to be dynamic because the F0 family needs to send it 4
 * times in order to make sure it actually stays at the repeating byte after DMA
 * ends.
 *
 * See crosbug.com/p/31390
 */
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
#define EC_SPI_PAST_END_LENGTH 4
#else
#define EC_SPI_PAST_END_LENGTH 1
#endif

/*
 * Our input and output buffers. These must be large enough for our largest
 * message, including protocol overhead, and must be 32-bit aligned.
 */
static uint8_t out_msg[SPI_MAX_RESPONSE_SIZE + sizeof(out_preamble) +
	EC_SPI_PAST_END_LENGTH] __aligned(4) __uncached;
static uint8_t in_msg[SPI_MAX_REQUEST_SIZE] __aligned(4) __uncached;
static uint8_t enabled;
#ifdef CONFIG_SPI_PROTOCOL_V2
static struct host_cmd_handler_args args;
#endif
static struct host_packet spi_packet;

/*
 * This is set if SPI NSS raises to high while EC is still processing a
 * command.
 */
static int setup_transaction_later;

enum spi_state {
	/* SPI not enabled (initial state, and when chipset is off) */
	SPI_STATE_DISABLED = 0,

	/* Setting up receive DMA */
	SPI_STATE_PREPARE_RX,

	/* Ready to receive next request */
	SPI_STATE_READY_TO_RX,

	/* Receiving request */
	SPI_STATE_RECEIVING,

	/* Processing request */
	SPI_STATE_PROCESSING,

	/* Sending response */
	SPI_STATE_SENDING,

	/*
	 * Received bad data - transaction started before we were ready, or
	 * packet header from host didn't parse properly.  Ignoring received
	 * data.
	 */
	SPI_STATE_RX_BAD,
} state;

/**
 * Wait until we have received a certain number of bytes
 *
 * Watch the DMA receive channel until it has the required number of bytes,
 * or a timeout occurs
 *
 * We keep an eye on the NSS line - if this goes high then the transaction is
 * over so there is no point in trying to receive the bytes.
 *
 * @param rxdma		RX DMA channel to watch
 * @param needed	Number of bytes that are needed
 * @param nss		GPIO signal for NSS control line
 * @return 0 if bytes received, -1 if we hit a timeout or NSS went high
 */
static int wait_for_bytes(dma_chan_t *rxdma, int needed,
			  enum gpio_signal nss)
{
	timestamp_t deadline;

	ASSERT(needed <= sizeof(in_msg));
	deadline.val = 0;
	while (1) {
		if (dma_bytes_done(rxdma, sizeof(in_msg)) >= needed)
			return 0;
		if (gpio_get_level(nss))
			return -1;
		if (!deadline.val) {
			deadline = get_time();
			deadline.val += SPI_CMD_RX_TIMEOUT_US;
		}
		if (timestamp_expired(deadline, NULL))
			return -1;
	}
}

#ifdef CONFIG_SPI_PROTOCOL_V2
/**
 * Send a reply on a given port.
 *
 * The format of a reply is as per the command interface, with a number of
 * preamble bytes before it.
 *
 * The format of a reply is a sequence of bytes:
 *
 * <hdr> <status> <len> <msg bytes> <sum> [<preamble byte>...]
 *
 * The hdr byte is just a tag to indicate that the real message follows. It
 * signals the end of any preamble required by the interface.
 *
 * The length is the entire packet size, including the header, length bytes,
 * message payload, checksum, and postamble byte.
 *
 * The preamble is at least 2 bytes, but can be longer if the STM takes ages
 * to react to the incoming message. Since we send our first byte as the AP
 * sends us the command, we clearly can't send anything sensible for that
 * byte. The second byte must be written to the output register just when the
 * command byte is ready (I think), so we can't do anything there either.
 * Any processing we do may increase this delay. That's the reason for the
 * preamble.
 *
 * It is interesting to note that it seems to be possible to run the SPI
 * interface faster than the CPU clock with this approach.
 *
 * We keep an eye on the NSS line - if this goes high then the transaction is
 * over so there is no point in trying to send the reply.
 *
 * @param txdma		TX DMA channel to send on
 * @param status	Status result to send
 * @param msg_ptr	Message payload to send, which normally starts
 *			SPI_PROTO2_OFFSET bytes into out_msg
 * @param msg_len	Number of message bytes to send
 */
static void reply(dma_chan_t *txdma,
		  enum ec_status status, char *msg_ptr, int msg_len)
{
	char *msg = out_msg;
	int need_copy = msg_ptr != msg + SPI_PROTO2_OFFSET;
	int sum, i;

	ASSERT(msg_len + SPI_PROTO2_OVERHEAD <= sizeof(out_msg));

	/* Add our header bytes - the first one might not actually be sent */
	msg[0] = EC_SPI_PROCESSING;
	msg[1] = EC_SPI_FRAME_START;
	msg[2] = status;
	msg[3] = msg_len & 0xff;

	/*
	 * Calculate the checksum; includes the status and message length bytes
	 * but not the pad and framing bytes since those are stripped by the AP
	 * driver.
	 */
	sum = status + msg_len;
	for (i = 0; i < msg_len; i++) {
		int ch = msg_ptr[i];
		sum += ch;
		if (need_copy)
			msg[i + SPI_PROTO2_OFFSET] = ch;
	}

	/* Add the checksum and get ready to send */
	msg[SPI_PROTO2_OFFSET + msg_len] = sum & 0xff;
	msg[SPI_PROTO2_OFFSET + msg_len + 1] = EC_SPI_PAST_END;
	dma_prepare_tx(&dma_tx_option, msg_len + SPI_PROTO2_OVERHEAD, msg);

	/* Kick off the DMA to send the data */
	dma_go(txdma);
}
#endif /* defined(CONFIG_SPI_PROTOCOL_V2) */

/**
 * Sends a byte over SPI without DMA
 *
 * This is mostly used when we want to relay status bytes to the AP while we're
 * receiving the message and we're thinking about it.
 *
 * @note It may be sent 0, 1, or >1 times, depending on whether the host clocks
 * the bus or not. Basically, the EC is saying "if you ask me what my status is,
 * you'll get this value.  But you're not required to ask, or you can ask
 * multiple times."
 *
 * @param byte	status byte to send, one of the EC_SPI_* #defines from
 *		ec_commands.h
 */
static void tx_status(uint8_t byte)
{
	stm32_spi_regs_t *spi __attribute__((unused)) = STM32_SPI1_REGS;

	SPI_TXDR = byte;
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
	/* It sends the byte 4 times in order to be sure it bypassed the FIFO
	 * from the STM32F0 line.
	 */
	spi->dr = byte;
	spi->dr = byte;
	spi->dr = byte;
#elif defined(CHIP_FAMILY_STM32H7)
	spi->udrdr = byte;
#endif
}

/**
 * Get ready to receive a message from the master.
 *
 * Set up our RX DMA and disable our TX DMA. Set up the data output so that
 * we will send preamble bytes.
 */
static void setup_for_transaction(void)
{
	stm32_spi_regs_t *spi __attribute__((unused)) = STM32_SPI1_REGS;
	volatile uint8_t unused __attribute__((unused));

	/* clear this as soon as possible */
	setup_transaction_later = 0;

#ifndef CHIP_FAMILY_STM32H7 /* H7 is not ready to set status here */
	/* Not ready to receive yet */
	tx_status(EC_SPI_NOT_READY);
#endif

	/* We are no longer actively processing a transaction */
	state = SPI_STATE_PREPARE_RX;

	/* Stop sending response, if any */
	dma_disable(STM32_DMAC_SPI1_TX);

	/*
	 * Read unused bytes in case there are some pending; this prevents the
	 * receive DMA from getting that byte right when we start it.
	 */
	unused = SPI_RXDR;
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
	/* 4 Bytes makes sure the RX FIFO on the F0 is empty as well. */
	unused = spi->dr;
	unused = spi->dr;
	unused = spi->dr;
#endif

	/* Start DMA */
	dma_start_rx(&dma_rx_option, sizeof(in_msg), in_msg);

	/* Ready to receive */
	state = SPI_STATE_READY_TO_RX;
	tx_status(EC_SPI_OLD_READY);<commit_msg>ec: stm32: Use EC_SPI_RX_READY macro<commit_after>/*
 * Copyright 2013 The Chromium OS Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * SPI driver for Chrome EC.
 *
 * This uses DMA to handle transmission and reception.
 */

#include "chipset.h"
#include "clock.h"
#include "console.h"
#include "dma.h"
#include "gpio.h"
#include "hooks.h"
#include "link_defs.h"
#include "registers.h"
#include "spi.h"
#include "stm32-dma.h"
#include "system.h"
#include "timer.h"
#include "util.h"

/* Console output macros */
#define CPUTS(outstr) cputs(CC_SPI, outstr)
#define CPRINTS(format, args...) cprints(CC_SPI, format, ## args)
#define CPRINTF(format, args...) cprintf(CC_SPI, format, ## args)

/* SPI FIFO registers */
#ifdef CHIP_FAMILY_STM32H7
#define SPI_TXDR REG8(&STM32_SPI1_REGS->txdr)
#define SPI_RXDR REG8(&STM32_SPI1_REGS->rxdr)
#else
#define SPI_TXDR STM32_SPI1_REGS->dr
#define SPI_RXDR STM32_SPI1_REGS->dr
#endif

/* DMA channel option */
static const struct dma_option dma_tx_option = {
	STM32_DMAC_SPI1_TX, (void *)&SPI_TXDR,
	STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT
#ifdef CHIP_FAMILY_STM32F4
	| STM32_DMA_CCR_CHANNEL(STM32_SPI1_TX_REQ_CH)
#endif
};

static const struct dma_option dma_rx_option = {
	STM32_DMAC_SPI1_RX, (void *)&SPI_RXDR,
	STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT
#ifdef CHIP_FAMILY_STM32F4
	| STM32_DMA_CCR_CHANNEL(STM32_SPI1_RX_REQ_CH)
#endif
};

/*
 * Timeout to wait for SPI request packet
 *
 * This affects the slowest SPI clock we can support.  A delay of 8192 us
 * permits a 512-byte request at 500 KHz, assuming the master starts sending
 * bytes as soon as it asserts chip select.  That's as slow as we would
 * practically want to run the SPI interface, since running it slower
 * significantly impacts firmware update times.
 */
#define SPI_CMD_RX_TIMEOUT_US 8192

#ifdef CONFIG_SPI_PROTOCOL_V2
/*
 * Offset of output parameters needs to account for pad and framing bytes and
 * one last past-end byte at the end so any additional bytes clocked out by
 * the AP will have a known and identifiable value.
 */
#define SPI_PROTO2_OFFSET (EC_PROTO2_RESPONSE_HEADER_BYTES + 2)
#define SPI_PROTO2_OVERHEAD (SPI_PROTO2_OFFSET +		\
			     EC_PROTO2_RESPONSE_TRAILER_BYTES + 1)
#endif /* defined(CONFIG_SPI_PROTOCOL_V2) */
/*
 * Max data size for a version 3 request/response packet.  This is big enough
 * to handle a request/response header, flash write offset/size, and 512 bytes
 * of flash data.
 */
#define SPI_MAX_REQUEST_SIZE 0x220
#define SPI_MAX_RESPONSE_SIZE 0x220

/*
 * The AP blindly clocks back bytes over the SPI interface looking for a
 * framing byte.  So this preamble must always precede the actual response
 * packet.  Search for "spi-frame-header" in U-boot to see how that's
 * implemented.
 *
 * The preamble must be 32-bit aligned so that the response buffer is also
 * 32-bit aligned.
 */
static const uint8_t out_preamble[4] = {
	EC_SPI_PROCESSING,
	EC_SPI_PROCESSING,
	EC_SPI_PROCESSING,
	EC_SPI_FRAME_START,  /* This is the byte which matters */
};

/*
 * Space allocation of the past-end status byte (EC_SPI_PAST_END) in the out_msg
 * buffer. This seems to be dynamic because the F0 family needs to send it 4
 * times in order to make sure it actually stays at the repeating byte after DMA
 * ends.
 *
 * See crosbug.com/p/31390
 */
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
#define EC_SPI_PAST_END_LENGTH 4
#else
#define EC_SPI_PAST_END_LENGTH 1
#endif

/*
 * Our input and output buffers. These must be large enough for our largest
 * message, including protocol overhead, and must be 32-bit aligned.
 */
static uint8_t out_msg[SPI_MAX_RESPONSE_SIZE + sizeof(out_preamble) +
	EC_SPI_PAST_END_LENGTH] __aligned(4) __uncached;
static uint8_t in_msg[SPI_MAX_REQUEST_SIZE] __aligned(4) __uncached;
static uint8_t enabled;
#ifdef CONFIG_SPI_PROTOCOL_V2
static struct host_cmd_handler_args args;
#endif
static struct host_packet spi_packet;

/*
 * This is set if SPI NSS raises to high while EC is still processing a
 * command.
 */
static int setup_transaction_later;

enum spi_state {
	/* SPI not enabled (initial state, and when chipset is off) */
	SPI_STATE_DISABLED = 0,

	/* Setting up receive DMA */
	SPI_STATE_PREPARE_RX,

	/* Ready to receive next request */
	SPI_STATE_READY_TO_RX,

	/* Receiving request */
	SPI_STATE_RECEIVING,

	/* Processing request */
	SPI_STATE_PROCESSING,

	/* Sending response */
	SPI_STATE_SENDING,

	/*
	 * Received bad data - transaction started before we were ready, or
	 * packet header from host didn't parse properly.  Ignoring received
	 * data.
	 */
	SPI_STATE_RX_BAD,
} state;

/**
 * Wait until we have received a certain number of bytes
 *
 * Watch the DMA receive channel until it has the required number of bytes,
 * or a timeout occurs
 *
 * We keep an eye on the NSS line - if this goes high then the transaction is
 * over so there is no point in trying to receive the bytes.
 *
 * @param rxdma		RX DMA channel to watch
 * @param needed	Number of bytes that are needed
 * @param nss		GPIO signal for NSS control line
 * @return 0 if bytes received, -1 if we hit a timeout or NSS went high
 */
static int wait_for_bytes(dma_chan_t *rxdma, int needed,
			  enum gpio_signal nss)
{
	timestamp_t deadline;

	ASSERT(needed <= sizeof(in_msg));
	deadline.val = 0;
	while (1) {
		if (dma_bytes_done(rxdma, sizeof(in_msg)) >= needed)
			return 0;
		if (gpio_get_level(nss))
			return -1;
		if (!deadline.val) {
			deadline = get_time();
			deadline.val += SPI_CMD_RX_TIMEOUT_US;
		}
		if (timestamp_expired(deadline, NULL))
			return -1;
	}
}

#ifdef CONFIG_SPI_PROTOCOL_V2
/**
 * Send a reply on a given port.
 *
 * The format of a reply is as per the command interface, with a number of
 * preamble bytes before it.
 *
 * The format of a reply is a sequence of bytes:
 *
 * <hdr> <status> <len> <msg bytes> <sum> [<preamble byte>...]
 *
 * The hdr byte is just a tag to indicate that the real message follows. It
 * signals the end of any preamble required by the interface.
 *
 * The length is the entire packet size, including the header, length bytes,
 * message payload, checksum, and postamble byte.
 *
 * The preamble is at least 2 bytes, but can be longer if the STM takes ages
 * to react to the incoming message. Since we send our first byte as the AP
 * sends us the command, we clearly can't send anything sensible for that
 * byte. The second byte must be written to the output register just when the
 * command byte is ready (I think), so we can't do anything there either.
 * Any processing we do may increase this delay. That's the reason for the
 * preamble.
 *
 * It is interesting to note that it seems to be possible to run the SPI
 * interface faster than the CPU clock with this approach.
 *
 * We keep an eye on the NSS line - if this goes high then the transaction is
 * over so there is no point in trying to send the reply.
 *
 * @param txdma		TX DMA channel to send on
 * @param status	Status result to send
 * @param msg_ptr	Message payload to send, which normally starts
 *			SPI_PROTO2_OFFSET bytes into out_msg
 * @param msg_len	Number of message bytes to send
 */
static void reply(dma_chan_t *txdma,
		  enum ec_status status, char *msg_ptr, int msg_len)
{
	char *msg = out_msg;
	int need_copy = msg_ptr != msg + SPI_PROTO2_OFFSET;
	int sum, i;

	ASSERT(msg_len + SPI_PROTO2_OVERHEAD <= sizeof(out_msg));

	/* Add our header bytes - the first one might not actually be sent */
	msg[0] = EC_SPI_PROCESSING;
	msg[1] = EC_SPI_FRAME_START;
	msg[2] = status;
	msg[3] = msg_len & 0xff;

	/*
	 * Calculate the checksum; includes the status and message length bytes
	 * but not the pad and framing bytes since those are stripped by the AP
	 * driver.
	 */
	sum = status + msg_len;
	for (i = 0; i < msg_len; i++) {
		int ch = msg_ptr[i];
		sum += ch;
		if (need_copy)
			msg[i + SPI_PROTO2_OFFSET] = ch;
	}

	/* Add the checksum and get ready to send */
	msg[SPI_PROTO2_OFFSET + msg_len] = sum & 0xff;
	msg[SPI_PROTO2_OFFSET + msg_len + 1] = EC_SPI_PAST_END;
	dma_prepare_tx(&dma_tx_option, msg_len + SPI_PROTO2_OVERHEAD, msg);

	/* Kick off the DMA to send the data */
	dma_go(txdma);
}
#endif /* defined(CONFIG_SPI_PROTOCOL_V2) */

/**
 * Sends a byte over SPI without DMA
 *
 * This is mostly used when we want to relay status bytes to the AP while we're
 * receiving the message and we're thinking about it.
 *
 * @note It may be sent 0, 1, or >1 times, depending on whether the host clocks
 * the bus or not. Basically, the EC is saying "if you ask me what my status is,
 * you'll get this value.  But you're not required to ask, or you can ask
 * multiple times."
 *
 * @param byte	status byte to send, one of the EC_SPI_* #defines from
 *		ec_commands.h
 */
static void tx_status(uint8_t byte)
{
	stm32_spi_regs_t *spi __attribute__((unused)) = STM32_SPI1_REGS;

	SPI_TXDR = byte;
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
	/* It sends the byte 4 times in order to be sure it bypassed the FIFO
	 * from the STM32F0 line.
	 */
	spi->dr = byte;
	spi->dr = byte;
	spi->dr = byte;
#elif defined(CHIP_FAMILY_STM32H7)
	spi->udrdr = byte;
#endif
}

/**
 * Get ready to receive a message from the master.
 *
 * Set up our RX DMA and disable our TX DMA. Set up the data output so that
 * we will send preamble bytes.
 */
static void setup_for_transaction(void)
{
	stm32_spi_regs_t *spi __attribute__((unused)) = STM32_SPI1_REGS;
	volatile uint8_t unused __attribute__((unused));

	/* clear this as soon as possible */
	setup_transaction_later = 0;

#ifndef CHIP_FAMILY_STM32H7 /* H7 is not ready to set status here */
	/* Not ready to receive yet */
	tx_status(EC_SPI_NOT_READY);
#endif

	/* We are no longer actively processing a transaction */
	state = SPI_STATE_PREPARE_RX;

	/* Stop sending response, if any */
	dma_disable(STM32_DMAC_SPI1_TX);

	/*
	 * Read unused bytes in case there are some pending; this prevents the
	 * receive DMA from getting that byte right when we start it.
	 */
	unused = SPI_RXDR;
#if defined(CHIP_FAMILY_STM32F0) || defined(CHIP_FAMILY_STM32L4)
	/* 4 Bytes makes sure the RX FIFO on the F0 is empty as well. */
	unused = spi->dr;
	unused = spi->dr;
	unused = spi->dr;
#endif

	/* Start DMA */
	dma_start_rx(&dma_rx_option, sizeof(in_msg), in_msg);

	/* Ready to receive */
	state = SPI_STATE_READY_TO_RX;
	tx_status(EC_SPI_RX_READY);
{'fraction_non_alphanumeric': 0.08481715489182462, 'fraction_numerical': 0.018380241240666284, 'mean_word_length': 3.9852063946552136, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '605890', 'n_tokens_mistral': 7503, 'n_tokens_neox': 7097, 'n_words': 2639}