<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
// Copyright 2017 Lyft, Inc.

package limits

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/aws/aws-sdk-go-v2/aws"

	ec2shim "github.com/cilium/cilium/pkg/aws/ec2"
	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
	"github.com/cilium/cilium/pkg/lock"
)

var limitsOnce sync.Once

// limit contains limits for adapter count and addresses. The mappings will be
// updated from agent configuration at bootstrap time.
//
// Source: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html?shortFooter=true#AvailableIpPerENI
//
// Generated using the following command (requires AWS cli & jq):
// AWS_REGION=us-east-1 aws ec2 describe-instance-types | jq -r '.InstanceTypes[] |
// 	"\"\(.InstanceType)\": {Adapters: \(.NetworkInfo.MaximumNetworkInterfaces), IPv4: \(.NetworkInfo.Ipv4AddressesPerInterface), IPv6: \(.NetworkInfo.Ipv6AddressesPerInterface)},"' \
// | sort
var limits struct {
	lock.RWMutex
	m map[string]ipamTypes.Limits
}

func populateStaticENILimits() {
	limits.m = map[string]ipamTypes.Limits{
		"a1.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"a1.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"a1.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"a1.medium":     {Adapters: 2, IPv4: 4, IPv6: 4},
		"a1.metal":      {Adapters: 8, IPv4: 30, IPv6: 30},
		"a1.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c1.medium":     {Adapters: 2, IPv4: 6, IPv6: 0},
		"c1.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 0},
		"c3.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c3.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c3.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c3.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c3.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c4.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c4.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c4.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c4.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c4.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5.12xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.18xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.24xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.9xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5.metal":      {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5a.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.16xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5a.24xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5a.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5a.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5a.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5ad.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5ad.24xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5ad.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5ad.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5ad.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5d.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.18xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.24xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5d.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.9xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5d.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5n.18xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5n.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5n.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5n.9xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5n.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5n.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5n.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6g.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.16xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6g.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6g.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6g.medium":    {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6g.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6g.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gd.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gd.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gd.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6gd.medium":   {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6gd.metal":    {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gd.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gn.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gn.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gn.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6gn.medium":   {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6gn.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"cc2.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 0},
		"d2.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"d2.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"d2.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"d2.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"d3.2xlarge":    {Adapters: 4, IPv4: 5, IPv6: 5},
		"d3.4xlarge":    {Adapters: 4, IPv4: 10, IPv6: 10},
		"d3.8xlarge":    {Adapters: 3, IPv4: 20, IPv6: 20},
		"d3.xlarge":     {Adapters: 4, IPv4: 3, IPv6: 3},
		"d3en.12xlarge": {Adapters: 3, IPv4: 30, IPv6: 30},
		"d3en.2xlarge":  {Adapters: 4, IPv4: 5, IPv6: 5},
		"d3en.4xlarge":  {Adapters: 4, IPv4: 10, IPv6: 10},
		"d3en.6xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"d3en.8xlarge":  {Adapters: 4, IPv4: 20, IPv6: 20},
		"d3en.xlarge":   {Adapters: 4, IPv4: 3, IPv6: 3},
		"f1.16xlarge":   {Adapters: 8, IPv4: 50, IPv6: 50},
		"f1.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"f1.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g2.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 0},
		"g2.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 0},
		"g3.16xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"g3.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g3.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g3s.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"g4ad.16xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"g4ad.4xlarge":  {Adapters: 3, IPv4: 10, IPv6: 10},
		"g4ad.8xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"g4dn.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"g4dn.16xlarge": {Adapters: 4, IPv4: 15, IPv6: 15},<commit_msg>Add Amazon EC2 c7g instance configuration<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
// Copyright 2017 Lyft, Inc.

package limits

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/aws/aws-sdk-go-v2/aws"

	ec2shim "github.com/cilium/cilium/pkg/aws/ec2"
	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
	"github.com/cilium/cilium/pkg/lock"
)

var limitsOnce sync.Once

// limit contains limits for adapter count and addresses. The mappings will be
// updated from agent configuration at bootstrap time.
//
// Source: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html?shortFooter=true#AvailableIpPerENI
//
// Generated using the following command (requires AWS cli & jq):
// AWS_REGION=us-east-1 aws ec2 describe-instance-types | jq -r '.InstanceTypes[] |
// 	"\"\(.InstanceType)\": {Adapters: \(.NetworkInfo.MaximumNetworkInterfaces), IPv4: \(.NetworkInfo.Ipv4AddressesPerInterface), IPv6: \(.NetworkInfo.Ipv6AddressesPerInterface)},"' \
// | sort
var limits struct {
	lock.RWMutex
	m map[string]ipamTypes.Limits
}

func populateStaticENILimits() {
	limits.m = map[string]ipamTypes.Limits{
		"a1.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"a1.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"a1.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"a1.medium":     {Adapters: 2, IPv4: 4, IPv6: 4},
		"a1.metal":      {Adapters: 8, IPv4: 30, IPv6: 30},
		"a1.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c1.medium":     {Adapters: 2, IPv4: 6, IPv6: 0},
		"c1.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 0},
		"c3.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c3.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c3.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c3.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c3.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c4.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c4.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c4.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c4.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c4.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5.12xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.18xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.24xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.9xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5.large":      {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5.metal":      {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5a.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.16xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5a.24xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5a.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5a.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5a.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5a.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5ad.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5ad.24xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5ad.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5ad.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5ad.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5ad.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5d.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.18xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.24xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5d.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.9xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5d.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5d.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5d.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5n.18xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5n.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c5n.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5n.9xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c5n.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c5n.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c5n.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6g.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.16xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6g.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6g.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6g.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6g.medium":    {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6g.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6g.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gd.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gd.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gd.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gd.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6gd.medium":   {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6gd.metal":    {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gd.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gn.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.16xlarge": {Adapters: 15, IPv4: 50, IPv6: 50},
		"c6gn.2xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"c6gn.4xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.8xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c6gn.large":    {Adapters: 3, IPv4: 10, IPv6: 10},
		"c6gn.medium":   {Adapters: 2, IPv4: 4, IPv6: 4},
		"c6gn.xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c7g.12xlarge":  {Adapters: 8, IPv4: 30, IPv6: 30},
		"c7g.16xlarge":  {Adapters: 15, IPv4: 50, IPv6: 50},
		"c7g.2xlarge":   {Adapters: 4, IPv4: 15, IPv6: 15},
		"c7g.4xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c7g.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 30},
		"c7g.large":     {Adapters: 3, IPv4: 10, IPv6: 10},
		"c7g.medium":    {Adapters: 2, IPv4: 4, IPv6: 4},
		"c7g.metal":     {Adapters: 15, IPv4: 50, IPv6: 50},
		"c7g.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"cc2.8xlarge":   {Adapters: 8, IPv4: 30, IPv6: 0},
		"d2.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"d2.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"d2.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"d2.xlarge":     {Adapters: 4, IPv4: 15, IPv6: 15},
		"d3.2xlarge":    {Adapters: 4, IPv4: 5, IPv6: 5},
		"d3.4xlarge":    {Adapters: 4, IPv4: 10, IPv6: 10},
		"d3.8xlarge":    {Adapters: 3, IPv4: 20, IPv6: 20},
		"d3.xlarge":     {Adapters: 4, IPv4: 3, IPv6: 3},
		"d3en.12xlarge": {Adapters: 3, IPv4: 30, IPv6: 30},
		"d3en.2xlarge":  {Adapters: 4, IPv4: 5, IPv6: 5},
		"d3en.4xlarge":  {Adapters: 4, IPv4: 10, IPv6: 10},
		"d3en.6xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"d3en.8xlarge":  {Adapters: 4, IPv4: 20, IPv6: 20},
		"d3en.xlarge":   {Adapters: 4, IPv4: 3, IPv6: 3},
		"f1.16xlarge":   {Adapters: 8, IPv4: 50, IPv6: 50},
		"f1.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"f1.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g2.2xlarge":    {Adapters: 4, IPv4: 15, IPv6: 0},
		"g2.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 0},
		"g3.16xlarge":   {Adapters: 15, IPv4: 50, IPv6: 50},
		"g3.4xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g3.8xlarge":    {Adapters: 8, IPv4: 30, IPv6: 30},
		"g3s.xlarge":    {Adapters: 4, IPv4: 15, IPv6: 15},
		"g4ad.16xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"g4ad.4xlarge":  {Adapters: 3, IPv4: 10, IPv6: 10},
		"g4ad.8xlarge":  {Adapters: 4, IPv4: 15, IPv6: 15},
		"g4dn.12xlarge": {Adapters: 8, IPv4: 30, IPv6: 30},
		"g4dn.16xlarge": {Adapters: 4, IPv4: 15, IPv6: 15},
{'fraction_non_alphanumeric': 0.21017083829956298, 'fraction_numerical': 0.14110713812740033, 'mean_word_length': 3.4976176295413937, 'pattern_counts': {'":': 239, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6410730', 'n_tokens_mistral': 9432, 'n_tokens_neox': 7213, 'n_words': 1829}