<commit_before>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/ghodss/yaml"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/mholt/archiver"
	"github.com/pkg/errors"
	"github.com/satori/go.uuid"
	"github.com/urfave/cli"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/fission/fission"
	"github.com/fission/fission/controller/client"
	"github.com/fission/fission/crd"
	"github.com/fission/fission/fission/log"
)

const SPEC_API_VERSION = "fission.io/v1"

const ARCHIVE_URL_PREFIX string = "archive://"

const SPEC_README = `
Fission Specs
=============

This is a set of specifications for a Fission app.  This includes functions,
environments, and triggers; we collectively call these things "resources".

How to use these specs
----------------------

These specs are handled with the 'fission spec' command.  See 'fission spec --help'.

'fission spec apply' will "apply" all resources specified in this directory to your
cluster.  That means it checks what resources exist on your cluster, what resources are
specified in the specs directory, and reconciles the difference by creating, updating or
deleting resources on the cluster.

'fission spec apply' will also package up your source code (or compiled binaries) and
upload the archives to the cluster if needed.  It uses 'ArchiveUploadSpec' resources in
this directory to figure out which files to archive.

You can use 'fission spec apply --watch' to watch for file changes and continuously keep
the cluster updated.

You can add YAMLs to this directory by writing them manually, but it's easier to generate
them.  Use 'fission function create --spec' to generate a function spec,
'fission environment create --spec' to generate an environment spec, and so on.

You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
which contains a UID that you should never change.  To apply your changes simply use
'fission spec apply'.

fission-deployment-config.yaml
------------------------------

fission-deployment-config.yaml contains a UID.  This UID is what fission uses to correlate
resources on the cluster to resources in this directory.

All resources created by 'fission spec apply' are annotated with this UID.  Resources on
the cluster that are _not_ annotated with this UID are never modified or deleted by
fission.

`

type (
	FissionResources struct {
		deploymentConfig        DeploymentConfig
		packages                []crd.Package
		functions               []crd.Function
		environments            []crd.Environment
		httpTriggers            []crd.HTTPTrigger
		kubernetesWatchTriggers []crd.KubernetesWatchTrigger
		timeTriggers            []crd.TimeTrigger
		messageQueueTriggers    []crd.MessageQueueTrigger
		archiveUploadSpecs      []ArchiveUploadSpec

		sourceMap sourceMap
	}

	resourceApplyStatus struct {
		created []*metav1.ObjectMeta
		updated []*metav1.ObjectMeta
		deleted []*metav1.ObjectMeta
	}

	location struct {
		path string
		line int
	}
	sourceMap struct {
		// kind -> namespace -> name -> location
		locations map[string](map[string](map[string]location))
	}
)

func getSpecDir(c *cli.Context) string {
	specDir := c.String("specdir")
	if len(specDir) == 0 {
		specDir = "specs"
	}
	return specDir
}

// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new
// fission-config.yaml in specDir.
func writeDeploymentConfig(specDir string, dc *DeploymentConfig) error {
	y, err := yaml.Marshal(dc)
	if err != nil {
		return err
	}

	msg := []byte("# This file is generated by the 'fission spec init' command.\n" +
		"# See the README in this directory for background and usage information.\n" +
		"# Do not edit the UID below: that will break 'fission spec apply'\n")

	err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644)
	if err != nil {
		return err
	}
	return nil
}

// specInit just initializes an empty spec directory and adds some
// sample YAMLs in there that might be useful.
func specInit(c *cli.Context) error {
	// Figure out spec directory
	specDir := getSpecDir(c)

	name := c.String("name")
	if len(name) == 0 {
		// come up with a name using the current dir
		dir, err := filepath.Abs(".")
		checkErr(err, "get current working directory")
		basename := filepath.Base(dir)
		name = kubifyName(basename)
	}

	// Create spec dir
	fmt.Printf("Creating fission spec directory '%v'\n", specDir)
	err := os.MkdirAll(specDir, 0755)
	checkErr(err, fmt.Sprintf("create spec directory '%v'", specDir))

	// Add a bit of documentation to the spec dir here
	err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644)
	if err != nil {
		return err
	}

	// Write the deployment config
	dc := DeploymentConfig{
		TypeMeta: TypeMeta{
			APIVersion: SPEC_API_VERSION,
			Kind:       "DeploymentConfig",
		},
		Name: name,

		// All resources will be annotated with the UID when they're created. This allows
		// us to be idempotent, as well as to delete resources when their specs are
		// removed.
		UID: uuid.NewV4().String(),
	}
	err = writeDeploymentConfig(specDir, &dc)
	checkErr(err, "write deployment config")

	// Other possible things to do here:
	// - add example specs to the dir to make it easy to manually
	//   add new ones

	return nil
}

// validateFunctionReference checks a function reference
func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fission.FunctionReference) error {
	if funcRef.Type == fission.FunctionReferenceTypeFunctionName {
		// triggers only reference functions in their own namespace
		namespace := meta.Namespace
		name := funcRef.Name
		m := &metav1.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		}
		if _, ok := functions[mapKey(m)]; !ok {
			return fmt.Errorf("%v: %v '%v' references unknown function '%v'",
				fr.sourceMap.locations[kind][meta.Namespace][meta.Name],
				kind,
				meta.Name,
				name)
		} else {
			functions[mapKey(m)] = true
		}
	}
	return nil
}

// specValidate parses a set of specs and checks for references to
// resources that don't exist.
func specValidate(c *cli.Context) error {
	// this will error on parse errors and on duplicates
	specDir := getSpecDir(c)
	fr, err := readSpecs(specDir)
	checkErr(err, "read specs")

	// this does the rest of the checks, like dangling refs
	err = fr.validate()
	if err != nil {
		fmt.Printf("Error validating specs: %v", err)
	}

	return nil
}

func (fr *FissionResources) validate() error {
	var result *multierror.Error

	// check references: both dangling refs + garbage
	//   packages -> archives
	//   functions -> packages
	//   functions -> environments + shared environments between functions [TODO]
	//   functions -> secrets + configmaps (same ns) [TODO]
	//   triggers -> functions

	// index archives
	archives := make(map[string]bool)
	for _, a := range fr.archiveUploadSpecs {
		archives[a.Name] = false
	}

	// index packages, check outgoing refs, mark archives that are referenced
	packages := make(map[string]bool)
	for _, p := range fr.packages {
		packages[mapKey(&p.Metadata)] = false

		// check archive refs from package
		aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX)
		if len(aname) > 0 {
			if _, ok := archives[aname]; !ok {
				result = multierror.Append(result, fmt.Errorf(
					"%v: package '%v' references unknown source archive %v%v",
					fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
					p.Metadata.Name,
					ARCHIVE_URL_PREFIX,
					aname))
			} else {
				archives[aname] = true
			}
		}

		aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX)
		if len(aname) > 0 {
			if _, ok := archives[aname]; !ok {
				result = multierror.Append(result, fmt.Errorf(
					"%v: package '%v' references unknown deployment archive %v%v",
					fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
					p.Metadata.Name,
					ARCHIVE_URL_PREFIX,
					aname))
			} else {
				archives[aname] = true
			}
		}
	}

	// error on unreferenced archives
	for name, referenced := range archives {
		if !referenced {
			result = multierror.Append(result, fmt.Errorf(
				"%v: archive '%v' is not used in any package",
				fr.sourceMap.locations["ArchiveUploadSpec"][""][name],
				name))
		}
	}

	// index functions, check function package refs, mark referenced packages
	functions := make(map[string]bool)
	for _, f := range fr.functions {
		functions[mapKey(&f.Metadata)] = false

		pkgMeta := &metav1.ObjectMeta{
			Name:      f.Spec.Package.PackageRef.Name,
			Namespace: f.Spec.Package.PackageRef.Namespace,
		}

		// check package ref from function
		packageRefExists := func() bool {
			_, ok := packages[mapKey(pkgMeta)]
			return ok
		}

		// check that the package referenced by each function is in the same ns as the function
		packageRefInFuncNs := func(f *crd.Function) bool {
			return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace
		}

		if !packageRefInFuncNs(&f) {
			result = multierror.Append(result, fmt.Errorf(
				"%v: function '%v' references a package outside of its namespace %v/%v",
				fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
				f.Metadata.Name,
				f.Spec.Package.PackageRef.Namespace,
				f.Spec.Package.PackageRef.Name))
		} else if !packageRefExists() {
			result = multierror.Append(result, fmt.Errorf(
				"%v: function '%v' references unknown package %v/%v",
				fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
				f.Metadata.Name,
				pkgMeta.Namespace,
				pkgMeta.Name))
		} else {
			packages[mapKey(pkgMeta)] = true
		}
	}

	// error on unreferenced packages
	for key, referenced := range packages {
		ks := strings.Split(key, ":")
		namespace, name := ks[0], ks[1]
		if !referenced {
			result = multierror.Append(result, fmt.Errorf(
				"%v: package '%v' is not used in any function",
				fr.sourceMap.locations["Package"][namespace][name],
				name))
		}
	}

	// check function refs from triggers
	for _, t := range fr.httpTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}
	for _, t := range fr.kubernetesWatchTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}
	for _, t := range fr.timeTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}
	for _, t := range fr.messageQueueTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}

	// we do not error on unreferenced functions (you can call a function through workflows,
	// `fission function test`, etc.)

	// (ErrorOrNil returns nil if there were no errors appended.)
	return result.ErrorOrNil()
}

func (loc location) String() string {
	return fmt.Sprintf("%v:%v", loc.path, loc.line)
}

// Keep track of source location of resources, and track duplicates
func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *location) error {<commit_msg>Do resources validation when validate spec files (#840)<commit_after>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/ghodss/yaml"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/mholt/archiver"
	"github.com/pkg/errors"
	"github.com/satori/go.uuid"
	"github.com/urfave/cli"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/fission/fission"
	"github.com/fission/fission/controller/client"
	"github.com/fission/fission/crd"
	"github.com/fission/fission/fission/log"
)

const SPEC_API_VERSION = "fission.io/v1"

const ARCHIVE_URL_PREFIX string = "archive://"

const SPEC_README = `
Fission Specs
=============

This is a set of specifications for a Fission app.  This includes functions,
environments, and triggers; we collectively call these things "resources".

How to use these specs
----------------------

These specs are handled with the 'fission spec' command.  See 'fission spec --help'.

'fission spec apply' will "apply" all resources specified in this directory to your
cluster.  That means it checks what resources exist on your cluster, what resources are
specified in the specs directory, and reconciles the difference by creating, updating or
deleting resources on the cluster.

'fission spec apply' will also package up your source code (or compiled binaries) and
upload the archives to the cluster if needed.  It uses 'ArchiveUploadSpec' resources in
this directory to figure out which files to archive.

You can use 'fission spec apply --watch' to watch for file changes and continuously keep
the cluster updated.

You can add YAMLs to this directory by writing them manually, but it's easier to generate
them.  Use 'fission function create --spec' to generate a function spec,
'fission environment create --spec' to generate an environment spec, and so on.

You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
which contains a UID that you should never change.  To apply your changes simply use
'fission spec apply'.

fission-deployment-config.yaml
------------------------------

fission-deployment-config.yaml contains a UID.  This UID is what fission uses to correlate
resources on the cluster to resources in this directory.

All resources created by 'fission spec apply' are annotated with this UID.  Resources on
the cluster that are _not_ annotated with this UID are never modified or deleted by
fission.

`

type (
	FissionResources struct {
		deploymentConfig        DeploymentConfig
		packages                []crd.Package
		functions               []crd.Function
		environments            []crd.Environment
		httpTriggers            []crd.HTTPTrigger
		kubernetesWatchTriggers []crd.KubernetesWatchTrigger
		timeTriggers            []crd.TimeTrigger
		messageQueueTriggers    []crd.MessageQueueTrigger
		archiveUploadSpecs      []ArchiveUploadSpec

		sourceMap sourceMap
	}

	resourceApplyStatus struct {
		created []*metav1.ObjectMeta
		updated []*metav1.ObjectMeta
		deleted []*metav1.ObjectMeta
	}

	location struct {
		path string
		line int
	}
	sourceMap struct {
		// kind -> namespace -> name -> location
		locations map[string](map[string](map[string]location))
	}
)

func getSpecDir(c *cli.Context) string {
	specDir := c.String("specdir")
	if len(specDir) == 0 {
		specDir = "specs"
	}
	return specDir
}

// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new
// fission-config.yaml in specDir.
func writeDeploymentConfig(specDir string, dc *DeploymentConfig) error {
	y, err := yaml.Marshal(dc)
	if err != nil {
		return err
	}

	msg := []byte("# This file is generated by the 'fission spec init' command.\n" +
		"# See the README in this directory for background and usage information.\n" +
		"# Do not edit the UID below: that will break 'fission spec apply'\n")

	err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644)
	if err != nil {
		return err
	}
	return nil
}

// specInit just initializes an empty spec directory and adds some
// sample YAMLs in there that might be useful.
func specInit(c *cli.Context) error {
	// Figure out spec directory
	specDir := getSpecDir(c)

	name := c.String("name")
	if len(name) == 0 {
		// come up with a name using the current dir
		dir, err := filepath.Abs(".")
		checkErr(err, "get current working directory")
		basename := filepath.Base(dir)
		name = kubifyName(basename)
	}

	// Create spec dir
	fmt.Printf("Creating fission spec directory '%v'\n", specDir)
	err := os.MkdirAll(specDir, 0755)
	checkErr(err, fmt.Sprintf("create spec directory '%v'", specDir))

	// Add a bit of documentation to the spec dir here
	err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644)
	if err != nil {
		return err
	}

	// Write the deployment config
	dc := DeploymentConfig{
		TypeMeta: TypeMeta{
			APIVersion: SPEC_API_VERSION,
			Kind:       "DeploymentConfig",
		},
		Name: name,

		// All resources will be annotated with the UID when they're created. This allows
		// us to be idempotent, as well as to delete resources when their specs are
		// removed.
		UID: uuid.NewV4().String(),
	}
	err = writeDeploymentConfig(specDir, &dc)
	checkErr(err, "write deployment config")

	// Other possible things to do here:
	// - add example specs to the dir to make it easy to manually
	//   add new ones

	return nil
}

// validateFunctionReference checks a function reference
func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fission.FunctionReference) error {
	if funcRef.Type == fission.FunctionReferenceTypeFunctionName {
		// triggers only reference functions in their own namespace
		namespace := meta.Namespace
		name := funcRef.Name
		m := &metav1.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		}
		if _, ok := functions[mapKey(m)]; !ok {
			return fmt.Errorf("%v: %v '%v' references unknown function '%v'",
				fr.sourceMap.locations[kind][meta.Namespace][meta.Name],
				kind,
				meta.Name,
				name)
		} else {
			functions[mapKey(m)] = true
		}
	}
	return nil
}

// specValidate parses a set of specs and checks for references to
// resources that don't exist.
func specValidate(c *cli.Context) error {
	// this will error on parse errors and on duplicates
	specDir := getSpecDir(c)
	fr, err := readSpecs(specDir)
	checkErr(err, "read specs")

	// this does the rest of the checks, like dangling refs
	err = fr.validate()
	if err != nil {
		fmt.Printf("Error validating specs: %v", err)
	}

	return nil
}

func (fr *FissionResources) validate() error {
	var result *multierror.Error

	// check references: both dangling refs + garbage
	//   packages -> archives
	//   functions -> packages
	//   functions -> environments + shared environments between functions [TODO]
	//   functions -> secrets + configmaps (same ns) [TODO]
	//   triggers -> functions

	// index archives
	archives := make(map[string]bool)
	for _, a := range fr.archiveUploadSpecs {
		archives[a.Name] = false
	}

	// index packages, check outgoing refs, mark archives that are referenced
	packages := make(map[string]bool)
	for _, p := range fr.packages {
		packages[mapKey(&p.Metadata)] = false

		// check archive refs from package
		aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX)
		if len(aname) > 0 {
			if _, ok := archives[aname]; !ok {
				result = multierror.Append(result, fmt.Errorf(
					"%v: package '%v' references unknown source archive %v%v",
					fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
					p.Metadata.Name,
					ARCHIVE_URL_PREFIX,
					aname))
			} else {
				archives[aname] = true
			}
		}

		aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX)
		if len(aname) > 0 {
			if _, ok := archives[aname]; !ok {
				result = multierror.Append(result, fmt.Errorf(
					"%v: package '%v' references unknown deployment archive %v%v",
					fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
					p.Metadata.Name,
					ARCHIVE_URL_PREFIX,
					aname))
			} else {
				archives[aname] = true
			}
		}

		result = multierror.Append(result, p.Validate())
	}

	// error on unreferenced archives
	for name, referenced := range archives {
		if !referenced {
			result = multierror.Append(result, fmt.Errorf(
				"%v: archive '%v' is not used in any package",
				fr.sourceMap.locations["ArchiveUploadSpec"][""][name],
				name))
		}
	}

	// index functions, check function package refs, mark referenced packages
	functions := make(map[string]bool)
	for _, f := range fr.functions {
		functions[mapKey(&f.Metadata)] = false

		pkgMeta := &metav1.ObjectMeta{
			Name:      f.Spec.Package.PackageRef.Name,
			Namespace: f.Spec.Package.PackageRef.Namespace,
		}

		// check package ref from function
		packageRefExists := func() bool {
			_, ok := packages[mapKey(pkgMeta)]
			return ok
		}

		// check that the package referenced by each function is in the same ns as the function
		packageRefInFuncNs := func(f *crd.Function) bool {
			return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace
		}

		if !packageRefInFuncNs(&f) {
			result = multierror.Append(result, fmt.Errorf(
				"%v: function '%v' references a package outside of its namespace %v/%v",
				fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
				f.Metadata.Name,
				f.Spec.Package.PackageRef.Namespace,
				f.Spec.Package.PackageRef.Name))
		} else if !packageRefExists() {
			result = multierror.Append(result, fmt.Errorf(
				"%v: function '%v' references unknown package %v/%v",
				fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
				f.Metadata.Name,
				pkgMeta.Namespace,
				pkgMeta.Name))
		} else {
			packages[mapKey(pkgMeta)] = true
		}

		result = multierror.Append(result, f.Validate())
	}

	// error on unreferenced packages
	for key, referenced := range packages {
		ks := strings.Split(key, ":")
		namespace, name := ks[0], ks[1]
		if !referenced {
			result = multierror.Append(result, fmt.Errorf(
				"%v: package '%v' is not used in any function",
				fr.sourceMap.locations["Package"][namespace][name],
				name))
		}
	}

	// check function refs from triggers
	for _, t := range fr.httpTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
		result = multierror.Append(result, t.Validate())
	}
	for _, t := range fr.kubernetesWatchTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
		result = multierror.Append(result, t.Validate())
	}
	for _, t := range fr.timeTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
		result = multierror.Append(result, t.Validate())
	}
	for _, t := range fr.messageQueueTriggers {
		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
		if err != nil {
			result = multierror.Append(result, err)
		}
		result = multierror.Append(result, t.Validate())
	}

	// we do not error on unreferenced functions (you can call a function through workflows,
	// `fission function test`, etc.)

	// (ErrorOrNil returns nil if there were no errors appended.)
	return result.ErrorOrNil()
}

func (loc location) String() string {
	return fmt.Sprintf("%v:%v", loc.path, loc.line)
}

// Keep track of source location of resources, and track duplicates
func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *location) error {
{'fraction_non_alphanumeric': 0.11492890995260663, 'fraction_numerical': 0.0032276515770550745, 'mean_word_length': 3.708926510196229, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2796784', 'n_tokens_mistral': 8213, 'n_tokens_neox': 7496, 'n_words': 2421}