<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"context"
	"flag"
	"fmt"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/gorilla/mux"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"
)

var port string
var dockercli, err = client.NewClient("unix:///var/run/docker.sock", "v1.25", nil, map[string]string{"User-Agent": "nanoserverless"})
var tagprefix = "nanoserverless"
var registry string

type base struct {
  Run string
  FromImg string
  ExtraBuild string
  PreCode string
  PostCode string
}

var bases = make(map[string]base)

func init() {
	flag.StringVar(&port, "port", "80", "give me a port number")
	registry = os.Getenv("REGISTRY_URL")
	if registry != "" {
		registry += "/"
	}
  bases["php7"] = base{"#!/bin/sh\nphp app", "php:7", "", "<?php", "?>"}
  bases["node7"] = base{"#!/bin/sh\nnode app", "node:7", "", "", ""}
  bases["java8"] = base{
    "#!/bin/sh\njava app",
    "openjdk:8",
    "RUN mv app app.java && javac app.java",
    "public class app {\npublic static void main(String[] args) {",
    "}\n}",
  }
  bases["go17"] = base{"", "golang:1.7", "WORKDIR /go/src\nENV CGO_ENABLED=0\nENV GO_PATH=/go/src\nRUN mv /app ./app.go && go build -a --installsuffix cgo --ldflags=-s -o /run", "package main\nimport \"fmt\"\nfunc init() {}\nfunc main() {", "}"}
}

func main() {
	// defer profile.Start().Stop()
	flag.Parse()

	// Docker

	// Router
	r := mux.NewRouter()
	r.HandleFunc("/{base}/{name}", infofunc)
	r.HandleFunc("/{base}/{name}/create", create)
	r.HandleFunc("/{base}/{name}/exec", exec)
	r.HandleFunc("/{base}/{name}/up", up)
	r.HandleFunc("/{base}/{name}/down", down)
	r.HandleFunc("/whoami", whoami)
	http.Handle("/", r)
	fmt.Println("Starting up on port " + port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func whoami(w http.ResponseWriter, req *http.Request) {
	u, _ := url.Parse(req.URL.String())
	queryParams := u.Query()
	wait := queryParams.Get("wait")
	if len(wait) > 0 {
		duration, err := time.ParseDuration(wait)
		if err == nil {
			time.Sleep(duration)
		}
	}

	hostname, _ := os.Hostname()
	fmt.Fprintln(w, "Hostname:", hostname)

	req.Write(w)
}

func infofunc(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	fmt.Fprintln(w, "Service", servicename, "status :")

	// Get tasks
	serviceNameFilter := filters.NewArgs()
	serviceNameFilter.Add("name", servicename)
	tasks, err := dockercli.TaskList(ctx, types.TaskListOptions{
		Filters: serviceNameFilter,
	})
	if err != nil {
		log.Fatal(err)
	}
	for _, task := range tasks {
		fmt.Fprintln(w, "Task", task.Slot, task.Status.ContainerStatus.ContainerID, task.Status.State, "("+task.Status.Message+")")
	}
	if len(tasks) == 0 {
		fmt.Fprintln(w, "Not UP")
	}

	//fmt.Fprintln(w, "You're trying to get info on the", base, name, "function")
	//fmt.Fprintln(w, "But it's not implement yet :D")
}

func down(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	err := dockercli.ServiceRemove(ctx, servicename)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintln(w, "Service", servicename, "removed")

}

func up(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	/* Goal :
	docker service create \
	  --name nanoserverless-node7-pi \
	  --network nanoserverless \
	  nanoserverless-node7-pi
	*/

	// Network
	network := swarm.NetworkAttachmentConfig{
		Target: "nanoserverless",
	}

	// Create
	service := swarm.ServiceSpec{
		Annotations: swarm.Annotations{
			Name: servicename,
			//Labels: runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
		},
		TaskTemplate: swarm.TaskSpec{
			ContainerSpec: swarm.ContainerSpec{
				Image: registry + tag,
				/*Args:     opts.args,
				Env:      currentEnv,
				Hostname: opts.hostname,
				Labels:   runconfigopts.ConvertKVStringsToMap(opts.containerLabels.GetAll()),
				Dir:      opts.workdir,
				User:     opts.user,
				Groups:   opts.groups.GetAll(),
				TTY:      opts.tty,
				ReadOnly: opts.readOnly,
				Mounts:   opts.mounts.Value(),
				DNSConfig: &swarm.DNSConfig{
					Nameservers: opts.dns.GetAll(),
					Search:      opts.dnsSearch.GetAll(),
					Options:     opts.dnsOption.GetAll(),
				},
				Hosts:           convertExtraHostsToSwarmHosts(opts.hosts.GetAll()),
				StopGracePeriod: opts.stopGrace.Value(),
				Secrets:         nil,
				Healthcheck:     healthConfig,*/
			},
			Networks: []swarm.NetworkAttachmentConfig{network},
			/*Resources:     opts.resources.ToResourceRequirements(),
			RestartPolicy: opts.restartPolicy.ToRestartPolicy(),
			Placement: &swarm.Placement{
				Constraints: opts.constraints.GetAll(),
			},
			LogDriver: opts.logDriver.toLogDriver(),*/
		},
		//Networks: convertNetworks(opts.networks.GetAll()),
		/*Mode:     serviceMode,
		UpdateConfig: &swarm.UpdateConfig{
			Parallelism:     opts.update.parallelism,
			Delay:           opts.update.delay,
			Monitor:         opts.update.monitor,
			FailureAction:   opts.update.onFailure,
			MaxFailureRatio: opts.update.maxFailureRatio.Value(),
		},
		EndpointSpec: opts.endpoint.ToEndpointSpec(),*/
	}

	resp, err := dockercli.ServiceCreate(ctx, service, types.ServiceCreateOptions{})
	if err != nil {
		panic(err)
	}

	fmt.Fprintln(w, "Service id ", resp.ID, "created")
}

func exec(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	containername := tag
	ctx := context.Background()

	// Test if we can http to the container
	resp_http, err := http.Get("http://" + containername)
	if err != nil {
		// Create
		resp, err := dockercli.ContainerCreate(ctx, &container.Config{
			Image:      registry + tag,
			Entrypoint: []string{"/run"},
			//      AttachStdout: true,
		}, nil, nil, containername)
		if err != nil {
			// Trying to pull image
			resp_pull, err := dockercli.ImagePull(ctx, registry+tag, types.ImagePullOptions{
				RegistryAuth: "<KEY>
			})
			// Wait pull finish
			buf_pull := new(bytes.Buffer)
			buf_pull.ReadFrom(resp_pull)
			buf_pull.String()

			if err != nil {
				panic(err)
			}

			resp, err = dockercli.ContainerCreate(ctx, &container.Config{
				Image:      registry + tag,
				Entrypoint: []string{"/run"},
			}, nil, nil, containername)
			if err != nil {
				panic(err)
			}
		}

		// Run
		if err := dockercli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
			panic(err)
		}

		// Wait
		if _, err = dockercli.ContainerWait(ctx, resp.ID); err != nil {
			panic(err)
		}

		// Logs
		responseBody, err := dockercli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
		if err != nil {
			panic(err)
		}
		defer responseBody.Close()

		// Print
		/*buf := new(bytes.Buffer)
		  buf.ReadFrom(out)
		  result := buf.String()
		  fmt.Fprintln(w, result)*/

		//io.Copy(w, []byte(out))
		/*fmt.Fprintln(w, "Result:")
		buf := new(bytes.Buffer)
		buf.ReadFrom(responseBody)
		newStr := buf.String()*/

		stdcopy.StdCopy(w, w, responseBody)
		//fmt.Fprintln(w, newStr)

		// Delete
		_ = dockercli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})

	} else {
		defer resp_http.Body.Close()
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp_http.Body)
		fmt.Fprint(w, buf.String())
	}

}

func create(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	bodyb, _ := ioutil.ReadAll(req.Body)
	body := string(bodyb)

  baseStruct, ok := bases[base]
  if ! ok {
    fmt.Fprintln(w, base, "not supported yet !")
    return
  }

	// Generate dockerfile
	dockerfile := "FROM "
  dockerfile += baseStruct.FromImg
	dockerfile += "\nCOPY shell2http /"
	dockerfile += "\nCOPY app /"
	dockerfile += "\nCOPY run /"
  dockerfile += "\n" + baseStruct.ExtraBuild
  dockerfile += "\nENTRYPOINT [\"/shell2http\", \"-port=80\", \"-cgi\", \"-export-all-vars\", \"/\", \"/run\"]"

	// Generate app
	app := ""
  app += baseStruct.PreCode + "\n"
	app += body + "\n"
  app += baseStruct.PostCode

	// Generate run
  run := baseStruct.Run

	// Buffer context
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)

	// Add some files
	var files = []struct {
		Name, Body string
	}{
		{"Dockerfile", dockerfile},
		{"app", app},
		{"run", run},
	}
	for _, file := range files {
		hdr := &tar.Header{
			Name: file.Name,<commit_msg>View Code<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"context"
	"flag"
	"fmt"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/gorilla/mux"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"
)

var port string
var dockercli, err = client.NewClient("unix:///var/run/docker.sock", "v1.25", nil, map[string]string{"User-Agent": "nanoserverless"})
var tagprefix = "nanoserverless"
var registry string

type base struct {
	Run        string
	ViewCode   []string
	FromImg    string
	ExtraBuild string
}

var bases = make(map[string]base)

func init() {
	flag.StringVar(&port, "port", "80", "give me a port number")
	registry = os.Getenv("REGISTRY_URL")
	if registry != "" {
		registry += "/"
	}
	bases["php7"] = base{
		"#!/bin/sh\nphp app",
		[]string{"cat", "/app"},
		"php:7",
		"",
	}
	bases["node7"] = base{
		"#!/bin/sh\nnode app",
		[]string{"cat", "/app"},
		"node:7",
		"",
	}
	bases["java8"] = base{
		"#!/bin/sh\njava app",
		[]string{"cat", "/app.java"},
		"openjdk:8",
		"RUN mv app app.java && javac app.java",
	}
	bases["go17"] = base{
		"",
		[]string{"cat", "/go/src/app.go"},
		"golang:1.7",
		"WORKDIR /go/src\nENV CGO_ENABLED=0\nENV GO_PATH=/go/src\nRUN mv /app ./app.go && go build -a --installsuffix cgo --ldflags=-s -o /run",
	}
}

func main() {
	// defer profile.Start().Stop()
	flag.Parse()

	// Docker

	// Router
	r := mux.NewRouter()
	r.HandleFunc("/{base}/{name}", infofunc)
	r.HandleFunc("/{base}/{name}/create", create)
	r.HandleFunc("/{base}/{name}/exec", exec)
	r.HandleFunc("/{base}/{name}/up", up)
	r.HandleFunc("/{base}/{name}/down", down)
	r.HandleFunc("/{base}/{name}/code", code)
	r.HandleFunc("/whoami", whoami)
	http.Handle("/", r)
	fmt.Println("Starting up on port " + port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func whoami(w http.ResponseWriter, req *http.Request) {
	u, _ := url.Parse(req.URL.String())
	queryParams := u.Query()
	wait := queryParams.Get("wait")
	if len(wait) > 0 {
		duration, err := time.ParseDuration(wait)
		if err == nil {
			time.Sleep(duration)
		}
	}

	hostname, _ := os.Hostname()
	fmt.Fprintln(w, "Hostname:", hostname)

	req.Write(w)
}

func infofunc(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	fmt.Fprintln(w, "Service", servicename, "status :")

	// Get tasks
	serviceNameFilter := filters.NewArgs()
	serviceNameFilter.Add("name", servicename)
	tasks, err := dockercli.TaskList(ctx, types.TaskListOptions{
		Filters: serviceNameFilter,
	})
	if err != nil {
		log.Fatal(err)
	}
	for _, task := range tasks {
		fmt.Fprintln(w, "Task", task.Slot, task.Status.ContainerStatus.ContainerID, task.Status.State, "("+task.Status.Message+")")
	}
	if len(tasks) == 0 {
		fmt.Fprintln(w, "Not UP")
	}

	//fmt.Fprintln(w, "You're trying to get info on the", base, name, "function")
	//fmt.Fprintln(w, "But it's not implement yet :D")
}

func down(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	err := dockercli.ServiceRemove(ctx, servicename)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintln(w, "Service", servicename, "removed")

}

func up(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	/* Goal :
	docker service create \
	  --name nanoserverless-node7-pi \
	  --network nanoserverless \
	  nanoserverless-node7-pi
	*/

	// Network
	network := swarm.NetworkAttachmentConfig{
		Target: "nanoserverless",
	}

	// Create
	service := swarm.ServiceSpec{
		Annotations: swarm.Annotations{
			Name: servicename,
			//Labels: runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
		},
		TaskTemplate: swarm.TaskSpec{
			ContainerSpec: swarm.ContainerSpec{
				Image: registry + tag,
				/*Args:     opts.args,
				Env:      currentEnv,
				Hostname: opts.hostname,
				Labels:   runconfigopts.ConvertKVStringsToMap(opts.containerLabels.GetAll()),
				Dir:      opts.workdir,
				User:     opts.user,
				Groups:   opts.groups.GetAll(),
				TTY:      opts.tty,
				ReadOnly: opts.readOnly,
				Mounts:   opts.mounts.Value(),
				DNSConfig: &swarm.DNSConfig{
					Nameservers: opts.dns.GetAll(),
					Search:      opts.dnsSearch.GetAll(),
					Options:     opts.dnsOption.GetAll(),
				},
				Hosts:           convertExtraHostsToSwarmHosts(opts.hosts.GetAll()),
				StopGracePeriod: opts.stopGrace.Value(),
				Secrets:         nil,
				Healthcheck:     healthConfig,*/
			},
			Networks: []swarm.NetworkAttachmentConfig{network},
			/*Resources:     opts.resources.ToResourceRequirements(),
			RestartPolicy: opts.restartPolicy.ToRestartPolicy(),
			Placement: &swarm.Placement{
				Constraints: opts.constraints.GetAll(),
			},
			LogDriver: opts.logDriver.toLogDriver(),*/
		},
		//Networks: convertNetworks(opts.networks.GetAll()),
		/*Mode:     serviceMode,
		UpdateConfig: &swarm.UpdateConfig{
			Parallelism:     opts.update.parallelism,
			Delay:           opts.update.delay,
			Monitor:         opts.update.monitor,
			FailureAction:   opts.update.onFailure,
			MaxFailureRatio: opts.update.maxFailureRatio.Value(),
		},
		EndpointSpec: opts.endpoint.ToEndpointSpec(),*/
	}

	resp, err := dockercli.ServiceCreate(ctx, service, types.ServiceCreateOptions{})
	if err != nil {
		panic(err)
	}

	fmt.Fprintln(w, "Service id ", resp.ID, "created")
}

func code(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	ctx := context.Background()

	baseStruct, ok := bases[base]
	if !ok {
		fmt.Fprintln(w, base, "not supported yet !")
		return
	}

	// Create
	resp, err := dockercli.ContainerCreate(ctx, &container.Config{
		Image:      registry + tag,
		Entrypoint: baseStruct.ViewCode,
		//      AttachStdout: true,
	}, nil, nil, "")
	if err != nil {
		// Trying to pull image
		resp_pull, err := dockercli.ImagePull(ctx, registry+tag, types.ImagePullOptions{
			RegistryAuth: "<KEY>
		})
		// Wait pull finish
		buf_pull := new(bytes.Buffer)
		buf_pull.ReadFrom(resp_pull)
		buf_pull.String()

		if err != nil {
			panic(err)
		}

		resp, err = dockercli.ContainerCreate(ctx, &container.Config{
			Image:      registry + tag,
			Entrypoint: baseStruct.ViewCode,
		}, nil, nil, "")
		if err != nil {
			panic(err)
		}
	}

	// Run
	if err := dockercli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		panic(err)
	}

	// Wait
	if _, err = dockercli.ContainerWait(ctx, resp.ID); err != nil {
		panic(err)
	}

	// Logs
	responseBody, err := dockercli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
	if err != nil {
		panic(err)
	}
	defer responseBody.Close()

	// Print
	/*buf := new(bytes.Buffer)
	  buf.ReadFrom(out)
	  result := buf.String()
	  fmt.Fprintln(w, result)*/

	//io.Copy(w, []byte(out))
	/*fmt.Fprintln(w, "Result:")
	  buf := new(bytes.Buffer)
	  buf.ReadFrom(responseBody)
	  newStr := buf.String()*/

	stdcopy.StdCopy(w, w, responseBody)
	//fmt.Fprintln(w, newStr)

	// Delete
	_ = dockercli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})

}

func exec(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	servicename := tag
	ctx := context.Background()

	// Test if we can http to the service
	resp_http, err := http.Get("http://" + servicename)
	if err != nil {
		// Create
		resp, err := dockercli.ContainerCreate(ctx, &container.Config{
			Image:      registry + tag,
			Entrypoint: []string{"/run"},
			//      AttachStdout: true,
		}, nil, nil, "")
		if err != nil {
			// Trying to pull image
			resp_pull, err := dockercli.ImagePull(ctx, registry+tag, types.ImagePullOptions{
				RegistryAuth: "<KEY>
			})
			// Wait pull finish
			buf_pull := new(bytes.Buffer)
			buf_pull.ReadFrom(resp_pull)
			buf_pull.String()

			if err != nil {
				panic(err)
			}

			resp, err = dockercli.ContainerCreate(ctx, &container.Config{
				Image:      registry + tag,
				Entrypoint: []string{"/run"},
			}, nil, nil, "")
			if err != nil {
				panic(err)
			}
		}

		// Run
		if err := dockercli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
			panic(err)
		}

		// Wait
		if _, err = dockercli.ContainerWait(ctx, resp.ID); err != nil {
			panic(err)
		}

		// Logs
		responseBody, err := dockercli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
		if err != nil {
			panic(err)
		}
		defer responseBody.Close()

		// Print
		/*buf := new(bytes.Buffer)
		  buf.ReadFrom(out)
		  result := buf.String()
		  fmt.Fprintln(w, result)*/

		//io.Copy(w, []byte(out))
		/*fmt.Fprintln(w, "Result:")
		buf := new(bytes.Buffer)
		buf.ReadFrom(responseBody)
		newStr := buf.String()*/

		stdcopy.StdCopy(w, w, responseBody)
		//fmt.Fprintln(w, newStr)

		// Delete
		_ = dockercli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})

	} else {
		defer resp_http.Body.Close()
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp_http.Body)
		fmt.Fprint(w, buf.String())
	}

}

func create(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	base := vars["base"]
	name := vars["name"]
	tag := tagprefix + "-" + base + "-" + name
	bodyb, _ := ioutil.ReadAll(req.Body)
	body := string(bodyb)

	baseStruct, ok := bases[base]
	if !ok {
		fmt.Fprintln(w, base, "not supported yet !")
		return
	}

	// Generate dockerfile
	dockerfile := "FROM "
	dockerfile += baseStruct.FromImg
	dockerfile += "\nCOPY shell2http /"
	dockerfile += "\nCOPY app /"
	dockerfile += "\nCOPY run /"
	dockerfile += "\n" + baseStruct.ExtraBuild
	dockerfile += "\nENTRYPOINT [\"/shell2http\", \"-port=80\", \"-cgi\", \"-export-all-vars\", \"/\", \"/run\"]"

	// Generate app
	//app := ""
	//app += baseStruct.PreCode + "\n"
	app := body
	//app += baseStruct.PostCode

	// Generate run
	run := baseStruct.Run

	// Buffer context
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)

	// Add some files
	var files = []struct {
		Name, Body string
	}{
		{"Dockerfile", dockerfile},
		{"app", app},
		{"run", run},
	}
	for _, file := range files {
		hdr := &tar.Header{
			Name: file.Name,
{'fraction_non_alphanumeric': 0.17743324720068906, 'fraction_numerical': 0.0024319805441556466, 'mean_word_length': 3.4107262569832404, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266490', 'n_tokens_mistral': 7867, 'n_tokens_neox': 7155, 'n_words': 1468}