<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"github.com/juju/cmd"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/arch"
	"github.com/juju/utils/featureflag"
	"github.com/juju/utils/series"
	"github.com/juju/utils/set"
	"github.com/juju/version"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/juju/application"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/cmd/juju/cloud"
	"github.com/juju/juju/cmd/modelcmd"
	cmdtesting "github.com/juju/juju/cmd/testing"
	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	_ "github.com/juju/juju/provider/dummy"
	"github.com/juju/juju/testing"
	jujuversion "github.com/juju/juju/version"
)

type MainSuite struct {
	testing.FakeJujuXDGDataHomeSuite
	gitjujutesting.PatchExecHelper
}

var _ = gc.Suite(&MainSuite{})

func deployHelpText() string {
	return cmdtesting.HelpText(application.NewDeployCommand(), "juju deploy")
}
func setconfigHelpText() string {
	return cmdtesting.HelpText(application.NewSetCommand(), "juju set-config")
}

func syncToolsHelpText() string {
	return cmdtesting.HelpText(newSyncToolsCommand(), "juju sync-tools")
}

func blockHelpText() string {
	return cmdtesting.HelpText(block.NewSuperBlockCommand(), "juju block")
}

func (s *MainSuite) TestRunMain(c *gc.C) {
	// The test array structure needs to be inline here as some of the
	// expected values below use deployHelpText().  This constructs the deploy
	// command and runs gets the help for it.  When the deploy command is
	// setting the flags (which is needed for the help text) it is accessing
	// osenv.JujuXDGDataHome(), which panics if SetJujuXDGDataHome has not been called.
	// The FakeHome from testing does this.
	for i, t := range []struct {
		summary string
		args    []string
		code    int
		out     string
	}{{
		summary: "juju help foo doesn't exist",
		args:    []string{"help", "foo"},
		code:    1,
		out:     "ERROR unknown command or topic for foo\n",
	}, {
		summary: "juju help deploy shows the default help without global options",
		args:    []string{"help", "deploy"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju --help deploy shows the same help as 'help deploy'",
		args:    []string{"--help", "deploy"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju deploy --help shows the same help as 'help deploy'",
		args:    []string{"deploy", "--help"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju --help set-config shows the same help as 'help set-config'",
		args:    []string{"--help", "set-config"},
		code:    0,
		out:     setconfigHelpText(),
	}, {
		summary: "juju set-config --help shows the same help as 'help set-config'",
		args:    []string{"set-config", "--help"},
		code:    0,
		out:     setconfigHelpText(),
	}, {
		summary: "unknown command",
		args:    []string{"discombobulate"},
		code:    1,
		out:     "ERROR unrecognized command: juju discombobulate\n",
	}, {
		summary: "unknown option before command",
		args:    []string{"--cheese", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "unknown option after command",
		args:    []string{"bootstrap", "--cheese"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "known option, but specified before command",
		args:    []string{"--model", "blah", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --model\n",
	}, {
		summary: "juju sync-tools registered properly",
		args:    []string{"sync-tools", "--help"},
		code:    0,
		out:     syncToolsHelpText(),
	}, {
		summary: "check version command returns a fully qualified version string",
		args:    []string{"version"},
		code:    0,
		out: version.Binary{
			Number: jujuversion.Current,
			Arch:   arch.HostArch(),
			Series: series.HostSeries(),
		}.String() + "\n",
	}, {
		summary: "check block command registered properly",
		args:    []string{"block", "-h"},
		code:    0,
		out:     blockHelpText(),
	}, {
		summary: "check unblock command registered properly",
		args:    []string{"unblock"},
		code:    0,
		out:     "error: must specify one of [destroy-model | remove-object | all-changes] to unblock\n",
	},
	} {
		c.Logf("test %d: %s", i, t.summary)
		out := badrun(c, t.code, t.args...)
		c.Assert(out, gc.Equals, t.out)
	}
}

func (s *MainSuite) TestActualRunJujuArgOrder(c *gc.C) {
	//TODO(bogdanteleaga): cannot read the env file because of some suite
	//problems. The juju home, when calling something from the command line is
	//not the same as in the test suite.
	if runtime.GOOS == "windows" {
		c.Skip("bug 1403084: cannot read env file on windows because of suite problems")
	}
	s.PatchEnvironment(osenv.JujuModelEnvKey, "current")
	logpath := filepath.Join(c.MkDir(), "log")
	tests := [][]string{
		{"--log-file", logpath, "--debug", "controllers"}, // global flags before
		{"controllers", "--log-file", logpath, "--debug"}, // after
		{"--log-file", logpath, "controllers", "--debug"}, // mixed
	}
	for i, test := range tests {
		c.Logf("test %d: %v", i, test)
		badrun(c, 0, test...)
		content, err := ioutil.ReadFile(logpath)
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(string(content), gc.Matches, "(.|\n)*running juju(.|\n)*command finished(.|\n)*")
		err = os.Remove(logpath)
		c.Assert(err, jc.ErrorIsNil)
	}
}

func (s *MainSuite) TestFirstRun2xFrom1xOnUbuntu(c *gc.C) {
	if runtime.GOOS == "windows" {
		// This test can't work on Windows and shouldn't need to
		c.Skip("test doesn't work on Windows because Juju's 1.x and 2.x config directory are the same")
	}

	// Code should only run on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)

	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	// remove the new juju-home and create a fake old juju home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)
	makeValidOldHome(c)

	var code int
	f := func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	}

	stdout, stderr := gitjujutesting.CaptureOutput(c, f)

	select {
	case args := <-argChan:
		c.Assert(args, gc.DeepEquals, []string{"juju-1", "version"})
	default:
		c.Fatalf("Exec function not called.")
	}

	c.Check(code, gc.Equals, 0)
	c.Check(string(stderr), gc.Equals, fmt.Sprintf(`
    Welcome to Juju %s. If you meant to use Juju 1.25.0 you can continue using it
    with the command juju-1 e.g. 'juju-1 switch'.
    See https://jujucharms.com/docs/stable/introducing-2 for more details.

Since Juju 2 is being run for the first time, downloading latest cloud information.
Fetching latest public cloud list...
Your list of public clouds is up to date, see `[1:]+"`juju clouds`.\n", jujuversion.Current))
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestFirstRun2xFrom1xNotUbuntu(c *gc.C) {
	// Code should only run on ubuntu series, so pretend to be something else.
	s.PatchValue(&series.HostSeries, func() string { return "win8" })

	argChan := make(chan []string, 1)

	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	// remove the new juju-home and create a fake old juju home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)

	makeValidOldHome(c)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)

	c.Assert(string(stderr), gc.Equals, `
Since Juju 2 is being run for the first time, downloading latest cloud information.
Fetching latest public cloud list...
Your list of public clouds is up to date, see `[1:]+"`juju clouds`.\n")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestNoWarn1xWith2xData(c *gc.C) {
	// Code should only rnu on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)

	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	// there should be a 2x home directory already created by the test setup.

	// create a fake old juju home.
	makeValidOldHome(c)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)
	c.Assert(string(stderr), gc.Equals, "")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestNoWarnWithNo1xOr2xData(c *gc.C) {
	// Code should only rnu on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)
	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	// remove the new juju-home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)

	// create fake (empty) old juju home.
	path := c.MkDir()
	s.PatchEnvironment("JUJU_HOME", path)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)
	c.Assert(string(stderr), gc.Equals, `
Since Juju 2 is being run for the first time, downloading latest cloud information.
Fetching latest public cloud list...
Your list of public clouds is up to date, see `[1:]+"`juju clouds`.\n")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) assertRunCommandUpdateCloud(c *gc.C, expectedCall string) {
	argChan := make(chan []string, 1)
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	stub := &gitjujutesting.Stub{}
	s.PatchValue(&cloud.NewUpdateCloudsCommand, func() cmd.Command {
		return &stubCommand{stub: stub}<commit_msg>Ensure main_test is isolated from internet<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"github.com/juju/cmd"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/arch"
	"github.com/juju/utils/featureflag"
	"github.com/juju/utils/series"
	"github.com/juju/utils/set"
	"github.com/juju/version"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/juju/application"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/cmd/juju/cloud"
	"github.com/juju/juju/cmd/modelcmd"
	cmdtesting "github.com/juju/juju/cmd/testing"
	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	_ "github.com/juju/juju/provider/dummy"
	"github.com/juju/juju/testing"
	jujuversion "github.com/juju/juju/version"
)

type MainSuite struct {
	testing.FakeJujuXDGDataHomeSuite
	gitjujutesting.PatchExecHelper
}

var _ = gc.Suite(&MainSuite{})

func deployHelpText() string {
	return cmdtesting.HelpText(application.NewDeployCommand(), "juju deploy")
}
func setconfigHelpText() string {
	return cmdtesting.HelpText(application.NewSetCommand(), "juju set-config")
}

func syncToolsHelpText() string {
	return cmdtesting.HelpText(newSyncToolsCommand(), "juju sync-tools")
}

func blockHelpText() string {
	return cmdtesting.HelpText(block.NewSuperBlockCommand(), "juju block")
}

func (s *MainSuite) TestRunMain(c *gc.C) {
	// The test array structure needs to be inline here as some of the
	// expected values below use deployHelpText().  This constructs the deploy
	// command and runs gets the help for it.  When the deploy command is
	// setting the flags (which is needed for the help text) it is accessing
	// osenv.JujuXDGDataHome(), which panics if SetJujuXDGDataHome has not been called.
	// The FakeHome from testing does this.
	for i, t := range []struct {
		summary string
		args    []string
		code    int
		out     string
	}{{
		summary: "juju help foo doesn't exist",
		args:    []string{"help", "foo"},
		code:    1,
		out:     "ERROR unknown command or topic for foo\n",
	}, {
		summary: "juju help deploy shows the default help without global options",
		args:    []string{"help", "deploy"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju --help deploy shows the same help as 'help deploy'",
		args:    []string{"--help", "deploy"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju deploy --help shows the same help as 'help deploy'",
		args:    []string{"deploy", "--help"},
		code:    0,
		out:     deployHelpText(),
	}, {
		summary: "juju --help set-config shows the same help as 'help set-config'",
		args:    []string{"--help", "set-config"},
		code:    0,
		out:     setconfigHelpText(),
	}, {
		summary: "juju set-config --help shows the same help as 'help set-config'",
		args:    []string{"set-config", "--help"},
		code:    0,
		out:     setconfigHelpText(),
	}, {
		summary: "unknown command",
		args:    []string{"discombobulate"},
		code:    1,
		out:     "ERROR unrecognized command: juju discombobulate\n",
	}, {
		summary: "unknown option before command",
		args:    []string{"--cheese", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "unknown option after command",
		args:    []string{"bootstrap", "--cheese"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "known option, but specified before command",
		args:    []string{"--model", "blah", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --model\n",
	}, {
		summary: "juju sync-tools registered properly",
		args:    []string{"sync-tools", "--help"},
		code:    0,
		out:     syncToolsHelpText(),
	}, {
		summary: "check version command returns a fully qualified version string",
		args:    []string{"version"},
		code:    0,
		out: version.Binary{
			Number: jujuversion.Current,
			Arch:   arch.HostArch(),
			Series: series.HostSeries(),
		}.String() + "\n",
	}, {
		summary: "check block command registered properly",
		args:    []string{"block", "-h"},
		code:    0,
		out:     blockHelpText(),
	}, {
		summary: "check unblock command registered properly",
		args:    []string{"unblock"},
		code:    0,
		out:     "error: must specify one of [destroy-model | remove-object | all-changes] to unblock\n",
	},
	} {
		c.Logf("test %d: %s", i, t.summary)
		out := badrun(c, t.code, t.args...)
		c.Assert(out, gc.Equals, t.out)
	}
}

func (s *MainSuite) TestActualRunJujuArgOrder(c *gc.C) {
	//TODO(bogdanteleaga): cannot read the env file because of some suite
	//problems. The juju home, when calling something from the command line is
	//not the same as in the test suite.
	if runtime.GOOS == "windows" {
		c.Skip("bug 1403084: cannot read env file on windows because of suite problems")
	}
	s.PatchEnvironment(osenv.JujuModelEnvKey, "current")
	logpath := filepath.Join(c.MkDir(), "log")
	tests := [][]string{
		{"--log-file", logpath, "--debug", "controllers"}, // global flags before
		{"controllers", "--log-file", logpath, "--debug"}, // after
		{"--log-file", logpath, "controllers", "--debug"}, // mixed
	}
	for i, test := range tests {
		c.Logf("test %d: %v", i, test)
		badrun(c, 0, test...)
		content, err := ioutil.ReadFile(logpath)
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(string(content), gc.Matches, "(.|\n)*running juju(.|\n)*command finished(.|\n)*")
		err = os.Remove(logpath)
		c.Assert(err, jc.ErrorIsNil)
	}
}

func (s *MainSuite) TestFirstRun2xFrom1xOnUbuntu(c *gc.C) {
	if runtime.GOOS == "windows" {
		// This test can't work on Windows and shouldn't need to
		c.Skip("test doesn't work on Windows because Juju's 1.x and 2.x config directory are the same")
	}

	// Code should only run on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)

	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})
	stub := &gitjujutesting.Stub{}
	s.PatchValue(&cloud.NewUpdateCloudsCommand, func() cmd.Command {
		return &stubCommand{stub: stub}
	})

	// remove the new juju-home and create a fake old juju home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)
	makeValidOldHome(c)

	var code int
	f := func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	}

	stdout, stderr := gitjujutesting.CaptureOutput(c, f)

	select {
	case args := <-argChan:
		c.Assert(args, gc.DeepEquals, []string{"juju-1", "version"})
	default:
		c.Fatalf("Exec function not called.")
	}

	c.Check(code, gc.Equals, 0)
	c.Check(string(stderr), gc.Equals, fmt.Sprintf(`
    Welcome to Juju %s. If you meant to use Juju 1.25.0 you can continue using it
    with the command juju-1 e.g. 'juju-1 switch'.
    See https://jujucharms.com/docs/stable/introducing-2 for more details.

Since Juju 2 is being run for the first time, downloading latest cloud information.`[1:]+"\n", jujuversion.Current))
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestFirstRun2xFrom1xNotUbuntu(c *gc.C) {
	// Code should only run on ubuntu series, so pretend to be something else.
	s.PatchValue(&series.HostSeries, func() string { return "win8" })

	argChan := make(chan []string, 1)

	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})
	stub := &gitjujutesting.Stub{}
	s.PatchValue(&cloud.NewUpdateCloudsCommand, func() cmd.Command {
		return &stubCommand{stub: stub}
	})

	// remove the new juju-home and create a fake old juju home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)

	makeValidOldHome(c)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)

	c.Check(string(stderr), gc.Equals, `
Since Juju 2 is being run for the first time, downloading latest cloud information.`[1:]+"\n")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestNoWarn1xWith2xData(c *gc.C) {
	// Code should only rnu on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)

	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	// there should be a 2x home directory already created by the test setup.

	// create a fake old juju home.
	makeValidOldHome(c)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)
	c.Assert(string(stderr), gc.Equals, "")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) TestNoWarnWithNo1xOr2xData(c *gc.C) {
	// Code should only rnu on ubuntu series, so patch out the series for
	// when non-ubuntu OSes run this test.
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })

	argChan := make(chan []string, 1)
	// we shouldn't actually be running anything, but if we do, this will
	// provide some consistent results.
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})
	stub := &gitjujutesting.Stub{}
	s.PatchValue(&cloud.NewUpdateCloudsCommand, func() cmd.Command {
		return &stubCommand{stub: stub}
	})

	// remove the new juju-home.
	err := os.RemoveAll(osenv.JujuXDGDataHomeDir())
	c.Assert(err, jc.ErrorIsNil)

	// create fake (empty) old juju home.
	path := c.MkDir()
	s.PatchEnvironment("JUJU_HOME", path)

	var code int
	stdout, stderr := gitjujutesting.CaptureOutput(c, func() {
		code = main{
			execCommand: execCommand,
		}.Run([]string{"juju", "version"})
	})

	c.Assert(code, gc.Equals, 0)

	assertNoArgs(c, argChan)
	c.Check(string(stderr), gc.Equals, `
Since Juju 2 is being run for the first time, downloading latest cloud information.`[1:]+"\n")
	checkVersionOutput(c, string(stdout))
}

func (s *MainSuite) assertRunCommandUpdateCloud(c *gc.C, expectedCall string) {
	argChan := make(chan []string, 1)
	execCommand := s.GetExecCommand(gitjujutesting.PatchExecConfig{
		Stdout: "1.25.0-trusty-amd64",
		Args:   argChan,
	})

	stub := &gitjujutesting.Stub{}
	s.PatchValue(&cloud.NewUpdateCloudsCommand, func() cmd.Command {
		return &stubCommand{stub: stub}
{'fraction_non_alphanumeric': 0.14788312462731068, 'fraction_numerical': 0.008898674372735196, 'mean_word_length': 3.845965770171149, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1758563', 'n_tokens_mistral': 8286, 'n_tokens_neox': 7636, 'n_words': 2125}