<commit_before>package main

import (
	"github.com/Sirupsen/logrus"
	"github.com/asteris-llc/hammer/hammer"
	"github.com/asteris-llc/hammer/hammer/cache"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
	"os"
	"os/signal"
)

var (
	buildCmd = &cobra.Command{
		Use:   "build [package...]",
		Short: "build packages",
		Long:  "build all packages by default, unless specific packages are specified",
		Run: func(cmd *cobra.Command, packageNames []string) {
			loader := hammer.NewLoader(viper.GetString("search"))
			loaded, err := loader.Load()
			if err != nil {
				logrus.WithField("error", err).Fatal("could not load packages")
			}

			var packages []*hammer.Package
			if len(packageNames) == 0 {
				packages = loaded
			} else {
				packages = []*hammer.Package{}
				for _, name := range packageNames {
					found := false

					for _, pkg := range loaded {
						if pkg.Name == name {
							packages = append(packages, pkg)
							found = true
							break
						}
					}

					if !found {
						logrus.WithField("name", name).Warn("could not find package")
					}
				}
			}

			if len(packages) == 0 {
				logrus.Fatal("no packages selected")
			}

			fsCache, err := cache.NewFSCache(viper.GetString("cache"))
			if err != nil {
				logrus.WithField("error", err).Fatal("could not make cache")
			}
			for _, pkg := range packages {
				pkg.SetCache(fsCache)
			}

			packager := hammer.NewPackager(packages)

			err = packager.EnsureOutputDir(viper.GetString("output"))
			if err != nil {
				logrus.WithField("error", err).Fatal("could not create output directory")
			}

			ctx, cancel := context.WithCancel(context.Background())

			// handle interrupts so we can clean up nicely
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)
			go func() {
				for {
					select {
					case <-c:
						logrus.Warn("interrupted, exiting cleanly")
						cancel()

					case <-ctx.Done():
						return
					}
				}
			}()<commit_msg>build: create log directory before we begin<commit_after>package main

import (
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	"github.com/asteris-llc/hammer/hammer"
	"github.com/asteris-llc/hammer/hammer/cache"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
)

var (
	buildCmd = &cobra.Command{
		Use:   "build [package...]",
		Short: "build packages",
		Long:  "build all packages by default, unless specific packages are specified",
		Run: func(cmd *cobra.Command, packageNames []string) {
			loader := hammer.NewLoader(viper.GetString("search"))
			loaded, err := loader.Load()
			if err != nil {
				logrus.WithField("error", err).Fatal("could not load packages")
			}

			var packages []*hammer.Package
			if len(packageNames) == 0 {
				packages = loaded
			} else {
				packages = []*hammer.Package{}
				for _, name := range packageNames {
					found := false

					for _, pkg := range loaded {
						if pkg.Name == name {
							packages = append(packages, pkg)
							found = true
							break
						}
					}

					if !found {
						logrus.WithField("name", name).Warn("could not find package")
					}
				}
			}

			if len(packages) == 0 {
				logrus.Fatal("no packages selected")
			}

			fsCache, err := cache.NewFSCache(viper.GetString("cache"))
			if err != nil {
				logrus.WithField("error", err).Fatal("could not make cache")
			}
			for _, pkg := range packages {
				pkg.SetCache(fsCache)
			}

			packager := hammer.NewPackager(packages)

			err = packager.EnsureOutputDir(viper.GetString("output"))
			if err != nil {
				logrus.WithField("error", err).Fatal("could not create output directory")
			}

			err = packager.EnsureOutputDir(viper.GetString("logs"))
			if err != nil {
				logrus.WithError(err).Fatal("could not create logs directory")
			}

			ctx, cancel := context.WithCancel(context.Background())

			// handle interrupts so we can clean up nicely
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)
			go func() {
				for {
					select {
					case <-c:
						logrus.Warn("interrupted, exiting cleanly")
						cancel()

					case <-ctx.Done():
						return
					}
				}
			}()
{'fraction_non_alphanumeric': 0.14416586306653809, 'fraction_numerical': 0.0033751205400192863, 'mean_word_length': 2.820441988950276, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2570299', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1366, 'n_words': 318}