<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"text/tabwriter"
	"time"

	"github.com/urfave/cli"
	yaml "gopkg.in/yaml.v2"
)

var (
	validfn = [...]string{
		"functions.yaml",
		"functions.yml",
		"function.yaml",
		"function.yml",
		"fn.yaml",
		"fn.yml",
		"functions.json",
		"function.json",
		"fn.json",
	}

	errDockerFileNotFound   = errors.New("no Dockerfile found for this function")
	errUnexpectedFileFormat = errors.New("unexpected file format for function file")
)

type funcfile struct {
	App   *string
	Image string
	Route *string
	Build []string
}

func parsefuncfile(path string) (*funcfile, error) {
	ext := filepath.Ext(path)
	switch ext {
	case ".json":
		return funcfileJSON(path)
	case ".yaml", ".yml":
		return funcfileYAML(path)
	}
	return nil, errUnexpectedFileFormat
}

func funcfileJSON(path string) (*funcfile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("could not open %s for parsing. Error: %v", path, err)
	}
	ff := new(funcfile)
	err = json.NewDecoder(f).Decode(ff)
	return ff, err
}

func funcfileYAML(path string) (*funcfile, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("could not open %s for parsing. Error: %v", path, err)
	}
	ff := new(funcfile)
	err = yaml.Unmarshal(b, ff)
	return ff, err
}

func isvalid(path string, info os.FileInfo) bool {
	if info.IsDir() {
		return false
	}

	basefn := filepath.Base(path)
	for _, fn := range validfn {
		if basefn == fn {
			return true
		}
	}

	return false
}

func walker(path string, info os.FileInfo, err error, w io.Writer, f func(path string) error) {
	fmt.Fprint(w, path, "\t")
	if err := f(path); err != nil {
		fmt.Fprintln(w, err)
	} else {
		fmt.Fprintln(w, "done")
	}
}

type commoncmd struct {
	wd      string
	verbose bool
	force   bool

	verbwriter io.Writer
}

func (c *commoncmd) flags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "d",
			Usage:       "working directory",
			Destination: &c.wd,
			EnvVar:      "WORK_DIR",
			Value:       "./",
		},
		cli.BoolFlag{
			Name:        "v",
			Usage:       "verbose mode",
			Destination: &c.verbose,
		},
		cli.BoolFlag{
			Name:        "f",
			Usage:       "force updating of all functions that are already up-to-date",
			Destination: &c.force,
		},
	}
}

func (c *commoncmd) scan(walker func(path string, info os.FileInfo, err error, w io.Writer) error) {
	c.verbwriter = ioutil.Discard
	if c.verbose {
		c.verbwriter = os.Stderr
	}

	var walked bool

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
	fmt.Fprint(w, "path", "\t", "result", "\n")

	err := filepath.Walk(c.wd, func(path string, info os.FileInfo, err error) error {
		if !isvalid(path, info) {
			return nil
		}

		if !c.force && !isstale(path) {
			return nil
		}

		e := walker(path, info, err, w)
		now := time.Now()
		os.Chtimes(path, now, now)
		walked = true
		return e
	})
	if err != nil {<commit_msg>fnctl recursive flag (#230)<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"text/tabwriter"
	"time"

	"github.com/urfave/cli"
	yaml "gopkg.in/yaml.v2"
)

var (
	validfn = [...]string{
		"functions.yaml",
		"functions.yml",
		"function.yaml",
		"function.yml",
		"fn.yaml",
		"fn.yml",
		"functions.json",
		"function.json",
		"fn.json",
	}

	errDockerFileNotFound   = errors.New("no Dockerfile found for this function")
	errUnexpectedFileFormat = errors.New("unexpected file format for function file")
)

type funcfile struct {
	App   *string
	Image string
	Route *string
	Build []string
}

func parsefuncfile(path string) (*funcfile, error) {
	ext := filepath.Ext(path)
	switch ext {
	case ".json":
		return funcfileJSON(path)
	case ".yaml", ".yml":
		return funcfileYAML(path)
	}
	return nil, errUnexpectedFileFormat
}

func funcfileJSON(path string) (*funcfile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("could not open %s for parsing. Error: %v", path, err)
	}
	ff := new(funcfile)
	err = json.NewDecoder(f).Decode(ff)
	return ff, err
}

func funcfileYAML(path string) (*funcfile, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("could not open %s for parsing. Error: %v", path, err)
	}
	ff := new(funcfile)
	err = yaml.Unmarshal(b, ff)
	return ff, err
}

func isvalid(path string, info os.FileInfo) bool {
	if info.IsDir() {
		return false
	}

	basefn := filepath.Base(path)
	for _, fn := range validfn {
		if basefn == fn {
			return true
		}
	}

	return false
}

func walker(path string, info os.FileInfo, err error, w io.Writer, f func(path string) error) {
	fmt.Fprint(w, path, "\t")
	if err := f(path); err != nil {
		fmt.Fprintln(w, err)
	} else {
		fmt.Fprintln(w, "done")
	}
}

type commoncmd struct {
	wd          string
	verbose     bool
	force       bool
	recursively bool

	verbwriter io.Writer
}

func (c *commoncmd) flags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "d",
			Usage:       "working directory",
			Destination: &c.wd,
			EnvVar:      "WORK_DIR",
			Value:       "./",
		},
		cli.BoolFlag{
			Name:        "v",
			Usage:       "verbose mode",
			Destination: &c.verbose,
		},
		cli.BoolFlag{
			Name:        "f",
			Usage:       "force updating of all functions that are already up-to-date",
			Destination: &c.force,
		},
		cli.BoolFlag{
			Name:        "r",
			Usage:       "recursively scan all functions",
			Destination: &c.recursively,
		},
	}
}

func (c *commoncmd) scan(walker func(path string, info os.FileInfo, err error, w io.Writer) error) {
	c.verbwriter = ioutil.Discard
	if c.verbose {
		c.verbwriter = os.Stderr
	}

	var walked bool

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
	fmt.Fprint(w, "path", "\t", "result", "\n")

	err := filepath.Walk(c.wd, func(path string, info os.FileInfo, err error) error {

		if !c.recursively && path != c.wd && info.IsDir() {
			return filepath.SkipDir
		}

		if !isvalid(path, info) {
			return nil
		}

		if c.recursively && !c.force && !isstale(path) {
			return nil
		}

		e := walker(path, info, err, w)
		now := time.Now()
		os.Chtimes(path, now, now)
		walked = true
		return e
	})
	if err != nil {
{'fraction_non_alphanumeric': 0.15422647527910685, 'fraction_numerical': 0.002073365231259968, 'mean_word_length': 3.1202365308804203, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7171337', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2393, 'n_words': 635}