<commit_before>package main

import (
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/cryptix/goPshdlRest/api"

	"github.com/visionmedia/go-debug"
)

var (
	apiClient *pshdlApi.Client
	workspace *pshdlApi.Workspace
)

const appName = "pshdlFetchSimCode"

var dbg = debug.Debug(appName)

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Usage = "request simulation code and download it"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "workspace,w", Value: "", Usage: "The workspace to use"},
		cli.StringFlag{Name: "module,m", Value: "", Usage: "The module that should be requested"},
	}
	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) {

	wid := c.String("workspace")
	if wid == "" {
		log.Println("please supply a workspace id")
		os.Exit(1)
	}

	moduleName := c.String("module")
	if moduleName == "" {
		log.Println("please supply a module name")
		os.Exit(1)
	}

	apiClient = pshdlApi.NewClientWithID(nil, wid)
	uris, err := apiClient.Compiler.RequestSimCode(pshdlApi.SimC, moduleName)
	check(err)

	// construct []Record
	recs := make([]pshdlApi.Record, len(uris))
	for i, uri := range uris {
		uriParts := strings.Split(uri, "/")
		relPath := strings.Replace(uriParts[len(uriParts)-1], ":", "/", -1)

		recs[i] = pshdlApi.Record{
			FileURI: uri,
			RelPath: relPath,
		}
	}
<commit_msg>fetchSimCode - bool flag to strip name to base<commit_after>package main

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/cryptix/goPshdlRest/api"
	"github.com/visionmedia/go-debug"
)

var (
	apiClient *pshdlApi.Client
	workspace *pshdlApi.Workspace
)

const appName = "pshdlFetchSimCode"

var dbg = debug.Debug(appName)

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Usage = "request simulation code and download it"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "workspace,w", Value: "", Usage: "The workspace to use"},
		cli.StringFlag{Name: "module,m", Value: "", Usage: "The module that should be requested"},
		cli.BoolFlag{Name: "base,b", Usage: "strip the relPath to it's base"},
	}
	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) {

	wid := c.String("workspace")
	if wid == "" {
		log.Println("please supply a workspace id")
		os.Exit(1)
	}

	moduleName := c.String("module")
	if moduleName == "" {
		log.Println("please supply a module name")
		os.Exit(1)
	}

	apiClient = pshdlApi.NewClientWithID(nil, wid)
	uris, err := apiClient.Compiler.RequestSimCode(pshdlApi.SimC, moduleName)
	check(err)

	// construct []Record
	recs := make([]pshdlApi.Record, len(uris))
	for i, uri := range uris {
		uriParts := strings.Split(uri, "/")
		relPath := strings.Replace(uriParts[len(uriParts)-1], ":", "/", -1)
		if c.Bool("base") {
			relPath = filepath.Base(relPath)
		}

		recs[i] = pshdlApi.Record{
			FileURI: uri,
			RelPath: relPath,
		}
	}

{'fraction_non_alphanumeric': 0.16468946266573622, 'fraction_numerical': 0.0027913468248429866, 'mean_word_length': 4.083333333333333, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6117418', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1104, 'n_words': 237}