<commit_before>package main

//go:generate go-bindata -nometadata -pkg assets -o generated/assets/assets.go -prefix assets/ assets/...
//go:generate swagger generate client --target generated --spec ./swagger.yml
import (
	"compress/gzip"
	"fmt"
	"github.com/bozaro/tech-db-forum/tests"
	"github.com/mkideal/cli"
	"github.com/op/go-logging"
	"io"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"strings"
	"time"
)

const (
	EXIT_INVALID_COMMAND = iota + 1
	EXIT_WAIT_ALIVE_TIMEOUT
	EXIT_FUNC_FAILED
	EXIT_FILL_FAILED
)

type parserUrl struct {
	ptr interface{}
}

type parserRegexp struct {
	ptr interface{}
}

func newParserUrl(ptr interface{}) cli.FlagParser {
	return &parserUrl{ptr}
}

func newParserRegexp(ptr interface{}) cli.FlagParser {
	return &parserRegexp{ptr}
}

func (parser *parserUrl) Parse(s string) error {
	u, err := url.Parse(s)
	if err == nil {
		val := reflect.ValueOf(parser.ptr)
		val.Elem().Set(reflect.ValueOf(*u))
	}
	return err
}

func (parser *parserRegexp) Parse(s string) error {
	u, err := regexp.Compile(s)
	if err == nil {
		val := reflect.ValueOf(parser.ptr)
		val.Elem().Set(reflect.ValueOf(*u))
	}
	return err
}

type CmdCommonT struct {
	cli.Helper
	Url             *url.URL `cli:"u,url" usage:"base url for testing API" parser:"url" dft:"http://localhost:5000/api"`
	WaitAlive       int      `cli:"wait" usage:"wait before remote API make alive (while connection refused or 5XX error on base url)" dft:"30"`
	DontCheckUpdate bool     `cli:"no-check" usage:"do not check version update"`
}

var root = &cli.Command{
	Desc: "https://github.com/bozaro/tech-db-forum",
	Argv: func() interface{} { return nil },
	Fn: func(ctx *cli.Context) error {
		ctx.WriteUsage()
		os.Exit(EXIT_INVALID_COMMAND)
		return nil
	},
}

type CmdFuncT struct {
	CmdCommonT
	Keep   bool           `cli:"k,keep" usage:"Don't stop after first failed test'"`
	Test   *regexp.Regexp `cli:"t,tests" usage:"Mask for running test names (regexp)" parser:"regexp" dft:".*"`
	Report string         `cli:"r,report" usage:"Detailed report file" dft:"report.html"`
}

var cmdFunc = &cli.Command{
	Name: "func",
	Desc: "run functional testing",
	Argv: func() interface{} { return new(CmdFuncT) },
	Fn: func(ctx *cli.Context) error {
		argv := ctx.Argv().(*CmdFuncT)
		commonPrepare(argv.CmdCommonT)
		if tests.Run(argv.Url, argv.Test, argv.Report, argv.Keep) > 0 {
			os.Exit(EXIT_FUNC_FAILED)
		}
		return nil
	},
}

type CmdFillT struct {
	CmdCommonT
	Threads   int    `cli:"t,thread" usage:"number of threads for generating data" dft:"8"`
	StateFile string `cli:"o,state" usage:"State file with information about database objects" dft:"tech-db-forum.dat.gz"`
}

var cmdFill = &cli.Command{
	Name: "fill",
	Desc: "fill database with random data",
	Argv: func() interface{} { return new(CmdFillT) },
	Fn: func(ctx *cli.Context) error {
		argv := ctx.Argv().(*CmdFillT)
		commonPrepare(argv.CmdCommonT)
		perf := tests.NewPerf(argv.Url, tests.NewPerfConfig())
		perf.Fill(argv.Threads, tests.NewPerfConfig())
		if perf == nil {
			os.Exit(EXIT_FILL_FAILED)
		}
		file, err := os.Create(argv.StateFile)
		defer file.Close()
		var writer io.Writer = file

		var zw *gzip.Writer
		if strings.HasSuffix(argv.StateFile, ".gz") {
			zw, _ = gzip.NewWriterLevel(writer, gzip.BestCompression)
			writer = zw
		}
		if err != nil {
			log.Error("Can't create file: " + argv.StateFile)
			os.Exit(EXIT_FILL_FAILED)
		}
		err = perf.Save(writer)
		if err != nil {
			log.Error("Can't save to file: " + argv.StateFile)
			os.Exit(EXIT_FILL_FAILED)
		}
		if zw != nil {
			zw.Flush()
			zw.Close()
		}
		return nil
	},
}

type CmdPerfT struct {
	CmdCommonT
	Threads   int    `cli:"t,thread" usage:"number of threads for performance testing" dft:"8"`
	StateFile string `cli:"i,state" usage:"State file with information about database objects" dft:"tech-db-forum.dat.gz"`
}<commit_msg>Уменьшил степень сжатия файла с результатом заполнения<commit_after>package main

//go:generate go-bindata -nometadata -pkg assets -o generated/assets/assets.go -prefix assets/ assets/...
//go:generate swagger generate client --target generated --spec ./swagger.yml
import (
	"compress/gzip"
	"fmt"
	"github.com/bozaro/tech-db-forum/tests"
	"github.com/mkideal/cli"
	"github.com/op/go-logging"
	"io"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"strings"
	"time"
)

const (
	EXIT_INVALID_COMMAND = iota + 1
	EXIT_WAIT_ALIVE_TIMEOUT
	EXIT_FUNC_FAILED
	EXIT_FILL_FAILED
)

type parserUrl struct {
	ptr interface{}
}

type parserRegexp struct {
	ptr interface{}
}

func newParserUrl(ptr interface{}) cli.FlagParser {
	return &parserUrl{ptr}
}

func newParserRegexp(ptr interface{}) cli.FlagParser {
	return &parserRegexp{ptr}
}

func (parser *parserUrl) Parse(s string) error {
	u, err := url.Parse(s)
	if err == nil {
		val := reflect.ValueOf(parser.ptr)
		val.Elem().Set(reflect.ValueOf(*u))
	}
	return err
}

func (parser *parserRegexp) Parse(s string) error {
	u, err := regexp.Compile(s)
	if err == nil {
		val := reflect.ValueOf(parser.ptr)
		val.Elem().Set(reflect.ValueOf(*u))
	}
	return err
}

type CmdCommonT struct {
	cli.Helper
	Url             *url.URL `cli:"u,url" usage:"base url for testing API" parser:"url" dft:"http://localhost:5000/api"`
	WaitAlive       int      `cli:"wait" usage:"wait before remote API make alive (while connection refused or 5XX error on base url)" dft:"30"`
	DontCheckUpdate bool     `cli:"no-check" usage:"do not check version update"`
}

var root = &cli.Command{
	Desc: "https://github.com/bozaro/tech-db-forum",
	Argv: func() interface{} { return nil },
	Fn: func(ctx *cli.Context) error {
		ctx.WriteUsage()
		os.Exit(EXIT_INVALID_COMMAND)
		return nil
	},
}

type CmdFuncT struct {
	CmdCommonT
	Keep   bool           `cli:"k,keep" usage:"Don't stop after first failed test'"`
	Test   *regexp.Regexp `cli:"t,tests" usage:"Mask for running test names (regexp)" parser:"regexp" dft:".*"`
	Report string         `cli:"r,report" usage:"Detailed report file" dft:"report.html"`
}

var cmdFunc = &cli.Command{
	Name: "func",
	Desc: "run functional testing",
	Argv: func() interface{} { return new(CmdFuncT) },
	Fn: func(ctx *cli.Context) error {
		argv := ctx.Argv().(*CmdFuncT)
		commonPrepare(argv.CmdCommonT)
		if tests.Run(argv.Url, argv.Test, argv.Report, argv.Keep) > 0 {
			os.Exit(EXIT_FUNC_FAILED)
		}
		return nil
	},
}

type CmdFillT struct {
	CmdCommonT
	Threads   int    `cli:"t,thread" usage:"number of threads for generating data" dft:"8"`
	StateFile string `cli:"o,state" usage:"State file with information about database objects" dft:"tech-db-forum.dat.gz"`
}

var cmdFill = &cli.Command{
	Name: "fill",
	Desc: "fill database with random data",
	Argv: func() interface{} { return new(CmdFillT) },
	Fn: func(ctx *cli.Context) error {
		argv := ctx.Argv().(*CmdFillT)
		commonPrepare(argv.CmdCommonT)
		perf := tests.NewPerf(argv.Url, tests.NewPerfConfig())
		perf.Fill(argv.Threads, tests.NewPerfConfig())
		if perf == nil {
			os.Exit(EXIT_FILL_FAILED)
		}
		file, err := os.Create(argv.StateFile)
		defer file.Close()
		var writer io.Writer = file

		var zw *gzip.Writer
		if strings.HasSuffix(argv.StateFile, ".gz") {
			zw = gzip.NewWriter(writer)
			writer = zw
		}
		if err != nil {
			log.Error("Can't create file: " + argv.StateFile)
			os.Exit(EXIT_FILL_FAILED)
		}
		err = perf.Save(writer)
		if err != nil {
			log.Error("Can't save to file: " + argv.StateFile)
			os.Exit(EXIT_FILL_FAILED)
		}
		if zw != nil {
			zw.Flush()
			zw.Close()
		}
		return nil
	},
}

type CmdPerfT struct {
	CmdCommonT
	Threads   int    `cli:"t,thread" usage:"number of threads for performance testing" dft:"8"`
	StateFile string `cli:"i,state" usage:"State file with information about database objects" dft:"tech-db-forum.dat.gz"`
}
{'fraction_non_alphanumeric': 0.1463571889103804, 'fraction_numerical': 0.0028368794326241137, 'mean_word_length': 3.971794871794872, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1381296', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2861, 'n_words': 695}