<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"github.com/semanticize/st/storage"
	"log"
	"os"
	"regexp"
)

var paraEnd = regexp.MustCompile(`\n\s*\n`)

// Paragraph splitter for bufio.Scanner.
// XXX If we use regexp's RuneReader support, we can do this without buffering.
func splitPara(data []byte, atEOF bool) (advance int, token []byte, err error) {
	loc := paraEnd.FindIndex(data)
	if loc != nil {
		advance = loc[1]
		token = data[:loc[0]]
	} else if atEOF {
		advance = len(data)
		token = data
	}
	return
}

var dohttp = flag.String("http", "",
	"Serve HTTP requests, e.g., -http=localhost:8080")

func main() {
	log.SetPrefix("semanticizest ")

	flag.Parse()

	var err error
	check := func() {
		if err != nil {
			log.Fatal(err)
		}
	}

	db, settings, err := storage.LoadModel(os.Args[1])
	check()
	ngramcount, err := storage.LoadCM(db)
	check()

	sem := semanticizer{db, ngramcount, settings.MaxNGram}

	if *dohttp == "" {
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Split(splitPara)

		out := json.NewEncoder(os.Stdout)

		for scanner.Scan() {
			var candidates []candidate
			candidates, err = sem.allCandidates(scanner.Text())
			check()

			err = out.Encode(candidates)
			check()
		}
		err = scanner.Err()
		check()
	} else {
		log.Fatal(restServer(*dohttp, settings))
	}
}<commit_msg>better argument/option parsing for client (kingpin)<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"github.com/semanticize/st/storage"
	"gopkg.in/alecthomas/kingpin.v1"
	"log"
	"os"
	"regexp"
)

var paraEnd = regexp.MustCompile(`\n\s*\n`)

// Paragraph splitter for bufio.Scanner.
// XXX If we use regexp's RuneReader support, we can do this without buffering.
func splitPara(data []byte, atEOF bool) (advance int, token []byte, err error) {
	loc := paraEnd.FindIndex(data)
	if loc != nil {
		advance = loc[1]
		token = data[:loc[0]]
	} else if atEOF {
		advance = len(data)
		token = data
	}
	return
}

var (
	dbpath = kingpin.Arg("model", "path to model file").Required().String()
	dohttp = kingpin.Flag("http", "HTTP server address").Default("").String()
)

func main() {
	kingpin.Parse()

	log.SetPrefix("semanticizest ")

	var err error
	check := func() {
		if err != nil {
			log.Fatal(err)
		}
	}

	db, settings, err := storage.LoadModel(*dbpath)
	check()
	ngramcount, err := storage.LoadCM(db)
	check()

	sem := semanticizer{db, ngramcount, settings.MaxNGram}

	if *dohttp == "" {
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Split(splitPara)

		out := json.NewEncoder(os.Stdout)

		for scanner.Scan() {
			var candidates []candidate
			candidates, err = sem.allCandidates(scanner.Text())
			check()

			err = out.Encode(candidates)
			check()
		}
		err = scanner.Err()
		check()
	} else {
		log.Fatal(restServer(*dohttp, settings))
	}
}
{'fraction_non_alphanumeric': 0.15780078543377365, 'fraction_numerical': 0.003570153516601214, 'mean_word_length': 3.7734241908006814, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1420184', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1049, 'n_words': 243}