<commit_before>// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.

// ---
// @Author Lion <<EMAIL>>
// @Date   2022/06/16

package main

import (
	"bufio"
	"fmt"
	"github.com/lionsoul2014/ip2region/binding/golang/xdb"
	"github.com/mitchellh/go-homedir"
	"log"
	"os"
	"strings"
	"time"
)

func printHelp() {
	fmt.Printf("ip2region xdb searcher\n")
	fmt.Printf("%s [command] [command options]\n", os.Args[0])
	fmt.Printf("Command: \n")
	fmt.Printf("  search    search input test\n")
	fmt.Printf("  bench     search bench test\n")
}

func testSearch() {
	var err error
	var dbFile, cachePolicy = "", "vectorIndex"
	for i := 2; i < len(os.Args); i++ {
		r := os.Args[i]
		if len(r) < 5 {
			continue
		}

		if strings.Index(r, "--") != 0 {
			continue
		}

		var eIdx = strings.Index(r, "=")
		if eIdx < 0 {
			fmt.Printf("missing = for args pair '%s'\n", r)
			return
		}

		switch r[2:eIdx] {
		case "db":
			dbFile = r[eIdx+1:]
		case "cache-policy":
			cachePolicy = r[eIdx+1:]
		}
	}

	if dbFile == "" {
		fmt.Printf("%s search [command options]\n", os.Args[0])
		fmt.Printf("options:\n")
		fmt.Printf(" --db string              ip2region binary xdb file path\n")
		fmt.Printf(" --cache-policy string    cache policy: file/vectorIndex/content\n")
		return
	}

	dbPath, err := homedir.Expand(dbFile)
	if err != nil {
		fmt.Printf("invalid xdb file path `%s`: %s", dbFile, err)
		return
	}

	// create the searcher with the cache policy setting
	searcher, err := createSearcher(dbPath, cachePolicy)
	if err != nil {
		fmt.Printf("failed to create searcher: %s\n", err.Error())
		return
	}
	defer func() {
		searcher.Close()
		fmt.Printf("searcher test program exited, thanks for trying\n")
	}()

	fmt.Printf(`ip2region xdb searcher test program, cachePolicy: %s
type 'quit' to exit
`, cachePolicy)
	reader := bufio.NewReader(os.Stdin)
	for {
		fmt.Print("ip2region>> ")
		str, err := reader.ReadString('\n')
		if err != nil {
			log.Fatalf("failed to read string: %s", err)
		}

		line := strings.TrimSpace(strings.TrimSuffix(str, "\n"))
		if len(line) == 0 {
			continue
		}

		if line == "quit" {
			break
		}

		tStart := time.Now()
		region, err := searcher.SearchByStr(line)
		if err != nil {
			fmt.Printf("\x1b[0;31m{err: %s, ioCount: %d}\x1b[0m\n", err.Error(), searcher.GetIOCount())
		} else {
			fmt.Printf("\x1b[0;32m{region: %s, ioCount: %d, took: %s}\x1b[0m\n", region, searcher.GetIOCount(), time.Since(tStart))
		}
	}
}

func testBench() {
	var err error
	var dbFile, srcFile, cachePolicy = "", "", "vectorIndex"
	for i := 2; i < len(os.Args); i++ {
		r := os.Args[i]
		if len(r) < 5 {
			continue
		}

		if strings.Index(r, "--") != 0 {
			continue
		}

		var eIdx = strings.Index(r, "=")
		if eIdx < 0 {
			fmt.Printf("missing = for args pair '%s'\n", r)
			return
		}

		switch r[2:eIdx] {
		case "db":
			dbFile = r[eIdx+1:]
		case "src":
			srcFile = r[eIdx+1:]
		case "cache-policy":
			cachePolicy = r[eIdx+1:]
		}
	}

	if dbFile == "" || srcFile == "" {
		fmt.Printf("%s bench [command options]\n", os.Args[0])
		fmt.Printf("options:\n")
		fmt.Printf(" --db string              ip2region binary xdb file path\n")
		fmt.Printf(" --src string             source ip text file path\n")
		fmt.Printf(" --cache-policy string    cache policy: file/vectorIndex/content\n")
		return
	}
<commit_msg>break for the undefine option<commit_after>// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.

// ---
// @Author Lion <<EMAIL>>
// @Date   2022/06/16

package main

import (
	"bufio"
	"fmt"
	"github.com/lionsoul2014/ip2region/binding/golang/xdb"
	"github.com/mitchellh/go-homedir"
	"log"
	"os"
	"strings"
	"time"
)

func printHelp() {
	fmt.Printf("ip2region xdb searcher\n")
	fmt.Printf("%s [command] [command options]\n", os.Args[0])
	fmt.Printf("Command: \n")
	fmt.Printf("  search    search input test\n")
	fmt.Printf("  bench     search bench test\n")
}

func testSearch() {
	var err error
	var dbFile, cachePolicy = "", "vectorIndex"
	for i := 2; i < len(os.Args); i++ {
		r := os.Args[i]
		if len(r) < 5 {
			continue
		}

		if strings.Index(r, "--") != 0 {
			continue
		}

		var sIdx = strings.Index(r, "=")
		if sIdx < 0 {
			fmt.Printf("missing = for args pair '%s'\n", r)
			return
		}

		switch r[2:sIdx] {
		case "db":
			dbFile = r[sIdx+1:]
		case "cache-policy":
			cachePolicy = r[sIdx+1:]
		default:
			fmt.Printf("undefined option `%s`\n", r)
			return
		}
	}

	if dbFile == "" {
		fmt.Printf("%s search [command options]\n", os.Args[0])
		fmt.Printf("options:\n")
		fmt.Printf(" --db string              ip2region binary xdb file path\n")
		fmt.Printf(" --cache-policy string    cache policy: file/vectorIndex/content\n")
		return
	}

	dbPath, err := homedir.Expand(dbFile)
	if err != nil {
		fmt.Printf("invalid xdb file path `%s`: %s", dbFile, err)
		return
	}

	// create the searcher with the cache policy setting
	searcher, err := createSearcher(dbPath, cachePolicy)
	if err != nil {
		fmt.Printf("failed to create searcher: %s\n", err.Error())
		return
	}
	defer func() {
		searcher.Close()
		fmt.Printf("searcher test program exited, thanks for trying\n")
	}()

	fmt.Printf(`ip2region xdb searcher test program, cachePolicy: %s
type 'quit' to exit
`, cachePolicy)
	reader := bufio.NewReader(os.Stdin)
	for {
		fmt.Print("ip2region>> ")
		str, err := reader.ReadString('\n')
		if err != nil {
			log.Fatalf("failed to read string: %s", err)
		}

		line := strings.TrimSpace(strings.TrimSuffix(str, "\n"))
		if len(line) == 0 {
			continue
		}

		if line == "quit" {
			break
		}

		tStart := time.Now()
		region, err := searcher.SearchByStr(line)
		if err != nil {
			fmt.Printf("\x1b[0;31m{err: %s, ioCount: %d}\x1b[0m\n", err.Error(), searcher.GetIOCount())
		} else {
			fmt.Printf("\x1b[0;32m{region: %s, ioCount: %d, took: %s}\x1b[0m\n", region, searcher.GetIOCount(), time.Since(tStart))
		}
	}
}

func testBench() {
	var err error
	var dbFile, srcFile, cachePolicy = "", "", "vectorIndex"
	for i := 2; i < len(os.Args); i++ {
		r := os.Args[i]
		if len(r) < 5 {
			continue
		}

		if strings.Index(r, "--") != 0 {
			continue
		}

		var sIdx = strings.Index(r, "=")
		if sIdx < 0 {
			fmt.Printf("missing = for args pair '%s'\n", r)
			return
		}

		switch r[2:sIdx] {
		case "db":
			dbFile = r[sIdx+1:]
		case "src":
			srcFile = r[sIdx+1:]
		case "cache-policy":
			cachePolicy = r[sIdx+1:]
		default:
			fmt.Printf("undefined option `%s`\n", r)
			return
		}
	}

	if dbFile == "" || srcFile == "" {
		fmt.Printf("%s bench [command options]\n", os.Args[0])
		fmt.Printf("options:\n")
		fmt.Printf(" --db string              ip2region binary xdb file path\n")
		fmt.Printf(" --src string             source ip text file path\n")
		fmt.Printf(" --cache-policy string    cache policy: file/vectorIndex/content\n")
		return
	}

{'fraction_non_alphanumeric': 0.17541357672561322, 'fraction_numerical': 0.015972618368511125, 'mean_word_length': 3.037420840529649, 'pattern_counts': {'":': 10, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4785705', 'n_tokens_mistral': 2953, 'n_tokens_neox': 2735, 'n_words': 680}