<commit_before>package main

import (
	"github.com/ak1t0/flame/crawler"
	"github.com/ak1t0/flame/format"
	"github.com/ak1t0/flame/reader"
	"github.com/urfave/cli"
	"log"
	"os"
)

var Version string = "0.0.1"

func main() {
	app := cli.NewApp()
	app.Name = "flame"
	app.Usage = "crawl onion services"
	app.Version = Version
	app.Author = "ak1t0"
	app.Email = "<EMAIL>"
	app.Commands = Commands

	app.Run(os.Args)
}

var Commands = []cli.Command{
	commandScan,
}

var commandScan = cli.Command{
	Name:    "scan",
	Usage:   "Scan onion services",
	Aliases: []string{"s"},
	Action:  doScan,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "f",
			Usage: "Select log file",
		},
		cli.StringFlag{
			Name:  "o",
			Usage: "Select output file",
		},
	},
}

func doScan(c *cli.Context) error {
	var target string
	if c.String("f") != "" {
		target = c.String("f")
	} else {
		target = "log.json"
	}

	parsed, err := reader.ReadJson(target)
	if err != nil {
		log.Fatal(err)
	}

	r := crawler.Crawl(format.NewOnionLogs(parsed))

	if c.String("o") != "" {
		f, err := os.OpenFile(c.String("o"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Println(r)
			log.Fatal(err)
		}
		log.SetOutput(f)
		log.Println(r)
	} else {
		log.Println(r)
	}

	return nil
}<commit_msg>add analyze command<commit_after>package main

import (
	"github.com/ak1t0/flame/crawler"
	"github.com/ak1t0/flame/format"
	"github.com/ak1t0/flame/reader"
	"github.com/urfave/cli"
	"log"
	"os"
	"sort"
)

var Version string = "0.0.1"

func main() {
	app := cli.NewApp()
	app.Name = "flame"
	app.Usage = "crawl onion services"
	app.Version = Version
	app.Author = "ak1t0"
	app.Email = "<EMAIL>"
	app.Commands = Commands

	app.Run(os.Args)
}

var Commands = []cli.Command{
	commandScan,
	commandAnalyze,
}

var commandScan = cli.Command{
	Name:    "scan",
	Usage:   "Scan onion services",
	Aliases: []string{"s"},
	Action:  doScan,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "f",
			Usage: "Select log file",
		},
		cli.StringFlag{
			Name:  "o",
			Usage: "Select output file",
		},
	},
}

func doScan(c *cli.Context) error {
	var target string
	if c.String("f") != "" {
		target = c.String("f")
	} else {
		target = "log.json"
	}

	parsed, err := reader.ReadJson(target)
	if err != nil {
		log.Fatal(err)
	}

	r := crawler.Crawl(format.NewOnionLogs(parsed))

	if c.String("o") != "" {
		f, err := os.OpenFile(c.String("o"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Println(r)
			log.Fatal(err)
		}
		log.SetOutput(f)
		log.Println(r)
	} else {
		log.Println(r)
	}

	return nil
}

var commandAnalyze = cli.Command{
	Name:    "analyze",
	Usage:   "Analyze tor-dns log",
	Aliases: []string{"a"},
	Action:  doAnalyze,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "f",
			Usage: "Select log file",
		},
		cli.StringFlag{
			Name:  "o",
			Usage: "Select output file",
		},
	},
}

func doAnalyze(c *cli.Context) error {
	var target string
	if c.String("f") != "" {
		target = c.String("f")
	} else {
		target = "log.json"
	}

	parsed, err := reader.ReadJson(target)
	if err != nil {
		log.Fatal(err)
	}

	r := format.NewOnionLogs(parsed)
	sort.Sort(r)

	if c.String("o") != "" {
		f, err := os.OpenFile(c.String("o"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Println(r)
			log.Fatal(err)
		}
		log.SetOutput(f)
		log.Println(r)
	} else {
		log.Println(r)
	}

	return nil
}
{'fraction_non_alphanumeric': 0.18259335489561893, 'fraction_numerical': 0.009997059688326962, 'mean_word_length': 3.257822277847309, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1097107', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1482, 'n_words': 287}