<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: post2mail <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string<commit_msg>Fix obsolete name in usage text<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: fetchpost <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string
{'fraction_non_alphanumeric': 0.14065040650406505, 'fraction_numerical': 0.002845528455284553, 'mean_word_length': 4.063786008230453, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6548044', 'n_tokens_mistral': 942, 'n_tokens_neox': 902, 'n_words': 264}