<commit_before>package main

import (
	"bufio"
	"flag"
	"os"
	"regexp"
	"sync"
	"time"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

const (
	WhoisRequestPerSecond = 128
	WhoisRequestPeriod    = time.Second / time.Duration(WhoisRequestPerSecond)
)

func main() {
	var file_path = flag.String("filepath", "/var/log/nginx/access.log", "Log files to analyze, wildcard allowed between quotes.")
	var period = flag.String("period", "24h", "period of time (past to now) to analyze")
	flag.Parse()

	files , err := filepath.Glob(*file_path)
	if err != nil {
		log.Fatal(err)
	}

	//        log_format main '[$time_local] $http_host $remote_addr $http_x_forwarded_for '
	//                        '"$request" $status $body_bytes_sent "$http_referer" '
	//                        '"$http_user_agent" $request_time $upstream_response_time';
	logline, err := regexp.Compile("^\\[([^\\]]+)\\] ([^ ]+) ([^ ]+) ([^ ]+) \"([^\"]*)\" ([^ ]+) ([^ ]+) \"([^\"]*)\" \"([^\"]*)\" ([^ ]+) ([^ ]+)")
	if err != nil {
		log.Fatal(err)
	}
	// GET /rancher-catalog.git/info/refs?service=git-upload-pack HTTP/1.1
	uploadpack, err := regexp.Compile("^GET /(rancher)-catalog(.git)?/info/refs\\?service=git-upload-pack")
	if err != nil {
		log.Fatal(err)
	}

	stats := NewStats()
	stats.StartLogging()

	var cutoffTime time.Time
	if *period != "" {<commit_msg>adjust default path<commit_after>package main

import (
	"bufio"
	"flag"
	"os"
	"regexp"
	"sync"
	"time"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

const (
	WhoisRequestPerSecond = 128
	WhoisRequestPeriod    = time.Second / time.Duration(WhoisRequestPerSecond)
)

func main() {
	var file_path = flag.String("filepath", "/var/log/nginx/access-*.log.1", "Log files to analyze, wildcard allowed between quotes.")
	var period = flag.String("period", "48h", "period of time (past to now) to analyze")
	flag.Parse()

	files , err := filepath.Glob(*file_path)
	if err != nil {
		log.Fatal(err)
	}

	//        log_format main '[$time_local] $http_host $remote_addr $http_x_forwarded_for '
	//                        '"$request" $status $body_bytes_sent "$http_referer" '
	//                        '"$http_user_agent" $request_time $upstream_response_time';
	logline, err := regexp.Compile("^\\[([^\\]]+)\\] ([^ ]+) ([^ ]+) ([^ ]+) \"([^\"]*)\" ([^ ]+) ([^ ]+) \"([^\"]*)\" \"([^\"]*)\" ([^ ]+) ([^ ]+)")
	if err != nil {
		log.Fatal(err)
	}
	// GET /rancher-catalog.git/info/refs?service=git-upload-pack HTTP/1.1
	uploadpack, err := regexp.Compile("^GET /(rancher)-catalog(.git)?/info/refs\\?service=git-upload-pack")
	if err != nil {
		log.Fatal(err)
	}

	stats := NewStats()
	stats.StartLogging()

	var cutoffTime time.Time
	if *period != "" {
{'fraction_non_alphanumeric': 0.2144981412639405, 'fraction_numerical': 0.0055762081784386614, 'mean_word_length': 3.375609756097561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4099090', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1011, 'n_words': 184}