<commit_before>package main

import (
	"flag"
	"net/http"
	"log"
	"./src"
)

func main() {
	var (
		maxWorkers   = flag.Int("max_workers", 5, "The number of workers to start")
		maxQueueSize = flag.Int("max_queue_size", 100, "The size of job queue")
		port         = flag.String("port", "8081", "The server port")
		source       = flag.String("source", "https://repo.gin.g-node.org", "The default URI")
		baseTarget   = flag.String("target", "./", "The default base path for storgae")
	)
	flag.Parse()
	ds := ginDoi.GinDataSource{GinURL: *source}
	storage := ginDoi.LocalStorage{Path:*baseTarget, Source:ds}

	// Create the job queue.
	jobQueue := make(chan ginDoi.Job, *maxQueueSize)
	// Start the dispatcher.
	dispatcher := ginDoi.NewDispatcher(jobQueue, *maxWorkers)
	dispatcher.Run(ginDoi.NewWorker)

	// Start the HTTP handler.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ginDoi.InitDoiJob(w, r, &ds)
	})
	http.HandleFunc("/do/", func(w http.ResponseWriter, r *http.Request) {
		ginDoi.DoDoiJob(w,r,jobQueue, storage)
	})<commit_msg>Updated server default config<commit_after>package main

import (
	"flag"
	"net/http"
	"log"
	"./src"
)

func main() {
	var (
		maxWorkers   = flag.Int("max_workers", 5, "The number of workers to start")
		maxQueueSize = flag.Int("max_queue_size", 100, "The size of job queue")
		port         = flag.String("port", "8083", "The server port")
		source       = flag.String("source", "https://repo.gin.g-node.org", "The default URI")
		baseTarget   = flag.String("target", "./", "The default base path for storgae")
	)
	flag.Parse()
	ds := ginDoi.GinDataSource{GinURL: *source}
	storage := ginDoi.LocalStorage{Path:*baseTarget, Source:ds}

	// Create the job queue.
	jobQueue := make(chan ginDoi.Job, *maxQueueSize)
	// Start the dispatcher.
	dispatcher := ginDoi.NewDispatcher(jobQueue, *maxWorkers)
	dispatcher.Run(ginDoi.NewWorker)

	// Start the HTTP handler.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ginDoi.InitDoiJob(w, r, &ds)
	})
	http.HandleFunc("/do/", func(w http.ResponseWriter, r *http.Request) {
		ginDoi.DoDoiJob(w,r,jobQueue, storage)
	})
{'fraction_non_alphanumeric': 0.16309412861137, 'fraction_numerical': 0.007455731593662628, 'mean_word_length': 4.136363636363637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2377841', 'n_tokens_mistral': 822, 'n_tokens_neox': 781, 'n_words': 179}