<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\"download">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {<commit_msg>Add log for simple index<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\"download">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Rendering simple index")
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {
{'fraction_non_alphanumeric': 0.18804034582132564, 'fraction_numerical': 0.002881844380403458, 'mean_word_length': 4.511904761904762, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '452025', 'n_tokens_mistral': 544, 'n_tokens_neox': 524, 'n_words': 107}