<commit_before>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/vharitonsky/iniflags"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
)

var (
	dataDir = flag.String("data", "data", "Path to data files relative to current working dir.")
	port    = flag.Int("port", 8080, "Webserver port number.")
	remote  = flag.String("remote", "http://gravatar.com/avatar", "Use this gravatar-compatible avatar service if "+
		"no avatar is found, use 'none' for no remote.")
	dflt = flag.String("default", "resources/mm", "Default avatar. Use 'remote' to use the default of the remote "+
		"service, or 'remote:<option>' to use a builtin default. For example: 'remote:monsterid'. This is passes as "+
		"'?d=monsterid' to the remote service. See https://nl.gravatar.com/site/implement/images/.")
)

var (
	defaultImage  = "resources/mm"
	remoteUrl     = ""
	remoteDefault = ""
)

type Avatar struct {
	mimetype string
	size     int
	data     []byte
}

func createPath(hash string) string {
	return fmt.Sprintf("%s/%s", *dataDir, hash)
}

func loadImage(hash string, w io.Writer) error {
	filename := createPath(hash)
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		if remoteUrl == "" {
			log.Printf("%s does not exist, falling back to default", filename)
			filename = defaultImage
		} else {
			log.Printf("%s does not exist, redirecting to %s", filename, remoteUrl)
			options := ""
			if remoteDefault != "" {
				options = fmt.Sprintf("?d=%s", remoteDefault)
			}
			remote := fmt.Sprintf("%s/%s%s", remoteUrl, hash, options)
			resp, err2 := http.Get(remote)
			if err2 != nil {
				log.Printf("Remote lookup of %s failed with error: %s", remote, err2)
			} else {
				if resp.StatusCode == 404 {
					log.Printf("Avatar not found on remote, falling back to default")
				} else {
					// TODO check for other status codes?
					log.Printf("Response: %v", resp)
					_, e := io.Copy(w, resp.Body)
					return e
				}
			}
			filename = "resources/mm"
		}
	}

	file, err := os.Open(filename)
	if err != nil {
		log.Printf("Error reading file: %v", err)
		return err
	}

	defer file.Close()

	_, e := io.Copy(w, file)
	return e
}

func avatarHandler(w http.ResponseWriter, r *http.Request, title string) {
	loadImage(title, w)
}

var templates = template.Must(template.ParseFiles("resources/upload.html", "resources/view.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, data interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func uploadHandler(w http.ResponseWriter, r *http.Request, title string) {<commit_msg>Using a struct (Avatar) to represent an avatar in memory. This can later be used for caching<commit_after>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/vharitonsky/iniflags"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"bytes"
)

var (
	dataDir = flag.String("data", "data", "Path to data files relative to current working dir.")
	port    = flag.Int("port", 8080, "Webserver port number.")
	remote  = flag.String("remote", "http://gravatar.com/avatar", "Use this gravatar-compatible avatar service if "+
		"no avatar is found, use 'none' for no remote.")
	dflt = flag.String("default", "resources/mm", "Default avatar. Use 'remote' to use the default of the remote "+
		"service, or 'remote:<option>' to use a builtin default. For example: 'remote:monsterid'. This is passes as "+
		"'?d=monsterid' to the remote service. See https://nl.gravatar.com/site/implement/images/.")
)

var (
	defaultImage  = "resources/mm"
	remoteUrl     = ""
	remoteDefault = ""
)

type Avatar struct {
	size     int
	data     []byte
}

type Request struct {
	hash string
	size int
}

// Retrieves the avatar from the remote service, returning nil if there is no avatar or it could not be retrieved
func retrieveFromRemote(request Request) *Avatar {
	if remoteUrl == "" {
		return nil
	}
	options := fmt.Sprintf("s=%d", request.size) 
	if remoteDefault != "" {
		options += "&d=" + remoteDefault
	}
	remote := remoteUrl + "/" + request.hash + "?" + options
	resp, err2 := http.Get(remote)
	if err2 != nil {
		log.Printf("Remote lookup of %s failed with error: %s", remote, err2)
		return nil
	}
	if resp.StatusCode == 404 {
		log.Printf("Avatar not found on remote")
		return nil
	} else {
		// TODO check for other status codes?
		log.Printf("Response: %v", resp)
		b := new(bytes.Buffer)
		_, e := io.Copy(b, resp.Body)
		if e != nil {
			log.Printf("Could not read image from remote: %s", e)
		}
		return &Avatar{size: -1, data: b.Bytes()}
	}
}

func createPath(hash string) string {
	return fmt.Sprintf("%s/%s", *dataDir, hash)
}

func writeAvatarResult(w http.ResponseWriter, avatar *Avatar) error {
	b := bytes.NewBuffer(avatar.data)
	_, e := io.Copy(w, b)
	// We could write the mimetype in the header, but wget -S suggest that the
	// mimetype is already automagically detected...
	return e
}

func loadImage(hash string, w http.ResponseWriter) error {
	filename := createPath(hash)
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		avatar := retrieveFromRemote(Request{hash: hash, size: -1})
		if avatar == nil {
			filename = defaultImage
		} else {
			e := writeAvatarResult(w, avatar)
			if e != nil {
				return e
			}
			return nil
		}
	}

	file, err := os.Open(filename)
	if err != nil {
		log.Printf("Error reading file: %v", err)
		return err
	}

	defer file.Close()

	_, e := io.Copy(w, file)
	return e
}

func avatarHandler(w http.ResponseWriter, r *http.Request, title string) {
	loadImage(title, w)
}

var templates = template.Must(template.ParseFiles("resources/upload.html", "resources/view.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, data interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func uploadHandler(w http.ResponseWriter, r *http.Request, title string) {
{'fraction_non_alphanumeric': 0.1456002671564535, 'fraction_numerical': 0.004007346802471197, 'mean_word_length': 3.7314375987361768, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2900258', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2035, 'n_words': 608}