<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 <NAME> <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"
)

var (
	ErrUpstreamFailed    = errors.New("upstream request failed")
	ErrUpstreamBadStatus = errors.New("upstream returned unexpected status")
	ErrInternal          = errors.New("internal error")
)

type ViaDownloadServer struct {
	Mirrors       *Mirrors
	Cache         *Cache
	ClientTimeout time.Duration
	Router        *mux.Router
}

func loggingMiddleware(next http.Handler) http.Handler {
	buf := bytes.Buffer{}
	lh := handlers.LoggingHandler(&buf, next)
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		lh.ServeHTTP(w, r)
		log.Info(strings.TrimSpace(buf.String()))
		buf.Reset()
	})
}

func NewViaDownloadServer(mirrors *Mirrors, cache *Cache, clientTimeout time.Duration) *ViaDownloadServer {
	vs := &ViaDownloadServer{
		Mirrors:       mirrors,
		Cache:         cache,
		ClientTimeout: clientTimeout,
	}
	r := mux.NewRouter()
	r.HandleFunc("/_viadown/count", vs.countHandler).Methods(http.MethodGet)
	r.HandleFunc("/_viadown/stats", vs.statsHandler).Methods(http.MethodGet)
	r.HandleFunc("/_viadown/data", vs.dataDeleteHandler).Methods(http.MethodDelete)
	r.Use(loggingMiddleware)
	vs.Router = r

	return vs
}

func (v *ViaDownloadServer) returnError(w http.ResponseWriter, status int, err error) {
	type apiError struct {
		Error string
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	enc := json.NewEncoder(w)
	enc.Encode(apiError{Error: err.Error()})
}

func (v *ViaDownloadServer) returnOk(w http.ResponseWriter, what interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	enc := json.NewEncoder(w)
	enc.Encode(what)
}

func (v *ViaDownloadServer) statsHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("stats handler")
	v.returnOk(w, v.Cache.Stats())
}

func (v *ViaDownloadServer) countHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("count handler")
	count, err := v.Cache.Count()
	if err != nil {
		v.returnError(w, http.StatusInternalServerError, err)
		return
	}
	v.returnOk(w, count)
}

func (v *ViaDownloadServer) dataDeleteHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("cache purge handler")
	r.ParseForm()
	s := r.FormValue("older-than-days")
	if s == "" {
		v.returnError(w, http.StatusBadRequest, errors.New("older-than-days not provided"))
		return
	}
	olderThanDays, err := strconv.ParseUint(s, 10, 64)
	if err != nil {
		v.returnError(w, http.StatusBadRequest, errors.New("older-than-days is not an integer"))
		return
	}
	removed, err := v.Cache.Purge(PurgeSelector{
		OlderThan: time.Now().Add(-time.Duration(olderThanDays) * 24 * time.Hour),
	})
	if err != nil {
		v.returnError(w, http.StatusInternalServerError, err)
		return
	}
	type removedInfo struct {
		Removed uint64
	}
	v.returnOk(w, removedInfo{Removed: removed})
}

func (v *ViaDownloadServer) localHandler(w http.ResponseWriter, r *http.Request) {
	v.Router.ServeHTTP(w, r)
}

func (v *ViaDownloadServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	upath := r.URL.Path
	log.Debugf("URL path: %v", upath)

	log.Debugf("client headers:")
	for h, v := range r.Header {
		log.Debugf("  %v: %v", h, v)
	}

	switch {
	case strings.HasPrefix(r.URL.Path, "/_viadown"):
		v.localHandler(w, r)
	case r.Method != http.MethodGet:
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "method %q is not supported", r.Method)
	default:
		v.maybeCachedHandler(w, r)
	}

}

func (v *ViaDownloadServer) maybeCachedHandler(w http.ResponseWriter, r *http.Request) {
	upath := r.URL.Path

	if since, err := http.ParseTime(r.Header.Get("If-Modified-Since")); err == nil {
		log.Debugf("has modified since: %v, poke upstream first", since)
	} else {
		// no modified since header, try to get from cache
		found, err := doFromCache(upath, w, r, v.Cache)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
		}
		if found {
			return
		}
	}

	client := http.Client{<commit_msg>via: drop custom routing<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 <NAME> <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"
)

var (
	ErrUpstreamFailed    = errors.New("upstream request failed")
	ErrUpstreamBadStatus = errors.New("upstream returned unexpected status")
	ErrInternal          = errors.New("internal error")
)

type ViaDownloadServer struct {
	Mirrors       *Mirrors
	Cache         *Cache
	ClientTimeout time.Duration
	Router        *mux.Router
}

func loggingMiddleware(next http.Handler) http.Handler {
	buf := bytes.Buffer{}
	lh := handlers.LoggingHandler(&buf, next)
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		lh.ServeHTTP(w, r)
		log.Info(strings.TrimSpace(buf.String()))
		buf.Reset()
	})
}

func NewViaDownloadServer(mirrors *Mirrors, cache *Cache, clientTimeout time.Duration) *ViaDownloadServer {
	vs := &ViaDownloadServer{
		Mirrors:       mirrors,
		Cache:         cache,
		ClientTimeout: clientTimeout,
	}
	r := mux.NewRouter()
	r.HandleFunc("/_viadown/count", vs.countHandler).Methods(http.MethodGet)
	r.HandleFunc("/_viadown/stats", vs.statsHandler).Methods(http.MethodGet)
	r.HandleFunc("/_viadown/data", vs.dataDeleteHandler).Methods(http.MethodDelete)
	r.PathPrefix("/").Methods(http.MethodGet).HandlerFunc(vs.maybeCachedHandler)
	r.Use(loggingMiddleware)
	vs.Router = r

	return vs
}

func (v *ViaDownloadServer) returnError(w http.ResponseWriter, status int, err error) {
	type apiError struct {
		Error string
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	enc := json.NewEncoder(w)
	enc.Encode(apiError{Error: err.Error()})
}

func (v *ViaDownloadServer) returnOk(w http.ResponseWriter, what interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	enc := json.NewEncoder(w)
	enc.Encode(what)
}

func (v *ViaDownloadServer) statsHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("stats handler")
	v.returnOk(w, v.Cache.Stats())
}

func (v *ViaDownloadServer) countHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("count handler")
	count, err := v.Cache.Count()
	if err != nil {
		v.returnError(w, http.StatusInternalServerError, err)
		return
	}
	v.returnOk(w, count)
}

func (v *ViaDownloadServer) dataDeleteHandler(w http.ResponseWriter, r *http.Request) {
	log.Infof("cache purge handler")
	r.ParseForm()
	s := r.FormValue("older-than-days")
	if s == "" {
		v.returnError(w, http.StatusBadRequest, errors.New("older-than-days not provided"))
		return
	}
	olderThanDays, err := strconv.ParseUint(s, 10, 64)
	if err != nil {
		v.returnError(w, http.StatusBadRequest, errors.New("older-than-days is not an integer"))
		return
	}
	removed, err := v.Cache.Purge(PurgeSelector{
		OlderThan: time.Now().Add(-time.Duration(olderThanDays) * 24 * time.Hour),
	})
	if err != nil {
		v.returnError(w, http.StatusInternalServerError, err)
		return
	}
	type removedInfo struct {
		Removed uint64
	}
	v.returnOk(w, removedInfo{Removed: removed})
}

func (v *ViaDownloadServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	v.Router.ServeHTTP(w, r)
}

func (v *ViaDownloadServer) maybeCachedHandler(w http.ResponseWriter, r *http.Request) {
	upath := r.URL.Path

	if since, err := http.ParseTime(r.Header.Get("If-Modified-Since")); err == nil {
		log.Debugf("has modified since: %v, poke upstream first", since)
	} else {
		// no modified since header, try to get from cache
		found, err := doFromCache(upath, w, r, v.Cache)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
		}
		if found {
			return
		}
	}

	client := http.Client{
{'fraction_non_alphanumeric': 0.12979728399921275, 'fraction_numerical': 0.002361739814997048, 'mean_word_length': 4.3098223615465, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1096812', 'n_tokens_mistral': 3493, 'n_tokens_neox': 3351, 'n_words': 933}