<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"text/template"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

type UserAgentInfo struct {
	AgentType         string `json:"agent_type"`
	AgentName         string `json:"agent_name"`
	AgentVersion      string `json:"agent_version"`
	OSType            string `json:"os_type"`
	OSName            string `json:"os_name"`
	OSVersionName     string `json:"os_versionName"`
	OSVersionNumber   string `json:"os_versionNumber"`
	LinuxDistribution string `json:"linux_distribution"`
}

type HttpRequest struct {
	Request
	Url           string        `json:"url"`
	FullUrl       string        `json:"full_url"`
	Method        string        `json:"method"`
	Headers       http.Header   `json:"headers"`
	Body          string        `json:"body"`
	Host          string        `json:"host"`
	PostForm      url.Values    `json:"post_form"`
	Form          url.Values    `json:"form"`
	JSON          interface{}   `json:"json"`
	BinId         string        `json:"bin_id"`
	UserAgentInfo UserAgentInfo `json:"user_agent_info"`
}

func (request *HttpRequest) ISO8601Time() string {
	return request.Time.Format(time.RFC3339)
}

func UrlEncoded(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	return u.String(), nil
}

func GetUserAgentInfo(r *http.Request) (UserAgentInfo, error) {
	var userAgentInfo UserAgentInfo
	userAgent := r.Header["User-Agent"]
	if userAgent != nil {
		userAgent, err := UrlEncoded(userAgent[0])
		if err != nil {
			return userAgentInfo, err
		}

		lookupUrl := fmt.Sprintf("http://www.useragentstring.com/?uas=%s&getJSON=all", userAgent)
		res, err := http.Get(lookupUrl)
		if err != nil {
			return userAgentInfo, err
		}

		defer res.Body.Close()

		decoder := json.NewDecoder(res.Body)
		err = decoder.Decode(&userAgentInfo)
		if err != nil {
			return userAgentInfo, err
		}
	}
	return userAgentInfo, nil
}

func startLoggingHttpServer(port int, staticRoot string, writers []HttpRequestWriter) {
	fmt.Printf("Starting HTTP logging server on port %d\n", port)

	handlers := []HttpRequestHandler{
		RedirectHttpRequestHandler{},
		StaticFileHttpRequestHandler{},
		DynamicFileHttpRequestHandler{},
	}
<commit_msg>Reorder static/dynamic to fallback on static files<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"text/template"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

type UserAgentInfo struct {
	AgentType         string `json:"agent_type"`
	AgentName         string `json:"agent_name"`
	AgentVersion      string `json:"agent_version"`
	OSType            string `json:"os_type"`
	OSName            string `json:"os_name"`
	OSVersionName     string `json:"os_versionName"`
	OSVersionNumber   string `json:"os_versionNumber"`
	LinuxDistribution string `json:"linux_distribution"`
}

type HttpRequest struct {
	Request
	Url           string        `json:"url"`
	FullUrl       string        `json:"full_url"`
	Method        string        `json:"method"`
	Headers       http.Header   `json:"headers"`
	Body          string        `json:"body"`
	Host          string        `json:"host"`
	PostForm      url.Values    `json:"post_form"`
	Form          url.Values    `json:"form"`
	JSON          interface{}   `json:"json"`
	BinId         string        `json:"bin_id"`
	UserAgentInfo UserAgentInfo `json:"user_agent_info"`
}

func (request *HttpRequest) ISO8601Time() string {
	return request.Time.Format(time.RFC3339)
}

func UrlEncoded(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	return u.String(), nil
}

func GetUserAgentInfo(r *http.Request) (UserAgentInfo, error) {
	var userAgentInfo UserAgentInfo
	userAgent := r.Header["User-Agent"]
	if userAgent != nil {
		userAgent, err := UrlEncoded(userAgent[0])
		if err != nil {
			return userAgentInfo, err
		}

		lookupUrl := fmt.Sprintf("http://www.useragentstring.com/?uas=%s&getJSON=all", userAgent)
		res, err := http.Get(lookupUrl)
		if err != nil {
			return userAgentInfo, err
		}

		defer res.Body.Close()

		decoder := json.NewDecoder(res.Body)
		err = decoder.Decode(&userAgentInfo)
		if err != nil {
			return userAgentInfo, err
		}
	}
	return userAgentInfo, nil
}

func startLoggingHttpServer(port int, staticRoot string, writers []HttpRequestWriter) {
	fmt.Printf("Starting HTTP logging server on port %d\n", port)

	handlers := []HttpRequestHandler{
		RedirectHttpRequestHandler{},
		DynamicFileHttpRequestHandler{},
		StaticFileHttpRequestHandler{},
	}

{'fraction_non_alphanumeric': 0.12705530642750373, 'fraction_numerical': 0.003843689942344651, 'mean_word_length': 3.4272211720226844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4562262', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1544, 'n_words': 350}