<commit_before>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strconv"

	"github.com/takama/router"
)

// data is a shortcut
type data map[string]interface{}

func isAlphaNumeric(str string) bool {
	isAlphaNum := true
	for _, b := range str {
		if !('0' <= b && b <= '9' || 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || b == '_' || b == '-' || b == '.') {
			isAlphaNum = false
			break
		}
	}

	return isAlphaNum
}

func decodeRecord(record interface{}, c *router.Control) (*[]byte, bool) {
	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "The body content is absent",
				"info":    err.Error(),
			})
		return &body, false
	}
	if err := json.Unmarshal(body, &record); err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "Could not recognize parameters",
				"info":    err.Error(),
			})
		return &body, false
	}

	return &body, true
}

func decodeRecords(body *[]byte, records interface{}, c *router.Control) bool {
	if err := json.Unmarshal(*body, &records); err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "Could not recognize parameters",
				"info":    err.Error(),
			})
		return false
	}

	return true
}

func decodeNumber(name string, c *router.Control) (uint64, bool) {
	number, err := strconv.ParseUint(c.Get(name), 10, 64)
	if err != nil {
		notRecognizedParameterError(name, err, c)
		return 0, false
	}
	if number == 0 {
		couldNotBeZero(name, c)
		return 0, false
	}

	return number, true
}

func decodeString(name string, c *router.Control) (string, bool) {
	str := c.Get(name)

	if !checkAlphaNumeric(str, c) {
		return "", false
	}
	if str == "" {
		couldNotBeEmpty(str, c)
		return "", false
	}

	return str, true
}

func checkAlphaNumeric(str string, c *router.Control) bool {
	if !isAlphaNumeric(str) {
		err := errors.New(str + " parameter is not alpha-numeric")
		notRecognizedParameterError(str, err, c)
		return false
	}

	return true
}

func couldNotBeZero(param string, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "The parameter '" + param + "' could not be zero value",
			"info":    "Please apply a non-zero value to the data",
		})
}

func couldNotBeEmpty(param string, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "The parameter '" + param + "' could not be empty",
			"info":    "Please apply a non-empty value to the data",
		})
}

func notRecognizedParameterError(param string, err error, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "Could not recognize " + param + " parameter",
			"info":    err.Error(),
		})
}

func recordNotFound(c *router.Control) {
	c.Code(http.StatusNotFound).Body(
		data{
			"success": false,
			"error":   http.StatusNotFound,
			"message": "Record(s) not found",
			"info":    "Please add a record(s) before using",
		})
}

func notFound(c *router.Control) {
	c.Code(http.StatusNotFound).Body(
		data{
			"Message": data{
				"Error":       "Method not found",
				"Information": "Please see list of the methods by using /list",
			},
		})
}

func infoHandler(c *router.Control) {
	host, _ := os.Hostname()
	m := new(runtime.MemStats)
	runtime.ReadMemStats(m)
	c.Code(http.StatusOK).Body(
		data{
			"Spawn Sync Service": data{
				"Host": host,
				"Runtime": data{
					"Compiler": runtime.Version(),
					"CPU":      runtime.NumCPU(),
					"Memory":   fmt.Sprintf("%.2fMB", float64(m.Alloc)/(1<<(10*2))),
					"Threads":  runtime.NumGoroutine(),
				},
				"Release": data{
					"Number": VERSION,
					"Date":   DATE,
				},
			},
		})
}

func logger(c *router.Control) {
	stdlog.Println(c.Request.RemoteAddr, c.Request.Method, c.Request.URL.Path)
}

func baseHandler(handle router.Handle) router.Handle {
	return func(c *router.Control) {
		if c.Get("pretty") != "true" {
			c.CompactJSON(true)
		}
		if origin := c.Request.Header.Get("Origin"); origin != "" {
			c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
			c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
		}
		handle(c)
	}
}

func panicHandler(c *router.Control) {
	c.Code(http.StatusInternalServerError).Body(c.Request)
}<commit_msg>Extend logger for use X-Forwarded-For<commit_after>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strconv"

	"github.com/takama/router"
)

// data is a shortcut
type data map[string]interface{}

func isAlphaNumeric(str string) bool {
	isAlphaNum := true
	for _, b := range str {
		if !('0' <= b && b <= '9' || 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || b == '_' || b == '-' || b == '.') {
			isAlphaNum = false
			break
		}
	}

	return isAlphaNum
}

func decodeRecord(record interface{}, c *router.Control) (*[]byte, bool) {
	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "The body content is absent",
				"info":    err.Error(),
			})
		return &body, false
	}
	if err := json.Unmarshal(body, &record); err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "Could not recognize parameters",
				"info":    err.Error(),
			})
		return &body, false
	}

	return &body, true
}

func decodeRecords(body *[]byte, records interface{}, c *router.Control) bool {
	if err := json.Unmarshal(*body, &records); err != nil {
		c.Code(http.StatusBadRequest).Body(
			data{
				"success": false,
				"error":   http.StatusBadRequest,
				"message": "Could not recognize parameters",
				"info":    err.Error(),
			})
		return false
	}

	return true
}

func decodeNumber(name string, c *router.Control) (uint64, bool) {
	number, err := strconv.ParseUint(c.Get(name), 10, 64)
	if err != nil {
		notRecognizedParameterError(name, err, c)
		return 0, false
	}
	if number == 0 {
		couldNotBeZero(name, c)
		return 0, false
	}

	return number, true
}

func decodeString(name string, c *router.Control) (string, bool) {
	str := c.Get(name)

	if !checkAlphaNumeric(str, c) {
		return "", false
	}
	if str == "" {
		couldNotBeEmpty(str, c)
		return "", false
	}

	return str, true
}

func checkAlphaNumeric(str string, c *router.Control) bool {
	if !isAlphaNumeric(str) {
		err := errors.New(str + " parameter is not alpha-numeric")
		notRecognizedParameterError(str, err, c)
		return false
	}

	return true
}

func couldNotBeZero(param string, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "The parameter '" + param + "' could not be zero value",
			"info":    "Please apply a non-zero value to the data",
		})
}

func couldNotBeEmpty(param string, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "The parameter '" + param + "' could not be empty",
			"info":    "Please apply a non-empty value to the data",
		})
}

func notRecognizedParameterError(param string, err error, c *router.Control) {
	c.Code(http.StatusBadRequest).Body(
		data{
			"success": false,
			"error":   http.StatusBadRequest,
			"message": "Could not recognize " + param + " parameter",
			"info":    err.Error(),
		})
}

func recordNotFound(c *router.Control) {
	c.Code(http.StatusNotFound).Body(
		data{
			"success": false,
			"error":   http.StatusNotFound,
			"message": "Record(s) not found",
			"info":    "Please add a record(s) before using",
		})
}

func notFound(c *router.Control) {
	c.Code(http.StatusNotFound).Body(
		data{
			"Message": data{
				"Error":       "Method not found",
				"Information": "Please see list of the methods by using /list",
			},
		})
}

func infoHandler(c *router.Control) {
	host, _ := os.Hostname()
	m := new(runtime.MemStats)
	runtime.ReadMemStats(m)
	c.Code(http.StatusOK).Body(
		data{
			"Spawn Sync Service": data{
				"Host": host,
				"Runtime": data{
					"Compiler": runtime.Version(),
					"CPU":      runtime.NumCPU(),
					"Memory":   fmt.Sprintf("%.2fMB", float64(m.Alloc)/(1<<(10*2))),
					"Threads":  runtime.NumGoroutine(),
				},
				"Release": data{
					"Number": VERSION,
					"Date":   DATE,
				},
			},
		})
}

func logger(c *router.Control) {
	remoteAddr := c.Request.Header.Get("X-Forwarded-For")
	if remoteAddr == "" {
		remoteAddr = c.Request.RemoteAddr
	}
	stdlog.Println(remoteAddr, c.Request.Method, c.Request.URL.Path)
}

func baseHandler(handle router.Handle) router.Handle {
	return func(c *router.Control) {
		if c.Get("pretty") != "true" {
			c.CompactJSON(true)
		}
		if origin := c.Request.Header.Get("Origin"); origin != "" {
			c.Writer.Header().Set("Access-Control-Allow-Origin", origin)
			c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
		}
		handle(c)
	}
}

func panicHandler(c *router.Control) {
	c.Code(http.StatusInternalServerError).Body(c.Request)
}
{'fraction_non_alphanumeric': 0.1659271489010422, 'fraction_numerical': 0.004540295119182747, 'mean_word_length': 3.4560919540229884, 'pattern_counts': {'":': 82, '<': 21, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6973250', 'n_tokens_mistral': 3814, 'n_tokens_neox': 3403, 'n_words': 848}