<commit_before>package server

import (
	"fmt"
	"github.com/siddontang/go-log/log"
	"github.com/siddontang/ledisdb/ledis"
	"io"
	"net/http"
	"strconv"
	"strings"

	"encoding/json"
	"github.com/ugorji/go/codec"
	"gopkg.in/mgo.v2/bson"
)

var allowedContentTypes = map[string]struct{}{
	"json":    struct{}{},
	"bson":    struct{}{},
	"msgpack": struct{}{},
}

type httpClient struct {
	app *App
	db  *ledis.DB
	ldb *ledis.Ledis

	resp responseWriter
	req  *requestContext
}

type httpWriter struct {
	contentType string
	cmd         string
	w           http.ResponseWriter
}

// http context

func newClientHTTP(app *App, w http.ResponseWriter, r *http.Request) {
	var err error
	c := new(httpClient)
	c.app = app
	c.ldb = app.ldb
	c.db, err = c.ldb.Select(0)
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}

	c.req, err = c.makeRequest(app, r, w)
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}
	c.req.perform()
}

func (c *httpClient) addr(r *http.Request) string {
	addr := r.Header.Get("X-Forwarded-For")
	if addr == "" {
		addr = r.Header.Get("X-Real-IP")
		if addr == "" {
			addr = r.Header.Get("Remote-Addr")
		}
	}
	return addr
}

func (c *httpClient) makeRequest(app *App, r *http.Request, w http.ResponseWriter) (*requestContext, error) {
	var err error

	db, cmd, argsStr, contentType := c.parseReqPath(r)

	c.db, err = app.ldb.Select(db)
	if err != nil {
		return nil, err
	}

	contentType = strings.ToLower(contentType)

	if _, ok := allowedContentTypes[contentType]; !ok {
		return nil, fmt.Errorf("unsupported content type: '%s', only json, bson, msgpack are supported", contentType)
	}

	req := newRequestContext(app)
	args := make([][]byte, len(argsStr))
	for i, arg := range argsStr {
		args[i] = []byte(arg)
	}

	req.cmd = strings.ToLower(cmd)
	req.args = args
	req.remoteAddr = c.addr(r)
	req.resp = &httpWriter{contentType, cmd, w}
	return req, nil
}

func (c *httpClient) parseReqPath(r *http.Request) (db int, cmd string, args []string, contentType string) {

	contentType = r.FormValue("type")
	if contentType == "" {
		contentType = "json"
	}

	substrings := strings.Split(strings.TrimLeft(r.URL.Path, "/"), "/")
	if len(substrings) == 1 {
		return 0, substrings[0], substrings[1:], contentType
	}
	db, err := strconv.Atoi(substrings[0])
	if err != nil {
		cmd = substrings[0]
		args = substrings[1:]
	} else {
		cmd = substrings[1]
		args = substrings[2:]
	}

	return
}

// http writer<commit_msg>http interface: not support cmds of repl<commit_after>package server

import (
	"fmt"
	"github.com/siddontang/go-log/log"
	"github.com/siddontang/ledisdb/ledis"
	"io"
	"net/http"
	"strconv"
	"strings"

	"encoding/json"
	"github.com/ugorji/go/codec"
	"gopkg.in/mgo.v2/bson"
)

var allowedContentTypes = map[string]struct{}{
	"json":    struct{}{},
	"bson":    struct{}{},
	"msgpack": struct{}{},
}

type httpClient struct {
	app *App
	db  *ledis.DB
	ldb *ledis.Ledis

	resp responseWriter
	req  *requestContext
}

type httpWriter struct {
	contentType string
	cmd         string
	w           http.ResponseWriter
}

func newClientHTTP(app *App, w http.ResponseWriter, r *http.Request) {
	var err error
	c := new(httpClient)
	c.app = app
	c.ldb = app.ldb
	c.db, err = c.ldb.Select(0)
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}

	c.req, err = c.makeRequest(app, r, w)
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}
	c.req.perform()
}

func (c *httpClient) addr(r *http.Request) string {
	addr := r.Header.Get("X-Forwarded-For")
	if addr == "" {
		addr = r.Header.Get("X-Real-IP")
		if addr == "" {
			addr = r.Header.Get("Remote-Addr")
		}
	}
	return addr
}

func (c *httpClient) makeRequest(app *App, r *http.Request, w http.ResponseWriter) (*requestContext, error) {
	var err error

	db, cmd, argsStr, contentType := c.parseReqPath(r)

	c.db, err = app.ldb.Select(db)
	if err != nil {
		return nil, err
	}

	contentType = strings.ToLower(contentType)

	if _, ok := allowedContentTypes[contentType]; !ok {
		return nil, fmt.Errorf("unsupported content type: '%s', only json, bson, msgpack are supported", contentType)
	}

	req := newRequestContext(app)
	args := make([][]byte, len(argsStr))
	for i, arg := range argsStr {
		args[i] = []byte(arg)
	}

	req.cmd = strings.ToLower(cmd)

	if req.cmd == "slaveof" || req.cmd == "fullsync" || req.cmd == "sync" {
		return nil, fmt.Errorf("unsupported command: '%s'", cmd)
	}

	req.remoteAddr = c.addr(r)
	req.resp = &httpWriter{contentType, cmd, w}
	return req, nil
}

func (c *httpClient) parseReqPath(r *http.Request) (db int, cmd string, args []string, contentType string) {

	contentType = r.FormValue("type")
	if contentType == "" {
		contentType = "json"
	}

	substrings := strings.Split(strings.TrimLeft(r.URL.Path, "/"), "/")
	if len(substrings) == 1 {
		return 0, substrings[0], substrings[1:], contentType
	}
	db, err := strconv.Atoi(substrings[0])
	if err != nil {
		cmd = substrings[0]
		args = substrings[1:]
	} else {
		cmd = substrings[1]
		args = substrings[2:]
	}

	return
}

// http writer
{'fraction_non_alphanumeric': 0.16739649681528662, 'fraction_numerical': 0.004378980891719745, 'mean_word_length': 3.6875, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2092246', 'n_tokens_mistral': 2022, 'n_tokens_neox': 1949, 'n_words': 450}