<commit_before>package server

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-kit/kit/log"

	"context"

	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
	"github.com/slotix/dataflowkit/splash"
)

var (
	// ErrBadRouting is returned when an expected path variable is missing.
	ErrBadRouting = errors.New("inconsistent mapping between route and handler (programmer error)")
	//ErrInvalidURL is returned if validation of URL fails
	ErrInvalidURL = errors.New("invalid URL specified")
)

//decodeFetchRequest
func decodeFetchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var request splash.Request
	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
		logger.Println(err)
		return nil, err
	}
	//logger.Println(request)
	//request.URL normalization and validation
	reqURL := strings.TrimSpace(request.URL)
	if _, err := url.ParseRequestURI(reqURL); err != nil {
		return nil, err
	}
	request.URL = reqURL
	logger.Println("transport request",request.URL)
	return request, nil
}

func encodeFetchResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		// Not a Go kit transport error, but a business-logic error.
		// Provide those as HTTP errors.
		encodeError(ctx, e.error(), w)
		return nil
	}
	sResponse := response.(*splash.Response)
	logger.Println("transport response", sResponse.Error)
	if sResponse.Error != ""{
		return errors.New(sResponse.Error)
	}
	content, err := sResponse.GetContent()
	if err != nil {
		return err
	}
	data, err := ioutil.ReadAll(content)

	if err != nil {
		return err
	}
	_, err = w.Write(data)

	if err != nil {
		return err
	}
	return nil
}

//decodeParseRequest
func decodeParseRequest(_ context.Context, r *http.Request) (interface{}, error) {
	request, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}

	return request, nil
}

func encodeParseResponse(_ context.Context, w http.ResponseWriter, response interface{}) error {
	data, err := ioutil.ReadAll(response.(io.Reader))
	if err != nil {
		return err
	}
	_, err = w.Write(data)

	if err != nil {
		return err
	}
	return nil
}

// errorer is implemented by all concrete response types that may contain
// errors. It allows us to change the HTTP response code without needing to
// trigger an endpoint (transport-level) error.
type errorer interface {
	error() error
}

// encode error
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	if err == nil {
		panic("encodeError with nil error")
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(http.StatusInternalServerError)
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error": err.Error(),
	})
}
<commit_msg>enchance 400 Bad Request error handling<commit_after>package server

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-kit/kit/log"

	"context"

	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
	"github.com/slotix/dataflowkit/splash"
)

var (
	// ErrBadRouting is returned when an expected path variable is missing.
	ErrBadRouting = errors.New("inconsistent mapping between route and handler (programmer error)")
	//ErrInvalidURL is returned if validation of URL fails
	ErrInvalidURL = errors.New("invalid URL specified")
)

type errorFetchRequest struct {
	err  error
}

func (e *errorFetchRequest) Error() string { return e.err.Error() }


//decodeFetchRequest 
//if error is not nil, server should return  
//400 Bad Request
//The server cannot or will not process the request due to an apparent client error (e.g., malformed request syntax, size too large, invalid request message framing, or deceptive request routing).
func decodeFetchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var request splash.Request
	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
		logger.Printf("Type: %T\n", err)
		return nil, &errorFetchRequest{err}//err
	}
	//request.URL normalization and validation
	reqURL := strings.TrimSpace(request.URL)
	if _, err := url.ParseRequestURI(reqURL); err != nil {
		//logger.Printf("Type: %T\n", err)
		//logger.Printf("Op: %s\n", err.(*url.Error).Op)
		return nil, &errorFetchRequest{err}
	}
	request.URL = reqURL
	logger.Println("transport request", request.URL)
	return request, nil
}

func encodeFetchResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		// Not a Go kit transport error, but a business-logic error.
		// Provide those as HTTP errors.
		encodeError(ctx, e.error(), w)
		return nil
	}
	sResponse := response.(*splash.Response)
	logger.Println("transport response", sResponse.Error)
	if sResponse.Error != "" {
		return errors.New(sResponse.Error)
	}
	content, err := sResponse.GetContent()
	if err != nil {
		return err
	}
	data, err := ioutil.ReadAll(content)

	if err != nil {
		return err
	}
	_, err = w.Write(data)

	if err != nil {
		return err
	}
	return nil
}

//decodeParseRequest
func decodeParseRequest(_ context.Context, r *http.Request) (interface{}, error) {
	request, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}

	return request, nil
}

func encodeParseResponse(_ context.Context, w http.ResponseWriter, response interface{}) error {
	data, err := ioutil.ReadAll(response.(io.Reader))
	if err != nil {
		return err
	}
	_, err = w.Write(data)

	if err != nil {
		return err
	}
	return nil
}

// errorer is implemented by all concrete response types that may contain
// errors. It allows us to change the HTTP response code without needing to
// trigger an endpoint (transport-level) error.
type errorer interface {
	error() error
}

// encode error
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	if err == nil {
		panic("encodeError with nil error")
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	//logger.Printf("Type: %T\n", err)
	//logger.Println(err)
	//t = err.(type)
	var httpStatus int
	switch err.(type) {
	default:
		httpStatus = http.StatusInternalServerError
		w.WriteHeader(httpStatus)
		
	case *errorFetchRequest:
		//return 400 Status
		httpStatus = http.StatusBadRequest
		w.WriteHeader(httpStatus)
	}
	
	//w.WriteHeader(http.StatusInternalServerError)
	json.NewEncoder(w).Encode(map[string]interface{}{
		//"errorType": "BadRequest",
		"httpStatus": httpStatus,
		"message": err.Error(),
	})
}

{'fraction_non_alphanumeric': 0.13102820746132848, 'fraction_numerical': 0.0016681831968456172, 'mean_word_length': 4.124320124320124, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5481710', 'n_tokens_mistral': 2212, 'n_tokens_neox': 2144, 'n_words': 616}