<commit_before>package dsdk

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	udc "github.com/Datera/go-udc/pkg/udc"
	uuid "github.com/google/uuid"
	greq "github.com/levigross/grequests"
	log "github.com/sirupsen/logrus"
)

var (
	InvalidRequest         = 400
	PermissionDenied       = 401
	RetryRequestAfterLogin = 9999
	badStatus              = map[int]error{
		InvalidRequest:         fmt.Errorf("InvalidRequest"),
		PermissionDenied:       fmt.Errorf("PermissionDenied"),
		RetryRequestAfterLogin: fmt.Errorf("RetryRequestAfterLogin"),
	}
)

type ApiConnection struct {
	username   string
	password   string
	apiVersion string
	tenant     string
	secure     bool
	baseUrl    *url.URL
	apikey     string
}

type ApiErrorResponse struct {
	Name         string            `json:"name"`
	Code         int               `json:"code"`
	Http         int               `json:"http"`
	Message      string            `json:"message"`
	Ts           string            `json:"ts"`
	Version      string            `json:"version"`
	Op           string            `json:"op"`
	Tenant       string            `json:"tenant"`
	Path         string            `json:"path"`
	Params       map[string]string `json:"params"`
	ConnInfo     map[string]string `json:"connInfo"`
	ClientId     string            `json:"client_id"`
	ClientType   string            `json:"client_type"`
	Id           string            `json:"api_req_id"`
	TenancyClass string            `json:"tenancy_class"`
}

type ApiLogin struct {
	Key     string `json:"key"`
	Version string `json:"version"`
	ReqTime int    `json:"request_time"`
}

type ApiVersions struct {
	ApiVersions []string `json:"api_versions"`
}

type ApiListOuter struct {
	Data     []interface{}          `json:"data"`
	Version  string                 `json:"version"`
	Metadata map[string]interface{} `json:"metadata"`
	ReqTime  int                    `json:"request_time"`
	Tenant   string                 `json:"tenant"`
	Path     string                 `json:"path"`
}

type ApiOuter struct {
	Data     map[string]interface{} `json:"data"`
	Version  string                 `json:"version"`
	Metadata map[string]interface{} `json:"metadata"`
	ReqTime  int                    `json:"request_time"`
	Tenant   string                 `json:"tenant"`
	Path     string                 `json:"path"`
}

func init() {
	// TODO(_alastor_): Disable this and do real certificate verification
	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}

func makeBaseUrl(h, apiv string, secure bool) (*url.URL, error) {
	h = strings.Trim(h, "/")
	if secure {
		return url.Parse(fmt.Sprintf("https://%s:7718/v%s", h, apiv))
	}
	return url.Parse(fmt.Sprintf("http://%s:7717/v%s", h, apiv))
}

func checkResponse(resp *greq.Response, err error, retry bool) (*ApiErrorResponse, error) {
	if err != nil {
		log.Error(err)
		return nil, err
	}
	if resp.StatusCode == PermissionDenied && retry {
		return nil, badStatus[RetryRequestAfterLogin]
	}
	if !resp.Ok {
		eresp := &ApiErrorResponse{}
		resp.JSON(eresp)
		return eresp, badStatus[resp.StatusCode]
	}
	return nil, nil
}

func (c *ApiConnection) do(ctxt context.Context, method, url string, ro *greq.RequestOptions, rs interface{}, retry, sensitive bool) (*ApiErrorResponse, error) {
	gurl := *c.baseUrl
	gurl.Path = path.Join(gurl.Path, url)
	reqId := uuid.Must(uuid.NewRandom()).String()
	sdata, err := json.Marshal(ro.JSON)
	if err != nil {
		log.Errorf("Couldn't stringify data, %s", ro.JSON)
	}
	if sensitive {
		sdata = []byte("********")
	}
	sheaders, err := json.Marshal(ro.Headers)
	if err != nil {
		log.Errorf("Couldn't stringify headers, %s", ro.Headers)
	}
	tid, ok := ctxt.Value("tid").(string)
	if !ok {
		tid = "nil"
	}
	log.Debugf(strings.Join([]string{"\nDatera Trace ID: %s",
		"Datera Request ID: %s",
		"Datera Request URL: %s",
		"Datera Request Method: %s",
		"Datera Request Payload: %s",
		"Datera Request Headers: %s\n"}, "\n"),
		tid, reqId, gurl.String(), method, string(sdata), sheaders)
	t1 := time.Now()
	resp, err := greq.DoRegularRequest(method, gurl.String(), ro)
	t2 := time.Now()
	tDelta := t2.Sub(t1)
	log.Debugf(strings.Join([]string{"\nDatera Trace ID: %s",
		"Datera Response ID: %s",
		"Datera Response TimeDelta: %fs",
		"Datera Response URL: %s",
		"Datera Response Payload: %s",
		"Datera Response Object: %s\n"}, "\n"),
		tid, reqId, tDelta.Seconds(), gurl.String(), resp.String(), "nil")
	eresp, err := checkResponse(resp, err, retry)
	if err == badStatus[RetryRequestAfterLogin] {
		if apiresp, err2 := c.Login(ctxt); err2 != nil {
			log.Errorf("%s", err)
			log.Errorf("%s", err2)
			return apiresp, err2
		}
		return c.do(ctxt, method, url, ro, rs, false, sensitive)
	}
	if err != nil {
		log.Error("Error during checkResponse: %s", err)
		return nil, err
	}
	if eresp != nil {
		log.Errorf("Recieved API Error %s", Pretty(eresp))
		return eresp, nil
	}
	err = resp.JSON(rs)
	if err != nil {
		log.Errorf("Could not unpack response, %s", err)
		return nil, err
	}
	return nil, nil
}<commit_msg>Fixing error logging in connection<commit_after>package dsdk

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	udc "github.com/Datera/go-udc/pkg/udc"
	uuid "github.com/google/uuid"
	greq "github.com/levigross/grequests"
	log "github.com/sirupsen/logrus"
)

var (
	InvalidRequest         = 400
	PermissionDenied       = 401
	RetryRequestAfterLogin = 9999
	badStatus              = map[int]error{
		InvalidRequest:         fmt.Errorf("InvalidRequest"),
		PermissionDenied:       fmt.Errorf("PermissionDenied"),
		RetryRequestAfterLogin: fmt.Errorf("RetryRequestAfterLogin"),
	}
)

type ApiConnection struct {
	username   string
	password   string
	apiVersion string
	tenant     string
	secure     bool
	baseUrl    *url.URL
	apikey     string
}

type ApiErrorResponse struct {
	Name         string            `json:"name"`
	Code         int               `json:"code"`
	Http         int               `json:"http"`
	Message      string            `json:"message"`
	Ts           string            `json:"ts"`
	Version      string            `json:"version"`
	Op           string            `json:"op"`
	Tenant       string            `json:"tenant"`
	Path         string            `json:"path"`
	Params       map[string]string `json:"params"`
	ConnInfo     map[string]string `json:"connInfo"`
	ClientId     string            `json:"client_id"`
	ClientType   string            `json:"client_type"`
	Id           string            `json:"api_req_id"`
	TenancyClass string            `json:"tenancy_class"`
}

type ApiLogin struct {
	Key     string `json:"key"`
	Version string `json:"version"`
	ReqTime int    `json:"request_time"`
}

type ApiVersions struct {
	ApiVersions []string `json:"api_versions"`
}

type ApiListOuter struct {
	Data     []interface{}          `json:"data"`
	Version  string                 `json:"version"`
	Metadata map[string]interface{} `json:"metadata"`
	ReqTime  int                    `json:"request_time"`
	Tenant   string                 `json:"tenant"`
	Path     string                 `json:"path"`
}

type ApiOuter struct {
	Data     map[string]interface{} `json:"data"`
	Version  string                 `json:"version"`
	Metadata map[string]interface{} `json:"metadata"`
	ReqTime  int                    `json:"request_time"`
	Tenant   string                 `json:"tenant"`
	Path     string                 `json:"path"`
}

func init() {
	// TODO(_alastor_): Disable this and do real certificate verification
	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}

func makeBaseUrl(h, apiv string, secure bool) (*url.URL, error) {
	h = strings.Trim(h, "/")
	if secure {
		return url.Parse(fmt.Sprintf("https://%s:7718/v%s", h, apiv))
	}
	return url.Parse(fmt.Sprintf("http://%s:7717/v%s", h, apiv))
}

func checkResponse(resp *greq.Response, err error, retry bool) (*ApiErrorResponse, error) {
	if err != nil {
		log.Error(err)
		return nil, err
	}
	if resp.StatusCode == PermissionDenied && retry {
		return nil, badStatus[RetryRequestAfterLogin]
	}
	if !resp.Ok {
		eresp := &ApiErrorResponse{}
		resp.JSON(eresp)
		return eresp, badStatus[resp.StatusCode]
	}
	return nil, nil
}

func (c *ApiConnection) do(ctxt context.Context, method, url string, ro *greq.RequestOptions, rs interface{}, retry, sensitive bool) (*ApiErrorResponse, error) {
	gurl := *c.baseUrl
	gurl.Path = path.Join(gurl.Path, url)
	reqId := uuid.Must(uuid.NewRandom()).String()
	sdata, err := json.Marshal(ro.JSON)
	if err != nil {
		log.Errorf("Couldn't stringify data, %s", ro.JSON)
	}
	if sensitive {
		sdata = []byte("********")
	}
	sheaders, err := json.Marshal(ro.Headers)
	if err != nil {
		log.Errorf("Couldn't stringify headers, %s", ro.Headers)
	}
	tid, ok := ctxt.Value("tid").(string)
	if !ok {
		tid = "nil"
	}
	log.Debugf(strings.Join([]string{"\nDatera Trace ID: %s",
		"Datera Request ID: %s",
		"Datera Request URL: %s",
		"Datera Request Method: %s",
		"Datera Request Payload: %s",
		"Datera Request Headers: %s\n"}, "\n"),
		tid, reqId, gurl.String(), method, string(sdata), sheaders)
	t1 := time.Now()
	resp, err := greq.DoRegularRequest(method, gurl.String(), ro)
	t2 := time.Now()
	tDelta := t2.Sub(t1)
	log.Debugf(strings.Join([]string{"\nDatera Trace ID: %s",
		"Datera Response ID: %s",
		"Datera Response TimeDelta: %fs",
		"Datera Response URL: %s",
		"Datera Response Payload: %s",
		"Datera Response Object: %s\n"}, "\n"),
		tid, reqId, tDelta.Seconds(), gurl.String(), resp.String(), "nil")
	eresp, err := checkResponse(resp, err, retry)
	if err == badStatus[RetryRequestAfterLogin] {
		if apiresp, err2 := c.Login(ctxt); err2 != nil {
			log.Errorf("%s", err)
			log.Errorf("%s", err2)
			return apiresp, err2
		}
		return c.do(ctxt, method, url, ro, rs, false, sensitive)
	}
	if err != nil {
		log.Errorf("Error during checkResponse: %s\n", err)
		return nil, err
	}
	if eresp != nil {
		log.Errorf("Recieved API Error %s\n", Pretty(eresp))
		return eresp, nil
	}
	err = resp.JSON(rs)
	if err != nil {
		log.Errorf("Could not unpack response, %s\n", err)
		return nil, err
	}
	return nil, nil
}
{'fraction_non_alphanumeric': 0.14727698614093304, 'fraction_numerical': 0.005075151278547726, 'mean_word_length': 3.0121378230227096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3790597', 'n_tokens_mistral': 3610, 'n_tokens_neox': 3520, 'n_words': 854}