<commit_before>// Package couch provides a CouchDB API.
package couch

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/dustin/httputil"
)

var defaultHdrs = map[string][]string{}

// HTTP Client used by typical requests.
//
// Defaults to http.DefaultClient
var HTTPClient = http.DefaultClient

func createReq(u string) (*http.Request, error) {
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	if req.URL.User != nil {
		if p, hasp := req.URL.User.Password(); hasp {
			req.SetBasicAuth(req.URL.User.Username(), p)
		}
	}
	return req, nil
}

func unmarshalURL(u string, results interface{}) error {
	req, err := createReq(u)
	if err != nil {
		return err
	}

	r, err := HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()

	if r.StatusCode != 200 {
		return httputil.HTTPError(r)
	}

	return json.NewDecoder(r.Body).Decode(results)
}

type idAndRev struct {
	ID  string `json:"_id"`
	Rev string `json:"_rev"`
}

// Sends a query to CouchDB and parses the response back.
// method: the name of the HTTP method (POST, PUT,...)
// url: the URL to interact with
// headers: additional headers to pass to the request
// in: body of the request
// out: a structure to fill in with the returned JSON document
func interact(method, u string, headers map[string][]string, in []byte, out interface{}) (int, error) {
	fullHeaders := map[string][]string{}
	for k, v := range headers {
		fullHeaders[k] = v
	}
	if in != nil {
		fullHeaders["Content-Type"] = []string{"application/json"}
	}

	req, err := http.NewRequest(method, u, bytes.NewReader(in))
	if err != nil {
		return 0, err
	}

	req.ContentLength = int64(len(in))
	req.Header = fullHeaders
	req.Close = true

	res, err := HTTPClient.Do(req)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return res.StatusCode, httputil.HTTPError(res)
	}
	return res.StatusCode, json.NewDecoder(res.Body).Decode(out)
}

// Database represents operations available on an existing CouchDB
type Database struct {
	Scheme   string
	Host     string
	Port     string
	Name     string
	authinfo *url.Userinfo

	changesDialer    func(string, string) (net.Conn, error)
	changesFailDelay time.Duration
}

// BaseURL returns the URL to the database server containing this database.
func (p Database) BaseURL() string {
	var scheme string
	if p.Scheme == "" {
		scheme = "http"
	} else {
		scheme = p.Scheme
	}
	if p.authinfo == nil {
		return fmt.Sprintf("%s://%s:%s", scheme, p.Host, p.Port)
	}
	return fmt.Sprintf("%s://%s@%s:%s", scheme, p.authinfo.String(), p.Host, p.Port)
}

// DBURL returns the URL to this specific database.
func (p Database) DBURL() string {
	return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Running returns true if CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
	var welcomeJson map[string]interface{}
	u := fmt.Sprintf("%s", p.BaseURL())
	err := unmarshalURL(u, &welcomeJson)
	if err != nil {
		return false
	}
	_, ok := welcomeJson["version"]
	return ok
}

type databaseInfo struct {
	DBName string `json:"db_name"`
	// other stuff too, ignore for now
}

// Exists returns true if this database exists on the CouchDB server
func (p Database) Exists() bool {
	di := &databaseInfo{}
	return unmarshalURL(p.DBURL(), &di) == nil && di.DBName == p.Name
}

func (p Database) simpleOp(method, url string, nokerr error) error {
	ir := Response{}
	if _, err := interact(method, url, defaultHdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return nokerr
	}
	return nil
}

var (
	errNewDB = errors.New("create database operation returned not-OK")
	errDelDB = errors.New("delete database operation returned not-OK")<commit_msg>feat: add posibility to disable certificates check for https connections<commit_after>// Package couch provides a CouchDB API.
package couch

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/dustin/httputil"
)

var defaultHdrs = map[string][]string{}

// HTTP Client used by typical requests.
//
// Defaults to http.DefaultClient
var HTTPClient *http.Client

func init() {
	if len(os.Getenv("GO_COUCH_DISABLE_CERT_CHECK")) > 0 {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		HTTPClient = &http.Client{Transport: tr}
	} else {
		HTTPClient = http.DefaultClient
	}
}

func createReq(u string) (*http.Request, error) {
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	if req.URL.User != nil {
		if p, hasp := req.URL.User.Password(); hasp {
			req.SetBasicAuth(req.URL.User.Username(), p)
		}
	}
	return req, nil
}

func unmarshalURL(u string, results interface{}) error {
	req, err := createReq(u)
	if err != nil {
		return err
	}

	r, err := HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()

	if r.StatusCode != 200 {
		return httputil.HTTPError(r)
	}

	return json.NewDecoder(r.Body).Decode(results)
}

type idAndRev struct {
	ID  string `json:"_id"`
	Rev string `json:"_rev"`
}

// Sends a query to CouchDB and parses the response back.
// method: the name of the HTTP method (POST, PUT,...)
// url: the URL to interact with
// headers: additional headers to pass to the request
// in: body of the request
// out: a structure to fill in with the returned JSON document
func interact(method, u string, headers map[string][]string, in []byte, out interface{}) (int, error) {
	fullHeaders := map[string][]string{}
	for k, v := range headers {
		fullHeaders[k] = v
	}
	if in != nil {
		fullHeaders["Content-Type"] = []string{"application/json"}
	}

	req, err := http.NewRequest(method, u, bytes.NewReader(in))
	if err != nil {
		return 0, err
	}

	req.ContentLength = int64(len(in))
	req.Header = fullHeaders
	req.Close = true

	res, err := HTTPClient.Do(req)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return res.StatusCode, httputil.HTTPError(res)
	}
	return res.StatusCode, json.NewDecoder(res.Body).Decode(out)
}

// Database represents operations available on an existing CouchDB
type Database struct {
	Scheme   string
	Host     string
	Port     string
	Name     string
	authinfo *url.Userinfo

	changesDialer    func(string, string) (net.Conn, error)
	changesFailDelay time.Duration
}

// BaseURL returns the URL to the database server containing this database.
func (p Database) BaseURL() string {
	var scheme string
	if p.Scheme == "" {
		scheme = "http"
	} else {
		scheme = p.Scheme
	}
	if p.authinfo == nil {
		return fmt.Sprintf("%s://%s:%s", scheme, p.Host, p.Port)
	}
	return fmt.Sprintf("%s://%s@%s:%s", scheme, p.authinfo.String(), p.Host, p.Port)
}

// DBURL returns the URL to this specific database.
func (p Database) DBURL() string {
	return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Running returns true if CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
	var welcomeJson map[string]interface{}
	u := fmt.Sprintf("%s", p.BaseURL())
	err := unmarshalURL(u, &welcomeJson)
	if err != nil {
		log.Printf("unmarshalURL failed: %s", err.Error())
		return false
	}
	_, ok := welcomeJson["version"]
	return ok
}

type databaseInfo struct {
	DBName string `json:"db_name"`
	// other stuff too, ignore for now
}

// Exists returns true if this database exists on the CouchDB server
func (p Database) Exists() bool {
	di := &databaseInfo{}
	return unmarshalURL(p.DBURL(), &di) == nil && di.DBName == p.Name
}

func (p Database) simpleOp(method, url string, nokerr error) error {
	ir := Response{}
	if _, err := interact(method, url, defaultHdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return nokerr
	}
	return nil
}

var (
	errNewDB = errors.New("create database operation returned not-OK")
	errDelDB = errors.New("delete database operation returned not-OK")
{'fraction_non_alphanumeric': 0.14064064064064064, 'fraction_numerical': 0.0033783783783783786, 'mean_word_length': 3.73238602723505, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5019896', 'n_tokens_mistral': 2851, 'n_tokens_neox': 2771, 'n_words': 855}