<commit_before>// Copyright 2011 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Reading and parsing of ICAP requests.

// Package icap provides an extensible ICAP server.
package icap

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

type badStringError struct {
	what string
	str  string
}

func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }

// A Request represents a parsed ICAP request.
type Request struct {
	Method     string               // REQMOD, RESPMOD, OPTIONS, etc.
	RawURL     string               // The URL given in the request.
	URL        *url.URL             // Parsed URL.
	Proto      string               // The protocol version.
	Header     textproto.MIMEHeader // The ICAP header
	RemoteAddr string               // the address of the computer sending the request
	Preview    []byte               // the body data for an ICAP preview

	// The HTTP messages.
	Request  *http.Request
	Response *http.Response
}

// ReadRequest reads and parses a request from b.
func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) {
	tp := textproto.NewReader(b.Reader)
	req = new(Request)

	// Read first line.
	var s string
	s, err = tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}

	f := strings.SplitN(s, " ", 3)
	if len(f) < 3 {
		return nil, &badStringError{"malformed ICAP request", s}
	}
	req.Method, req.RawURL, req.Proto = f[0], f[1], f[2]

	req.URL, err = url.ParseRequest(req.RawURL)
	if err != nil {
		return nil, err
	}

	req.Header, err = tp.ReadMIMEHeader()
	if err != nil {
		return nil, err
	}

	s = req.Header.Get("Encapsulated")
	if s == "" {
		return req, nil // No HTTP headers or body.
	}
	eList := strings.Split(s, ", ")
	var initialOffset, reqHdrLen, respHdrLen int
	var hasBody bool
	var prevKey string
	var prevValue int
	for _, item := range eList {
		eq := strings.Index(item, "=")
		if eq == -1 {
			return nil, &badStringError{"malformed Encapsulated: header", s}
		}
		key := item[:eq]
		value, err := strconv.Atoi(item[eq+1:])<commit_msg>Update for Go 1<commit_after>// Copyright 2011 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Reading and parsing of ICAP requests.

// Package icap provides an extensible ICAP server.
package icap

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

type badStringError struct {
	what string
	str  string
}

func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }

// A Request represents a parsed ICAP request.
type Request struct {
	Method     string               // REQMOD, RESPMOD, OPTIONS, etc.
	RawURL     string               // The URL given in the request.
	URL        *url.URL             // Parsed URL.
	Proto      string               // The protocol version.
	Header     textproto.MIMEHeader // The ICAP header
	RemoteAddr string               // the address of the computer sending the request
	Preview    []byte               // the body data for an ICAP preview

	// The HTTP messages.
	Request  *http.Request
	Response *http.Response
}

// ReadRequest reads and parses a request from b.
func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) {
	tp := textproto.NewReader(b.Reader)
	req = new(Request)

	// Read first line.
	var s string
	s, err = tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}

	f := strings.SplitN(s, " ", 3)
	if len(f) < 3 {
		return nil, &badStringError{"malformed ICAP request", s}
	}
	req.Method, req.RawURL, req.Proto = f[0], f[1], f[2]

	req.URL, err = url.ParseRequestURI(req.RawURL)
	if err != nil {
		return nil, err
	}

	req.Header, err = tp.ReadMIMEHeader()
	if err != nil {
		return nil, err
	}

	s = req.Header.Get("Encapsulated")
	if s == "" {
		return req, nil // No HTTP headers or body.
	}
	eList := strings.Split(s, ", ")
	var initialOffset, reqHdrLen, respHdrLen int
	var hasBody bool
	var prevKey string
	var prevValue int
	for _, item := range eList {
		eq := strings.Index(item, "=")
		if eq == -1 {
			return nil, &badStringError{"malformed Encapsulated: header", s}
		}
		key := item[:eq]
		value, err := strconv.Atoi(item[eq+1:])
{'fraction_non_alphanumeric': 0.12232280856701258, 'fraction_numerical': 0.005078383749172003, 'mean_word_length': 2.700980392156863, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6360204', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1554, 'n_words': 516}