<commit_before>package web

import (
    "bytes"
    "fmt"
    "http"
    "io"
    "io/ioutil"
    "mime"
    "mime/multipart"
    "net"
    "os"
    "reflect"
    "strconv"
    "strings"
    "url"
)

type filedata struct {
    Filename string
    Data     []byte
}

type Request struct {
    Method     string   // GET, POST, PUT, etc.
    RawURL     string   // The raw URL given in the request.
    URL        *url.URL // Parsed URL.
    Proto      string   // "HTTP/1.0"
    ProtoMajor int      // 1
    ProtoMinor int      // 0
    Headers    http.Header
    Body       io.Reader
    Close      bool
    Host       string
    Referer    string
    UserAgent  string
    FullParams map[string][]string
    Params     map[string]string
    ParamData  []byte
    Cookies    map[string]string
    Cookie     []*http.Cookie
    Files      map[string]filedata
    RemoteAddr string
    RemotePort int
}

type badStringError struct {
    what string
    str  string
}

func (e *badStringError) String() string { return fmt.Sprintf("%s %q", e.what, e.str) }

func flattenParams(fullParams map[string][]string) map[string]string {
    params := map[string]string{}
    for name, lst := range fullParams {
        if len(lst) > 0 {
            params[name] = lst[0]
        }
    }
    return params
}

func newRequest(hr *http.Request, hc http.ResponseWriter) *Request {

    remoteAddr, _ := net.ResolveTCPAddr("tcp", hr.RemoteAddr)

    req := Request{
        Method:     hr.Method,
        URL:        hr.URL,
        Proto:      hr.Proto,
        ProtoMajor: hr.ProtoMajor,
        ProtoMinor: hr.ProtoMinor,
        Headers:    hr.Header,
        Body:       hr.Body,
        Close:      hr.Close,
        Host:       hr.Host,
        Referer:    hr.Referer(),
        UserAgent:  hr.UserAgent(),
        FullParams: hr.Form,
        Cookie:     hr.Cookies(),
        RemoteAddr: remoteAddr.IP.String(),
        RemotePort: remoteAddr.Port,
    }
    return &req
}

func newRequestCgi(headers http.Header, body io.Reader) *Request {
    var httpheader = make(http.Header)
    for header, value := range headers {
        if strings.HasPrefix(header, "Http_") {
            newHeader := header[5:]
            newHeader = strings.Replace(newHeader, "_", "-", -1)
            newHeader = http.CanonicalHeaderKey(newHeader)
            httpheader[newHeader] = value
        }
    }

    host := httpheader.Get("Host")
    method := headers.Get("REQUEST_METHOD")
    path := headers.Get("REQUEST_URI")
    port := headers.Get("SERVER_PORT")
    proto := headers.Get("SERVER_PROTOCOL")
    rawurl := "http://" + host + ":" + port + path
    url_, _ := url.Parse(rawurl)
    useragent := headers.Get("USER_AGENT")
    remoteAddr := headers.Get("REMOTE_ADDR")
    remotePort, _ := strconv.Atoi(headers.Get("REMOTE_PORT"))

    if method == "POST" {
        if ctype, ok := headers["CONTENT_TYPE"]; ok {
            httpheader["Content-Type"] = ctype
        }

        if clength, ok := headers["CONTENT_LENGTH"]; ok {
            httpheader["Content-Length"] = clength
        }
    }

    //read the cookies
    cookies := readCookies(httpheader)

    req := Request{
        Method:     method,
        RawURL:     rawurl,
        URL:        url_,
        Proto:      proto,
        Host:       host,
        UserAgent:  useragent,
        Body:       body,
        Headers:    httpheader,
        RemoteAddr: remoteAddr,
        RemotePort: remotePort,
        Cookie:     cookies,
    }

    return &req
}

func parseForm(m map[string][]string, query string) (err os.Error) {
    for _, kv := range strings.Split(query, "&") {
        kvPair := strings.SplitN(kv, "=", 2)

        var key, value string
        var e os.Error
        key, e = url.QueryUnescape(kvPair[0])
        if e == nil && len(kvPair) > 1 {
            value, e = url.QueryUnescape(kvPair[1])
        }
        if e != nil {
            err = e
        }

        vec, ok := m[key]
        if !ok {
            vec = []string{}
        }
        m[key] = append(vec, value)
    }

    return
}

// ParseForm parses the request body as a form for POST requests, or the raw query for GET requests.
// It is idempotent.
func (r *Request) parseParams() (err os.Error) {

    if r.Params != nil {
        return
    }
    r.FullParams = make(map[string][]string)
    queryParams := r.URL.RawQuery
    switch r.Method {
    case "POST":
        if r.Body == nil {
            return os.NewError("missing form body")
        }

        ct := r.Headers.Get("Content-Type")
        switch strings.SplitN(ct, ";", 2)[0] {
        case "text/plain", "application/x-www-form-urlencoded", "":
            var b []byte
            if b, err = ioutil.ReadAll(r.Body); err != nil {
                return err
            }
            r.ParamData = b
        case "multipart/form-data":
            _, params := mime.ParseMediaType(ct)
            boundary, ok := params["boundary"]
            if !ok {
                return os.NewError("Missing Boundary")
            }

            reader := multipart.NewReader(r.Body, boundary)
            r.Files = make(map[string]filedata)
            for {
                part, err := reader.NextPart()
                if part == nil && err == os.EOF {
                    break
                }

                if err != nil {
                    return err
                }

                //read the data
                data, _ := ioutil.ReadAll(part)
                //check for the 'filename' param
                v := part.Header.Get("Content-Disposition")
                if v == "" {
                    continue
                }
                name := part.FormName()
                d, params := mime.ParseMediaType(v)
                if d != "form-data" {
                    continue
                }
                if params["filename"] != "" {
                    r.Files[name] = filedata{params["filename"], data}
                } else {
                    var params []string = r.FullParams[name]
                    params = append(params, string(data))
                    r.FullParams[name] = params
                }

            }
        default:
            return &badStringError{"unknown Content-Type", ct}
        }
    }

    if queryParams != "" {
        err = parseForm(r.FullParams, queryParams)
        if err != nil {
            return err
        }
    }

    if len(r.ParamData) > 0 {
        err = parseForm(r.FullParams, string(r.ParamData))
        if err != nil {
            return err
        }
        r.Body = bytes.NewBuffer(r.ParamData)<commit_msg>Update for weekly 2011-09-01<commit_after>package web

import (
    "bytes"
    "fmt"
    "http"
    "io"
    "io/ioutil"
    "mime"
    "mime/multipart"
    "net"
    "os"
    "reflect"
    "strconv"
    "strings"
    "url"
)

type filedata struct {
    Filename string
    Data     []byte
}

type Request struct {
    Method     string   // GET, POST, PUT, etc.
    RawURL     string   // The raw URL given in the request.
    URL        *url.URL // Parsed URL.
    Proto      string   // "HTTP/1.0"
    ProtoMajor int      // 1
    ProtoMinor int      // 0
    Headers    http.Header
    Body       io.Reader
    Close      bool
    Host       string
    Referer    string
    UserAgent  string
    FullParams map[string][]string
    Params     map[string]string
    ParamData  []byte
    Cookies    map[string]string
    Cookie     []*http.Cookie
    Files      map[string]filedata
    RemoteAddr string
    RemotePort int
}

type badStringError struct {
    what string
    str  string
}

func (e *badStringError) String() string { return fmt.Sprintf("%s %q", e.what, e.str) }

func flattenParams(fullParams map[string][]string) map[string]string {
    params := map[string]string{}
    for name, lst := range fullParams {
        if len(lst) > 0 {
            params[name] = lst[0]
        }
    }
    return params
}

func newRequest(hr *http.Request, hc http.ResponseWriter) *Request {

    remoteAddr, _ := net.ResolveTCPAddr("tcp", hr.RemoteAddr)

    req := Request{
        Method:     hr.Method,
        URL:        hr.URL,
        Proto:      hr.Proto,
        ProtoMajor: hr.ProtoMajor,
        ProtoMinor: hr.ProtoMinor,
        Headers:    hr.Header,
        Body:       hr.Body,
        Close:      hr.Close,
        Host:       hr.Host,
        Referer:    hr.Referer(),
        UserAgent:  hr.UserAgent(),
        FullParams: hr.Form,
        Cookie:     hr.Cookies(),
        RemoteAddr: remoteAddr.IP.String(),
        RemotePort: remoteAddr.Port,
    }
    return &req
}

func newRequestCgi(headers http.Header, body io.Reader) *Request {
    var httpheader = make(http.Header)
    for header, value := range headers {
        if strings.HasPrefix(header, "Http_") {
            newHeader := header[5:]
            newHeader = strings.Replace(newHeader, "_", "-", -1)
            newHeader = http.CanonicalHeaderKey(newHeader)
            httpheader[newHeader] = value
        }
    }

    host := httpheader.Get("Host")
    method := headers.Get("REQUEST_METHOD")
    path := headers.Get("REQUEST_URI")
    port := headers.Get("SERVER_PORT")
    proto := headers.Get("SERVER_PROTOCOL")
    rawurl := "http://" + host + ":" + port + path
    url_, _ := url.Parse(rawurl)
    useragent := headers.Get("USER_AGENT")
    remoteAddr := headers.Get("REMOTE_ADDR")
    remotePort, _ := strconv.Atoi(headers.Get("REMOTE_PORT"))

    if method == "POST" {
        if ctype, ok := headers["CONTENT_TYPE"]; ok {
            httpheader["Content-Type"] = ctype
        }

        if clength, ok := headers["CONTENT_LENGTH"]; ok {
            httpheader["Content-Length"] = clength
        }
    }

    //read the cookies
    cookies := readCookies(httpheader)

    req := Request{
        Method:     method,
        RawURL:     rawurl,
        URL:        url_,
        Proto:      proto,
        Host:       host,
        UserAgent:  useragent,
        Body:       body,
        Headers:    httpheader,
        RemoteAddr: remoteAddr,
        RemotePort: remotePort,
        Cookie:     cookies,
    }

    return &req
}

func parseForm(m map[string][]string, query string) (err os.Error) {
    for _, kv := range strings.Split(query, "&") {
        kvPair := strings.SplitN(kv, "=", 2)

        var key, value string
        var e os.Error
        key, e = url.QueryUnescape(kvPair[0])
        if e == nil && len(kvPair) > 1 {
            value, e = url.QueryUnescape(kvPair[1])
        }
        if e != nil {
            err = e
        }

        vec, ok := m[key]
        if !ok {
            vec = []string{}
        }
        m[key] = append(vec, value)
    }

    return
}

// ParseForm parses the request body as a form for POST requests, or the raw query for GET requests.
// It is idempotent.
func (r *Request) parseParams() (err os.Error) {

    if r.Params != nil {
        return
    }
    r.FullParams = make(map[string][]string)
    queryParams := r.URL.RawQuery
    switch r.Method {
    case "POST":
        if r.Body == nil {
            return os.NewError("missing form body")
        }

        ct := r.Headers.Get("Content-Type")
        switch strings.SplitN(ct, ";", 2)[0] {
        case "text/plain", "application/x-www-form-urlencoded", "":
            var b []byte
            if b, err = ioutil.ReadAll(r.Body); err != nil {
                return err
            }
            r.ParamData = b
        case "multipart/form-data":
            _, params, err := mime.ParseMediaType(ct)
            if err != nil {
                return err
            }
            boundary, ok := params["boundary"]
            if !ok {
                return os.NewError("Missing Boundary")
            }

            reader := multipart.NewReader(r.Body, boundary)
            r.Files = make(map[string]filedata)
            for {
                part, err := reader.NextPart()
                if part == nil && err == os.EOF {
                    break
                }

                if err != nil {
                    return err
                }

                //read the data
                data, _ := ioutil.ReadAll(part)
                //check for the 'filename' param
                v := part.Header.Get("Content-Disposition")
                if v == "" {
                    continue
                }
                name := part.FormName()
                d, params, err := mime.ParseMediaType(v)
                if err != nil {
                    return err
                }
                if d != "form-data" {
                    continue
                }
                if params["filename"] != "" {
                    r.Files[name] = filedata{params["filename"], data}
                } else {
                    var params []string = r.FullParams[name]
                    params = append(params, string(data))
                    r.FullParams[name] = params
                }

            }
        default:
            return &badStringError{"unknown Content-Type", ct}
        }
    }

    if queryParams != "" {
        err = parseForm(r.FullParams, queryParams)
        if err != nil {
            return err
        }
    }

    if len(r.ParamData) > 0 {
        err = parseForm(r.FullParams, string(r.ParamData))
        if err != nil {
            return err
        }
        r.Body = bytes.NewBuffer(r.ParamData)
{'fraction_non_alphanumeric': 0.11785794308760923, 'fraction_numerical': 0.002838150720740907, 'mean_word_length': 1.539833080424886, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5575967', 'n_tokens_mistral': 4090, 'n_tokens_neox': 3910, 'n_words': 956}