<commit_before>/*
 *
 * Author     : <NAME> <vkuznet AT gmail dot com>
 * Description: URL fetch proxy server concurrently fetches data from
 *              provided URL list. It provides a POST HTTP interface
 *              "/getdata" which accepts urls as newline separated encoded
 *              string
 * Created    : Wed Mar 20 13:29:48 EDT 2013
 * License    : MIT
 *
 */
package urlfetch

import (
    "os"
    "log"
    "strings"
    "net/http"
    "io/ioutil"
    "crypto/tls"
)

/*
 * Return array of certificates
 */
func Certs() (tls_certs []tls.Certificate) {
    uproxy := os.Getenv("X509_USER_PROXY")
    uckey  := os.Getenv("X509_USER_KEY")
    ucert  := os.Getenv("X509_USER_CERT")
    log.Println("X509_USER_PROXY", uproxy)
    log.Println("X509_USER_KEY", uckey)
    log.Println("X509_USER_CERT", ucert)
    if  len(uproxy) > 0 {
        x509cert, err := tls.LoadX509KeyPair(uproxy, uproxy)
        if  err != nil {
            log.Println("Fail to parser proxy X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else if len(uckey) > 0 {
        x509cert, err := tls.LoadX509KeyPair(ucert, uckey)
        if  err != nil {
            log.Println("Fail to parser user X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else {
        return
    }
    return
}

/*
 * HTTP client for urlfetch server
 */
func HttpClient() (client *http.Client) {
    // create HTTP client
    certs := Certs()
    if  len(certs) == 0 {
        client = &http.Client{}
        return
    }
    tr := &http.Transport{
        TLSClientConfig: &tls.Config{tls.Certificates: certs},
    }
    client = &http.Client{Transport: tr}
    return
}

/*
 * Getdata(client *http.Client, url string, ch chan<- []byte)
 * Using given client and chanel fetches data for provided URL.
 * Results are redirected to specified channel
 */
func Getdata(client *http.Client, url string, ch chan<- []byte) {
    msg := ""
    resp, err := client.Get(url)
    if  err != nil {
        msg = "Fail to contact " + url
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if  err != nil {
        msg = "Fail to parse reponse body"
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    ch <- body
}

/*
 * RequestHandler is used by web server to handle incoming requests
 */
func RequestHandler(w http.ResponseWriter, r *http.Request) {
    // we only accept POST request with urls (this is by design)
    if  r.Method != "POST" {
        w.WriteHeader(http.StatusBadRequest)
        return
    }

    // parse input request parameter, in this case we should pass urls
    r.ParseForm()
    urls := []string{}
    for k, v := range r.Form {
        if k == "urls" {
            urls = strings.Split(v[0], "\n")
        }
    }
//    log.Println(urls)

    // create HTTP client
    client := HttpClient()

    // loop concurently over url list and store results into channel
    ch := make(chan []byte)
    n := 0
    for _, url := range urls {
        n++
        go Getdata(client, url, ch)
    }
    // once channels are ready fill out results to response writer
    for i:=0; i<n; i++ {
        w.Write(<-ch)
        w.Write([]byte("\n"))
    }
}

// proxy server. It defines /getdata public interface
func Server(port string) {
    log.Printf("Start server localhost:%s/getdata", port)
    http.HandleFunc("/getdata", RequestHandler)
    err := http.ListenAndServe(":" + port, nil)
    // NOTE: later this can be replaced with secure connection
    // replace ListenAndServe(addr string, handler Handler)
    // with TLS function
    // ListenAndServeTLS(addr string, certFile string, keyFile string, handler
    // Handler)
    if  err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}
<commit_msg>Make global HttpClient and re-use it for all requests; remove client argiment in Getdata; code clean-up<commit_after>/*
 *
 * Author     : <NAME> <<EMAIL>>
 * Description: URL fetch proxy server concurrently fetches data from
 *              provided URL list. It provides a POST HTTP interface
 *              "/getdata" which accepts urls as newline separated encoded
 *              string
 * Created    : Wed Mar 20 13:29:48 EDT 2013
 * License    : MIT
 *
 */
package urlfetch

import (
    "os"
    "log"
    "strings"
    "net/http"
    "io/ioutil"
    "crypto/tls"
)

/*
 * Return array of certificates
 */
func Certs() (tls_certs []tls.Certificate) {
    uproxy := os.Getenv("X509_USER_PROXY")
    uckey  := os.Getenv("X509_USER_KEY")
    ucert  := os.Getenv("X509_USER_CERT")
    log.Println("X509_USER_PROXY", uproxy)
    log.Println("X509_USER_KEY", uckey)
    log.Println("X509_USER_CERT", ucert)
    if  len(uproxy) > 0 {
        x509cert, err := tls.LoadX509KeyPair(uproxy, uproxy)
        if  err != nil {
            log.Println("Fail to parser proxy X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else if len(uckey) > 0 {
        x509cert, err := tls.LoadX509KeyPair(ucert, uckey)
        if  err != nil {
            log.Println("Fail to parser user X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else {
        return
    }
    return
}

/*
 * HTTP client for urlfetch server
 */
func HttpClient() (client *http.Client) {
    // create HTTP client
    certs := Certs()
    if  len(certs) == 0 {
        client = &http.Client{}
        return
    }
    tr := &http.Transport{
        TLSClientConfig: &tls.Config{tls.Certificates: certs},
    }
    client = &http.Client{Transport: tr}
    return
}

// create global HTTP client and re-use it through the code
var client = HttpClient()

/*
 * Getdata(url string, ch chan<- []byte)
 * Fetch data for provided URL and redirect results to given channel
 */
func Getdata(url string, ch chan<- []byte) {
    msg := ""
    resp, err := client.Get(url)
    if  err != nil {
        msg = "Fail to contact " + url
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if  err != nil {
        msg = "Fail to parse reponse body"
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    ch <- body
}

/*
 * RequestHandler is used by web server to handle incoming requests
 */
func RequestHandler(w http.ResponseWriter, r *http.Request) {
    // we only accept POST request with urls (this is by design)
    if  r.Method != "POST" {
        w.WriteHeader(http.StatusBadRequest)
        return
    }

    // parse input request parameter, in this case we should pass urls
    r.ParseForm()
    urls := []string{}
    for k, v := range r.Form {
        if k == "urls" {
            urls = strings.Split(v[0], "\n")
        }
    }
    num := len(urls)
    if  num > 0 {
        first := urls[0]
        last := urls[num-1]
        log.Println("Fetch", num, "URLs:", first, "...", last)
    } else {
        w.Write([]byte("No URLs provided\n"))
        return
    }

    // loop concurently over url list and store results into channel
    ch := make(chan []byte)
    for _, url := range urls {
        go Getdata(url, ch)
    }
    // once channels are ready fill out results to response writer
    for i:=0; i<len(urls); i++ {
        w.Write(<-ch)
        w.Write([]byte("\n"))
    }
}

// proxy server. It defines /getdata public interface
func Server(port string) {
    log.Printf("Start server localhost:%s/getdata", port)
    http.HandleFunc("/getdata", RequestHandler)
    err := http.ListenAndServe(":" + port, nil)
    // NOTE: later this can be replaced with secure connection
    // replace ListenAndServe(addr string, handler Handler)
    // with TLS function
    // ListenAndServeTLS(addr string, certFile string, keyFile string, handler
    // Handler)
    if  err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}

{'fraction_non_alphanumeric': 0.1234812794445822, 'fraction_numerical': 0.015125216960079346, 'mean_word_length': 2.219074221867518, 'pattern_counts': {'":': 2, '<': 22, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3704129', 'n_tokens_mistral': 2701, 'n_tokens_neox': 2570, 'n_words': 758}