<commit_before>iothrottler
===========

A Go package for throttling application IO (such as bandwidth).

Installation
------------

If you have the Go Language installed type
`go get "github.com/efarrer/iothrottler"`

Example
-------
import (
. "github.com/efarrer/iothrottler"
)

/*
 * Throttle web requests using an IOThrottlerPool
 */
func ExampleIOThrottlerPool_AddConn() {
	// Construct a bandwidth throttling pool that's limited to 30 kilobits per
    // second
	pool := NewIOThrottlerPool(Kbps * 30)
	defer pool.ReleasePool()

	// Create our own Dial function that will be used for the http connection
	throttledDial := func(nt, addr string) (c net.Conn, err error) {
		conn, err := net.Dial(nt, addr)
		if err != nil {
			return nil, err
		}

		return pool.AddConn(conn)
	}

	// Create a transport that will use our throttled Dial function
	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial:  throttledDial,
	}

	// Download the page
	client := &http.Client{Transport: tr}
	resp, err := client.Get("http://www.google.com")
	if err != nil {
		// handle error
		return
	}
	defer resp.Body.Close()

	// Read the entire contents of the body
	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		// handle error
		return
	}

	fmt.Println("Done")
	// Output: Done
}<commit_msg>Mark example code as a GFM code block<commit_after>iothrottler
===========

A Go package for throttling application IO (such as bandwidth).

Installation
------------

If you have the Go Language installed type
`go get "github.com/efarrer/iothrottler"`

Example
-------
```go
import (
. "github.com/efarrer/iothrottler"
)

// Throttle web requests using an IOThrottlerPool
func ExampleIOThrottlerPool_AddConn() {
	// Construct a bandwidth throttling pool that's limited to 30 kilobits per
    // second
	pool := NewIOThrottlerPool(Kbps * 30)
	defer pool.ReleasePool()

	// Create our own Dial function that will be used for the http connection
	throttledDial := func(nt, addr string) (c net.Conn, err error) {
		conn, err := net.Dial(nt, addr)
		if err != nil {
			return nil, err
		}

		return pool.AddConn(conn)
	}

	// Create a transport that will use our throttled Dial function
	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial:  throttledDial,
	}

	// Download the page
	client := &http.Client{Transport: tr}
	resp, err := client.Get("http://www.google.com")
	if err != nil {
		// handle error
		return
	}
	defer resp.Body.Close()

	// Read the entire contents of the body
	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		// handle error
		return
	}

	fmt.Println("Done")
	// Output: Done
}
```
{'fraction_non_alphanumeric': 0.12943871706758306, 'fraction_numerical': 0.0030546009927453228, 'mean_word_length': 3.3092105263157894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4950769', 'n_tokens_mistral': 928, 'n_tokens_neox': 876, 'n_words': 278}