<commit_before>package fweight

import (
	"compress/flate"
	"compress/gzip"
	"io"
	"net/http"
	"strings"
)

type Extension interface {
	TransformRequest(rw http.ResponseWriter,
		rq *http.Request) (http.ResponseWriter, *http.Request)
}

type RequestCompleteHooker interface {
	RequestCompleted(rw http.ResponseWriter, rq *http.Request)
}

type IgnorePort struct {
}

func (i IgnorePort) TransformRequest(rw http.ResponseWriter, rq *http.Request) (http.ResponseWriter, *http.Request) {
	if spl := strings.SplitN(rq.Host, ":", 2); len(spl) > 1 {
		rq.Host = spl[0]
	}
	return rw, rq
}

type Compression struct {
	compressor
}

func (c Compression) RequestCompleted(rw http.ResponseWriter, rq *http.Request) {
	rw.(compressionWrap).Close()
}

type compressor interface {
	io.Writer
	Close() error
	Flush() error
}

type compressionWrap struct {
	rw         http.ResponseWriter
	compressor compressor
}

func (c compressionWrap) Close() {
	c.compressor.Flush()
	c.compressor.Close()
}

func (c compressionWrap) Write(b []byte) (n int, err error) {

	var bt = b
	var cnt int = 0
	for ; cnt < len(b); bt = bt[cnt:] {
		var thisW int
		thisW, err = c.compressor.Write(bt)
		cnt += thisW
		if err != nil {
			return cnt, err
		}
	}

	return cnt, err
}

func (c compressionWrap) Header() http.Header {
	return c.rw.Header()
}

func (c compressionWrap) WriteHeader(an int) {
	c.rw.WriteHeader(an)
}
<commit_msg>Fixed bug in compression that would cause a nil pointer dereference<commit_after>package fweight

import (
	"compress/flate"
	"compress/gzip"
	"io"
	"net/http"
	"strings"
)

type Extension interface {
	TransformRequest(rw http.ResponseWriter,
		rq *http.Request) (http.ResponseWriter, *http.Request)
}

type RequestCompleteHooker interface {
	RequestCompleted(rw http.ResponseWriter, rq *http.Request)
}

type IgnorePort struct {
}

func (i IgnorePort) TransformRequest(rw http.ResponseWriter, rq *http.Request) (http.ResponseWriter, *http.Request) {
	if spl := strings.SplitN(rq.Host, ":", 2); len(spl) > 1 {
		rq.Host = spl[0]
	}
	return rw, rq
}

type Compression struct {
	compressor
}

func (c Compression) RequestCompleted(rw http.ResponseWriter, rq *http.Request) {
	if l, ok := rw.(compressionWrap); ok{
		l.Close()
	}
}

type compressor interface {
	io.Writer
	Close() error
	Flush() error
}

type compressionWrap struct {
	rw         http.ResponseWriter
	compressor compressor
}

func (c compressionWrap) Close() {
	if c.compressor != nil {
		c.compressor.Flush()
		c.compressor.Close()
	}
}

func (c compressionWrap) Write(b []byte) (n int, err error) {

	var bt = b
	var cnt int = 0
	for ; cnt < len(b); bt = bt[cnt:] {
		var thisW int
		thisW, err = c.compressor.Write(bt)
		cnt += thisW
		if err != nil {
			return cnt, err
		}
	}

	return cnt, err
}

func (c compressionWrap) Header() http.Header {
	return c.rw.Header()
}

func (c compressionWrap) WriteHeader(an int) {
	c.rw.WriteHeader(an)
}

{'fraction_non_alphanumeric': 0.12813466162830642, 'fraction_numerical': 0.0027481964960494676, 'mean_word_length': 4.020689655172414, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3975585', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1049, 'n_words': 280}