<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"net/http"
	"regexp"
)

// FilteredWriter is a custom writer
// that filter deprecation warnings and juju log output.
type FilteredWriter struct {
	writer http.ResponseWriter
}

// WriteHeader calls the w.Header
func (w *FilteredWriter) Header() http.Header {
	return w.writer.Header()
}

// Write writes the data, filtering the juju warnings.
func (w *FilteredWriter) Write(data []byte) (int, error) {
	_, err := w.writer.Write(filterOutput(data))
	// returning the len(data) to skip the 'short write' error
	return len(data), err
}

// WriteHeader calls the w.WriteHeader
func (w *FilteredWriter) WriteHeader(code int) {
	w.writer.WriteHeader(code)
}

// filterOutput filters output from juju.
//
// It removes all lines that does not represent useful output, like juju's
// logging and Python's deprecation warnings.
func filterOutput(output []byte) []byte {
	var result [][]byte
	var ignore bool
	deprecation := []byte("DeprecationWarning")<commit_msg>flushing the data in FilteredWriter<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"net/http"
	"regexp"
)

// FilteredWriter is a custom writer
// that filter deprecation warnings and juju log output.
type FilteredWriter struct {
	writer http.ResponseWriter
}

// WriteHeader calls the w.Header
func (w *FilteredWriter) Header() http.Header {
	return w.writer.Header()
}

// Write writes and flushes the data, filtering the juju warnings.
func (w *FilteredWriter) Write(data []byte) (int, error) {
	_, err := w.writer.Write(filterOutput(data))
	if f, ok := w.writer.(http.Flusher); ok {
		f.Flush()
	}
	// returning the len(data) to skip the 'short write' error
	return len(data), err
}

// WriteHeader calls the w.WriteHeader
func (w *FilteredWriter) WriteHeader(code int) {
	w.writer.WriteHeader(code)
}

// filterOutput filters output from juju.
//
// It removes all lines that does not represent useful output, like juju's
// logging and Python's deprecation warnings.
func filterOutput(output []byte) []byte {
	var result [][]byte
	var ignore bool
	deprecation := []byte("DeprecationWarning")
{'fraction_non_alphanumeric': 0.10918114143920596, 'fraction_numerical': 0.0033085194375516956, 'mean_word_length': 4.125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5795346', 'n_tokens_mistral': 750, 'n_tokens_neox': 729, 'n_words': 281}