<commit_before>package plaintext

import (
	"bytes"
	"strings"
	"text/scanner"
)

// GolangText extracts plaintext from Golang and other similar C or Java like files
type GolangText struct {
}

// NewGolangText creates a new extractor
func NewGolangText() (*GolangText, error) {
	return &GolangText{}, nil
}

// Text satifies the Extractor interface
//
//ReplaceGo is a specialized routine for correcting Golang source
// files.  Currently only checks comments, not identifiers for
// spelling.
//
// Other items:
//   - check strings, but need to ignore
//      * import "statements" blocks
//      * import ( "blocks" )
//   - skip first comment (line 0) if build comment
//
func (p *GolangText) Text(raw []byte) []byte {
	out := bytes.Buffer{}
	var s scanner.Scanner
	s.Init(strings.NewReader(string(raw)))
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	for {
		switch s.Scan() {
		case scanner.Comment:
			out.WriteString(s.TokenText())
			out.WriteByte('\n')
		case scanner.EOF:
			return out.Bytes()
		}
	}
}
<commit_msg>simplify<commit_after>package plaintext

import (
	"bytes"
	"text/scanner"
)

// GolangText extracts plaintext from Golang and other similar C or Java like files
type GolangText struct {
}

// NewGolangText creates a new extractor
func NewGolangText() (*GolangText, error) {
	return &GolangText{}, nil
}

// Text satifies the Extractor interface
//
//ReplaceGo is a specialized routine for correcting Golang source
// files.  Currently only checks comments, not identifiers for
// spelling.
//
// Other items:
//   - check strings, but need to ignore
//      * import "statements" blocks
//      * import ( "blocks" )
//   - skip first comment (line 0) if build comment
//
func (p *GolangText) Text(raw []byte) []byte {
	out := bytes.Buffer{}
	s := scanner.Scanner{}
	s.Init(bytes.NewReader(raw))
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	for {
		switch s.Scan() {
		case scanner.Comment:
			out.WriteString(s.TokenText())
			out.WriteByte('\n')
		case scanner.EOF:
			return out.Bytes()
		}
	}
}

{'fraction_non_alphanumeric': 0.12178619756427606, 'fraction_numerical': 0.0009021199819576004, 'mean_word_length': 3.5544147843942504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3970300', 'n_tokens_mistral': 759, 'n_tokens_neox': 703, 'n_words': 214}