<commit_before>// We handle the checking as a lexing program.
// We use the lexer like Rob Pike lectures about in this
// clip: http://www.youtube.com/watch?v=HxaD_trXwRE

package main

import (
	"dns"
	"fmt"
)

type itemType int

type item struct {
	typ itemType
	val string
}

const (
	itemError        itemType = iota
	itemSoftware              // the name of the DNS server software
	itemVendor                // vendor of the DNS software
	itemVersionMinor          // the minor version of the software (empty if not determined)
	itemVersionMajor          // the major version of the software (empty if not determined)
	itemVersionPatch          // the patch level of the software (empty if not determined)
)

// stateFn represents the state of the scanner as a function that returns the next state.
type stateFn func(*lexer) stateFn

type lexer struct {
	client *dns.Client  // client used.
	addr   string       // addr of the server being scanned.
	fp     *fingerprint // fingerprint to test.
	q      dns.Question // question to ask.
	items  chan item    // channel of scanned items.
	state  stateFn      // the next function to enter.
	debug  bool         // if true, the fingerprints are printed.
}

func (l *lexer) probe() *fingerprint {
	f := sendProbe(l.client, l.addr, l.fp, l.q)
	if l.debug {
		fmt.Printf("      QR fp: %s\n", f)
	}
	return f
}

func (l *lexer) emit(i *item) {<commit_msg>better ordering<commit_after>// We handle the checking as a lexing program.
// We use the lexer like Rob Pike lectures about in this
// clip: http://www.youtube.com/watch?v=HxaD_trXwRE

package main

import (
	"dns"
	"fmt"
)

type itemType int

type item struct {
	typ itemType
	val string
}

const (
	itemError        itemType = iota
	itemSoftware              // the name of the DNS server software
	itemVendor                // vendor of the DNS software
	itemVersionMajor          // the major version of the software (empty if not determined)
	itemVersionMinor          // the minor version of the software (empty if not determined)
	itemVersionPatch          // the patch level of the software (empty if not determined)
)

// stateFn represents the state of the scanner as a function that returns the next state.
type stateFn func(*lexer) stateFn

type lexer struct {
	client *dns.Client  // client used.
	addr   string       // addr of the server being scanned.
	fp     *fingerprint // fingerprint to test.
	q      dns.Question // question to ask.
	items  chan item    // channel of scanned items.
	state  stateFn      // the next function to enter.
	debug  bool         // if true, the fingerprints are printed.
}

func (l *lexer) probe() *fingerprint {
	f := sendProbe(l.client, l.addr, l.fp, l.q)
	if l.debug {
		fmt.Printf("      QR fp: %s\n", f)
	}
	return f
}

func (l *lexer) emit(i *item) {
{'fraction_non_alphanumeric': 0.08404558404558404, 'fraction_numerical': 0.0, 'mean_word_length': 2.520050125313283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4778992', 'n_tokens_mistral': 887, 'n_tokens_neox': 849, 'n_words': 357}