<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"math"
	"os"
	"runtime"
	"strings"
)

type textdata struct {
	rawdata []string
	data    []string
	freq    map[string]int
	total   int
}

func (t *textdata) input(f io.Reader) {
	scanner := bufio.NewScanner(f)

	if t.data == nil {
		t.rawdata = make([]string, 0)
	}

	for scanner.Scan() {
		someline := scanner.Text()
		t.rawdata = append(t.rawdata, someline)
	}
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func parse(lines []string) <-chan string {
	out := make(chan string, 8*100)

	go func() {
		for _, line := range lines {
			if len(line) > 0 {
				out <- line
			}
		}

		close(out)
	}()

	return out
}

func parselower(in <-chan string) <-chan string {
	out := make(chan string, cap(in))

	go func() {
		for line := range in {
			out <- strings.ToLower(line)
		}
		close(out)
	}()

	return out
}

func parsesplit(in <-chan string) <-chan string {
	out := make(chan string, cap(in))

	go func() {
		for line := range in {
			grams := strings.Split(line, " ")
			var prev string
			for _, gram := range grams {
				if len(gram) > 0 {
					out <- (prev + gram)
				}
				prev = gram
			}
		}
		close(out)
	}()

	return out
}

func (t *textdata) resulter(in <-chan string) {
	if t.freq == nil {
		t.freq = make(map[string]int)
	}

	for gram := range in {
		t.total += 1
		t.freq[gram] += 1
	}
}

func (t *textdata) lexd() float64 {
	return float64(len(t.freq)) / float64(t.total)
}

func (t *textdata) tf(term string) float64 {
	return float64(t.freq[term]) / float64(t.total)
}

func idf(textdatum []textdata, term string) float64 {
	count := 0
	for _, t := range textdatum {
		if t.freq[term] != 0 {
			count += 1
		}
	}

	if count == 0 {
		return 0.0
	}

	return 1.0 + math.Log(float64(len(textdatum))/float64(count))
}

func tf_idf(t textdata, textdatum []textdata, term string) float64 {
	return t.tf(term) * idf(textdatum, term)
}

func main() {
	t := make([]textdata, len(os.Args[1:]))

	for index, args := range os.Args[1:] {
		f, err := os.Open(args)

		if err != nil {
			fmt.Println(err)
		}

		defer f.Close()

		t[index].input(f)

		parser := parse(t[index].rawdata)
		parselower := parselower(parser)
		parsespliter := parsesplit(parselower)
		t[index].resulter(parsespliter)
	}

	for _, td := range t {
		fmt.Println(tf_idf(td, t, "for"))
	}
}<commit_msg>vacation<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"math"
	"os"
	"runtime"
	"strings"
)

type textdata struct {
	rawdata []string
	data    []string
	freq    map[string]int
	total   int
}

func (t *textdata) input(f io.Reader) {
	scanner := bufio.NewScanner(f)

	if t.data == nil {
		t.rawdata = make([]string, 0)
	}

	for scanner.Scan() {
		someline := scanner.Text()
		t.rawdata = append(t.rawdata, someline)
	}
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func parse(lines []string) <-chan string {
	out := make(chan string, 8*100)

	go func() {
		for _, line := range lines {
			if len(line) > 0 {
				out <- line
			}
		}

		close(out)
	}()

	return out
}

func parselower(in <-chan string) <-chan string {
	out := make(chan string, cap(in))

	go func() {
		for line := range in {
			out <- strings.ToLower(line)
		}
		close(out)
	}()

	return out
}

func parsesplit(in <-chan string) <-chan string {
	out := make(chan string, cap(in))

	go func() {
		for line := range in {
			grams := strings.Split(line, " ")
			var prev string
			for _, gram := range grams {
				if len(gram) > 0 {
					out <- (prev + gram)
				}
				prev = gram
			}
		}
		close(out)
	}()

	return out
}

func (t *textdata) resulter(in <-chan string) {
	if t.freq == nil {
		t.freq = make(map[string]int)
	}

	for gram := range in {
		t.total += 1
		t.freq[gram] += 1
	}
}

func (t *textdata) lexd() float64 {
	return float64(len(t.freq)) / float64(t.total)
}

func (t *textdata) tf(term string) float64 {
	return float64(t.freq[term]) / float64(t.total)
}

func idf(textdatum []textdata, term string) float64 {
	count := 0
	for _, t := range textdatum {
		if t.freq[term] != 0 {
			count += 1
		}
	}

	if count == 0 {
		return 0.0
	}

	return 1.0 + math.Log(float64(len(textdatum))/float64(count))
}

func tf_idf(t textdata, textdatum []textdata, term string) float64 {
	return t.tf(term) * idf(textdatum, term)
}

func main() {
	t := make([]textdata, len(os.Args[1:]))

	for index, args := range os.Args[1:] {
		f, err := os.Open(args)

		if err != nil {
			fmt.Println(err)
		}

		defer f.Close()

		t[index].input(f)

		parser := parse(t[index].rawdata)
		parselower := parselower(parser)
		parsespliter := parsesplit(parselower)
		t[index].resulter(parsespliter)
	}

	for _, td := range t {
		fmt.Println(tf_idf(td, t, "for"))
		fmt.Println(tf_idf(td, t, "a"))
		fmt.Println(tf_idf(td, t, "i"))
	}
}
{'fraction_non_alphanumeric': 0.1698549505991171, 'fraction_numerical': 0.016396888795459322, 'mean_word_length': 3.0701454234388366, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1939226', 'n_tokens_mistral': 2179, 'n_tokens_neox': 1938, 'n_words': 468}