<commit_before>package main

import (
	"math"

	"bytes"

	"strconv"

	"github.com/mjibson/go-dsp/spectral"
)

// Rasterizer fills one or more buffers with discreet audio samples
type Rasterizer func([][]float64)

// RenderMono rasterizes a TFunc into a mono Portaudio channel
func RenderMono(tfunc TFunc) Rasterizer {
	var t uint64
	return func(out [][]float64) {
		for i := range out[0] {
			var tc = Timecode(t + uint64(i))
			out[0][i] = float64(tfunc(tc))
		}
		t = t + uint64(len(out[0]))
	}
}

// RenderStereo rasterizes two TFuncs into a stereo Portaudio channel
func RenderStereo(left TFunc, right TFunc) Rasterizer {
	var t uint64
	return func(out [][]float64) {
		for i := range out[0] {
			var tc = Timecode(t + uint64(i))
			out[0][i] = float64(left(tc))
			out[1][i] = float64(right(tc))
		}
		t = t + uint64(len(out[0]))
	}
}

// Visualize renders a Rasterizer's power spectral density to the screen
func Visualize(sampleRate Frequency, colormap Colormap, inner Rasterizer) Rasterizer {
	lowerAudible := Hz(20)
	upperAudible := KHz(20)

	segmentLength := 256

	// stores the rendered output
	outLine := bytes.NewBuffer(make([]byte, 30*segmentLength, 30*segmentLength))
	var r, g, b byte

	options := new(spectral.PwelchOptions)
	options.Scale_off = true
	options.Pad = segmentLength

	return func(out [][]float64) {
		// compute raster values from the inner rasterizer
		inner(out)

		// divide the rasterized sound into [segmentLength] sample segments
		var segments [][]float64
		for _, chanBuf := range out {
			chanSegs := spectral.Segment(chanBuf, segmentLength, 0)
			segments = append(segments, chanSegs...)
		}

		// compute the averge PSD for all segments
		psd := make([]float64, options.Pad/2+1) // output of FFT is len(time domain)/2+1
		var freqs []float64
		for _, seg := range segments {
			var pxx []float64
			// compute the PSD for each segment
			pxx, freqs = spectral.Pwelch(seg, float64(sampleRate), options)
			for i := range pxx {
				psd[i] += pxx[i]
			}
			// println()
		}

		// render samples to the output buffer
		outLine.Reset()
		for i := range psd {
			// only render samples in the audible range
			if float64(lowerAudible) < freqs[i] && freqs[i] < float64(upperAudible) {
				// divide by number of segments to get average
				p := psd[i] / float64(len(segments))
				// linearize the PSD
				p = p * math.Sqrt(freqs[i]) * 0.5

				// render a 'pixel'
				r, g, b = colormap(p)
				outLine.WriteString("\x1b[48;2;")
				outLine.WriteString(strconv.Itoa(int(r)))
				outLine.WriteString(";")
				outLine.WriteString(strconv.Itoa(int(g)))
				outLine.WriteString(";")
				outLine.WriteString(strconv.Itoa(int(b)))
				outLine.WriteString("m \x1b[0m")
			}
		}
		println(outLine.String())
	}
}

// func printColor(value float64, colormap Colormap) {
// 	r, g, b := colormap(value)
// 	fmt.Printf("\x1b[48;2;%d;%d;%dm \x1b[0m", r, g, b)
// }<commit_msg>Raster output performance WIP<commit_after>package main

import (
	"bytes"
	"fmt"
	"math"
	"strconv"

	"github.com/mjibson/go-dsp/spectral"
)

// Rasterizer fills one or more buffers with discreet audio samples
type Rasterizer func([][]float64)

// RenderMono rasterizes a TFunc into a mono Portaudio channel
func RenderMono(tfunc TFunc) Rasterizer {
	var t uint64
	return func(out [][]float64) {
		for i := range out[0] {
			var tc = Timecode(t + uint64(i))
			out[0][i] = float64(tfunc(tc))
		}
		t = t + uint64(len(out[0]))
	}
}

// RenderStereo rasterizes two TFuncs into a stereo Portaudio channel
func RenderStereo(left TFunc, right TFunc) Rasterizer {
	var t uint64
	return func(out [][]float64) {
		for i := range out[0] {
			var tc = Timecode(t + uint64(i))
			out[0][i] = float64(left(tc))
			out[1][i] = float64(right(tc))
		}
		t = t + uint64(len(out[0]))
	}
}

// Visualize renders a Rasterizer's power spectral density to the screen
func Visualize(sampleRate Frequency, colormap Colormap, inner Rasterizer) Rasterizer {
	lowerAudible := Hz(20)
	upperAudible := KHz(20)

	segmentLength := 256

	// stores the rendered output
	lineQueue := make(chan *bytes.Buffer, 16)
	go printer(lineQueue)

	options := new(spectral.PwelchOptions)
	options.Scale_off = true
	options.Pad = segmentLength

	return func(out [][]float64) {
		// compute raster values from the inner rasterizer
		inner(out)

		// divide the rasterized sound into [segmentLength] sample segments
		var segments [][]float64
		for _, chanBuf := range out {
			chanSegs := spectral.Segment(chanBuf, segmentLength, 0)
			segments = append(segments, chanSegs...)
		}

		// compute the averge PSD for all segments
		psd := make([]float64, options.Pad/2+1) // output of FFT is len(time domain)/2+1
		var freqs []float64
		for _, seg := range segments {
			var pxx []float64
			// compute the PSD for each segment
			pxx, freqs = spectral.Pwelch(seg, float64(sampleRate), options)
			for i := range pxx {
				psd[i] += pxx[i]
			}
			// println()
		}

		// normalize the PSD
		for i := range psd {
			// only render samples in the audible range
			if float64(lowerAudible) < freqs[i] && freqs[i] < float64(upperAudible) {
				// divide by number of segments to get average
				psd[i] /= float64(len(segments))
				// linearize the PSD
				psd[i] *= math.Sqrt(freqs[i]) * 0.5
			}
		}

		// render a line
		lineQueue <- printColor(psd, colormap)
	}
}

func printColor(psd []float64, colormap Colormap) *bytes.Buffer {
	outLine := bytes.NewBuffer(make([]byte, 30*len(psd), 30*len(psd)))
	for i := range psd {
		r, g, b := colormap(psd[i])
		outLine.WriteString("\x1b[48;2;")
		outLine.WriteString(strconv.Itoa(int(r)))
		outLine.WriteString(";")
		outLine.WriteString(strconv.Itoa(int(g)))
		outLine.WriteString(";")
		outLine.WriteString(strconv.Itoa(int(b)))
		outLine.WriteString("m \x1b[0m")
	}
	return outLine
}

func printer(lineQueue chan *bytes.Buffer) {
	for {
		buffer := <-lineQueue
		fmt.Println(buffer.String())
	}
}
{'fraction_non_alphanumeric': 0.139704634187744, 'fraction_numerical': 0.026820573756577832, 'mean_word_length': 3.6724821570182393, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '552788', 'n_tokens_mistral': 2466, 'n_tokens_neox': 2163, 'n_words': 565}