<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 <NAME>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"log"
	"math"
)

type PacketConfig struct {
	DataRate                    int
	BlockSize, BlockSize2       int
	SymbolLength, SymbolLength2 int
	SampleRate                  int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	BufferLength                   int
	Preamble                       string
}

func (cfg PacketConfig) Log() {
	log.Println("BlockSize:", cfg.BlockSize)
	log.Println("SampleRate:", cfg.SampleRate)
	log.Println("DataRate:", cfg.DataRate)
	log.Println("SymbolLength:", cfg.SymbolLength)
	log.Println("PreambleSymbols:", cfg.PreambleSymbols)
	log.Println("PreambleLength:", cfg.PreambleLength)
	log.Println("PacketSymbols:", cfg.PacketSymbols)
	log.Println("PacketLength:", cfg.PacketLength)
	log.Println("Preamble:", cfg.Preamble)
}

type Decoder struct {
	cfg PacketConfig

	iq        []byte
	signal    []float64
	quantized []byte

	lut MagnitudeLUT

	preamble []byte
	slices   [][]byte

	pkt []byte
}

func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.cfg = cfg

	d.iq = make([]byte, d.cfg.BufferLength<<1)
	d.signal = make([]float64, d.cfg.BufferLength)
	d.quantized = make([]byte, d.cfg.BufferLength)

	if *fastMag {
		d.lut = NewAlphaMaxBetaMinLUT()
	} else {
		d.lut = NewSqrtMagLUT()
	}

	d.preamble = make([]byte, len(d.cfg.Preamble))
	for idx := range d.cfg.Preamble {
		if d.cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	d.slices = make([][]byte, d.cfg.SymbolLength2)
	flat := make([]byte, d.cfg.BlockSize2-(d.cfg.BlockSize2%d.cfg.SymbolLength2))

	for symbolOffset := range d.slices {
		lower := symbolOffset * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		upper := (symbolOffset + 1) * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		d.slices[symbolOffset] = flat[lower:upper]
	}

	d.pkt = make([]byte, d.cfg.PacketSymbols>>3)

	return
}

func (d Decoder) Decode(input []byte) (pkts [][]byte) {
	// Shift new block into buffers.
	copy(d.iq, d.iq[d.cfg.BlockSize<<1:])
	copy(d.signal, d.signal[d.cfg.BlockSize:])
	copy(d.quantized, d.quantized[d.cfg.BlockSize:])
	copy(d.iq[d.cfg.PacketLength<<1:], input[:])

	iqBlock := d.iq[d.cfg.PacketLength<<1:]
	signalBlock := d.signal[d.cfg.PacketLength:]
	d.lut.Execute(iqBlock, signalBlock)

	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	d.Filter(signalBlock)
	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	Quantize(signalBlock, d.quantized[d.cfg.PacketLength-d.cfg.SymbolLength2:])
	d.Pack(d.quantized[:d.cfg.BlockSize2], d.slices)

	indexes := d.Search(d.slices, d.preamble)

	seen := make(map[string]bool)

	for _, qIdx := range indexes {
		if qIdx > d.cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.quantized[qIdx+(pIdx*d.cfg.SymbolLength2)]
		}

		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}
	return
}

type MagnitudeLUT interface {
	Execute([]byte, []float64)
}

type MagLUT []float64

func NewSqrtMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

func (lut MagLUT) Execute(input []byte, output []float64) {
	for idx := range output {
		lutIdx := idx << 1
		output[idx] = math.Sqrt(lut[input[lutIdx]] + lut[input[lutIdx+1]])
	}
}

type AlphaMaxBetaMinLUT []float64

func NewAlphaMaxBetaMinLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = math.Abs(127.4 - float64(idx))
	}
	return
}

func (lut AlphaMaxBetaMinLUT) Execute(input []byte, output []float64) {
	const (
		α = 0.948059448969
		ß = 0.392699081699
	)

	for idx := range output {
		lutIdx := idx << 1
		i := lut[input[lutIdx]]
		q := lut[input[lutIdx+1]]
		if i > q {
			output[idx] = α*i + ß*q
		} else {
			output[idx] = α*q + ß*i
		}
	}
}

func (d Decoder) Filter(input []float64) {<commit_msg>Whoops, the magic is in the Execute method on AlphaMaxBetaMinLUT, need to return that for it to actually work.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 <NAME>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"log"
	"math"
)

type PacketConfig struct {
	DataRate                    int
	BlockSize, BlockSize2       int
	SymbolLength, SymbolLength2 int
	SampleRate                  int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	BufferLength                   int
	Preamble                       string
}

func (cfg PacketConfig) Log() {
	log.Println("BlockSize:", cfg.BlockSize)
	log.Println("SampleRate:", cfg.SampleRate)
	log.Println("DataRate:", cfg.DataRate)
	log.Println("SymbolLength:", cfg.SymbolLength)
	log.Println("PreambleSymbols:", cfg.PreambleSymbols)
	log.Println("PreambleLength:", cfg.PreambleLength)
	log.Println("PacketSymbols:", cfg.PacketSymbols)
	log.Println("PacketLength:", cfg.PacketLength)
	log.Println("Preamble:", cfg.Preamble)
}

type Decoder struct {
	cfg PacketConfig

	iq        []byte
	signal    []float64
	quantized []byte

	lut MagnitudeLUT

	preamble []byte
	slices   [][]byte

	pkt []byte
}

func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.cfg = cfg

	d.iq = make([]byte, d.cfg.BufferLength<<1)
	d.signal = make([]float64, d.cfg.BufferLength)
	d.quantized = make([]byte, d.cfg.BufferLength)

	if *fastMag {
		d.lut = NewAlphaMaxBetaMinLUT()
	} else {
		d.lut = NewSqrtMagLUT()
	}

	d.preamble = make([]byte, len(d.cfg.Preamble))
	for idx := range d.cfg.Preamble {
		if d.cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	d.slices = make([][]byte, d.cfg.SymbolLength2)
	flat := make([]byte, d.cfg.BlockSize2-(d.cfg.BlockSize2%d.cfg.SymbolLength2))

	for symbolOffset := range d.slices {
		lower := symbolOffset * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		upper := (symbolOffset + 1) * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		d.slices[symbolOffset] = flat[lower:upper]
	}

	d.pkt = make([]byte, d.cfg.PacketSymbols>>3)

	return
}

func (d Decoder) Decode(input []byte) (pkts [][]byte) {
	// Shift new block into buffers.
	copy(d.iq, d.iq[d.cfg.BlockSize<<1:])
	copy(d.signal, d.signal[d.cfg.BlockSize:])
	copy(d.quantized, d.quantized[d.cfg.BlockSize:])
	copy(d.iq[d.cfg.PacketLength<<1:], input[:])

	iqBlock := d.iq[d.cfg.PacketLength<<1:]
	signalBlock := d.signal[d.cfg.PacketLength:]
	d.lut.Execute(iqBlock, signalBlock)

	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	d.Filter(signalBlock)
	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	Quantize(signalBlock, d.quantized[d.cfg.PacketLength-d.cfg.SymbolLength2:])
	d.Pack(d.quantized[:d.cfg.BlockSize2], d.slices)

	indexes := d.Search(d.slices, d.preamble)

	seen := make(map[string]bool)

	for _, qIdx := range indexes {
		if qIdx > d.cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.quantized[qIdx+(pIdx*d.cfg.SymbolLength2)]
		}

		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}
	return
}

type MagnitudeLUT interface {
	Execute([]byte, []float64)
}

type MagLUT []float64

func NewSqrtMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

func (lut MagLUT) Execute(input []byte, output []float64) {
	for idx := range output {
		lutIdx := idx << 1
		output[idx] = math.Sqrt(lut[input[lutIdx]] + lut[input[lutIdx+1]])
	}
}

type AlphaMaxBetaMinLUT []float64

func NewAlphaMaxBetaMinLUT() (lut AlphaMaxBetaMinLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = math.Abs(127.4 - float64(idx))
	}
	return
}

func (lut AlphaMaxBetaMinLUT) Execute(input []byte, output []float64) {
	const (
		α = 0.948059448969
		ß = 0.392699081699
	)

	for idx := range output {
		lutIdx := idx << 1
		i := lut[input[lutIdx]]
		q := lut[input[lutIdx+1]]
		if i > q {
			output[idx] = α*i + ß*q
		} else {
			output[idx] = α*q + ß*i
		}
	}
}

func (d Decoder) Filter(input []float64) {
{'fraction_non_alphanumeric': 0.1485755131216175, 'fraction_numerical': 0.022465026039007456, 'mean_word_length': 3.637310606060606, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7163580', 'n_tokens_mistral': 3983, 'n_tokens_neox': 3688, 'n_words': 880}