<commit_before>package marketassembley

import (
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"reflect"
	"fmt"
	"regexp"
	"encoding/binary"
	"strings"
)

/*
19    -> Indicator of if there is more packets
20:24 -> Packet ID
24:28 -> ID of first packet in the bundle
28:32 -> Expected number of packets
32:36 -> Packet ID in bundle
44:54 -> Location of market response start indicator
44:   -> Data when not first packet of market response
55:   -> Data when first packet of market response
 */


var marketStartIndicator = []byte{243, 3, 1, 0, 0, 42, 0, 2, 0, 121}
var morePacketsIndicator = byte(164)

type MarketAssembler struct {
	processing bool
	itemCount int
	itemsBuffer []byte

}

func NewMarketAssembler() *MarketAssembler {
	return &MarketAssembler{}
}

func (ma *MarketAssembler) ProcessPacket(packet gopacket.Packet) {
	udp := packet.Layer(layers.LayerTypeUDP)
	if udp != nil {
		udp := udp.(*layers.UDP)

		if len(udp.Payload) < 56 {
			return
		}

		marketHeader := udp.Payload[44:54]

		if !ma.processing && reflect.DeepEqual(marketHeader, marketStartIndicator) {
			ma.itemCount = int(binary.BigEndian.Uint16(udp.Payload[54:56]))


			ma.processing = true
			ma.itemsBuffer = nil
			ma.itemsBuffer = append(ma.itemsBuffer, udp.Payload[59:]...)

		} else if ma.processing {
			ma.itemsBuffer = append(ma.itemsBuffer, udp.Payload[44:]...)

			if udp.Payload[19] != morePacketsIndicator {
				ma.processing = false
				results := extractStrings(ma.itemsBuffer)
				fmt.Println(results)
				fmt.Println(len(results))
				fmt.Println(ma.itemCount)

				ma.itemsBuffer = nil
			}
		}
	}
}

func extractStrings(payload []byte) []string {
	var results []string
	r, _ := regexp.Compile("\\{[^\\{\\}]*\\}")

	for _, match :=  range r.FindAllStringSubmatch(string(payload), -1) {
		results = append(results, strings.Join(match, ""))
	}

	return results
}<commit_msg>Some final tweaking and minor cleanup<commit_after>package marketassembley

import (
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"reflect"
	"fmt"
	"regexp"
	"encoding/binary"
	"strings"
)

/*
19    -> Location of more packets indicator
20:24 -> Packet ID
24:28 -> ID of first packet in the bundle
28:32 -> Expected number of packets
32:36 -> Packet ID in bundle
44:   -> Data when not first packet of market response
44:54 -> Location of market response start indicator
54:56 -> Number of market items to be expected in total
56:   -> Data when first packet of market response
 */


var marketStartIndicator = []byte{243, 3, 1, 0, 0, 42, 0, 2, 0, 121}
var morePacketsIndicator = byte(164)

type MarketAssembler struct {
	itemCount int
	packetCount int
	processedCount int
	itemsBuffer []byte
	processing bool

}

func NewMarketAssembler() *MarketAssembler {
	return &MarketAssembler{}
}

func (ma *MarketAssembler) ProcessPacket(packet gopacket.Packet) {
	udp := packet.Layer(layers.LayerTypeUDP)
	if udp != nil {
		udp := udp.(*layers.UDP)

		if len(udp.Payload) < 56 {
			return
		}

		morePackets := udp.Payload[19]
		currentPacket := int(binary.BigEndian.Uint32(udp.Payload[32:36]))
		marketHeader := udp.Payload[44:54]

		if reflect.DeepEqual(marketHeader, marketStartIndicator) {
			fmt.Println("Start of market")
			// Ensure things are reset
			ma.processing = true
			ma.itemsBuffer = nil
			ma.packetCount = 0

			ma.itemCount = int(binary.BigEndian.Uint16(udp.Payload[54:56]))

			// Minus 1 because packet counts themselves start at zero
			ma.packetCount = int(binary.BigEndian.Uint32(udp.Payload[28:32])) - 1

			ma.itemsBuffer = append(ma.itemsBuffer, udp.Payload[59:]...)
		} else if !ma.processing {
			// We are not processing so may as well ignore! :D
			return
		} else if morePackets != morePacketsIndicator && currentPacket != ma.packetCount {
			// If we don't have the more packets indicator AND the current packet
			// doesn't have a packet count matching the number of packets we expect
			// move on, possibly got some other UDP packet we don't care about.
			return
		} else if morePackets != morePacketsIndicator && currentPacket == ma.packetCount {
			// There are no more packets and this is confirmed as the last packet
			ma.itemsBuffer = append(ma.itemsBuffer, udp.Payload[44:]...)

			results := extractStrings(ma.itemsBuffer)
			fmt.Println(results)
			fmt.Println(len(results))
			fmt.Println(ma.itemCount)

			ma.processing = false
		} else {
			ma.itemsBuffer = append(ma.itemsBuffer, udp.Payload[44:]...)
		}
	}
}

func extractStrings(payload []byte) []string {
	var results []string
	r, _ := regexp.Compile("\\{[^\\{\\}]*\\}")

	for _, match :=  range r.FindAllStringSubmatch(string(payload), -1) {
		results = append(results, strings.Join(match, ""))
	}

	return results
}
{'fraction_non_alphanumeric': 0.12679628064243448, 'fraction_numerical': 0.03106508875739645, 'mean_word_length': 3.8050761421319796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4011997', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1558, 'n_words': 433}