<commit_before>package main

import (
  "errors"

  "github.com/google/gopacket"
  "github.com/google/gopacket/layers"

  "golang.org/x/net/ipv4"
//  "golang.org/x/net/ipv6"

  "net"
)

func CreateStream(destiantion string) chan []byte {
  flow := make(chan []byte)
  return flow
}

var (
  packet4Conn net.PacketConn
  raw4Conn *ipv4.RawConn
  ipv4Layer layers.IPv4
  ipv4Parser *gopacket.DecodingLayerParser
)

func SetupSockets() {
  var err error
  ipv4Parser = gopacket.NewDecodingLayerParser(layers.LayerTypeIPv4, &ipv4Layer)

  packet4Conn, err = net.ListenPacket("ipv4", "127.0.0.1")
  if err != nil {
    panic(err)
  }
  raw4Conn, err = ipv4.NewRawConn(packet4Conn)
  if err != nil {<commit_msg>connect up channel & threads to processing.<commit_after>package main

import (
  "errors"

  "github.com/google/gopacket"
  "github.com/google/gopacket/layers"

  "golang.org/x/net/ipv4"
//  "golang.org/x/net/ipv6"

  "net"
)

func CreateStream(destination string) chan []byte {
  if packet4Conn == nil {
    SetupSockets()
  }

  dest := net.ParseIP(destination)
  flow := make(chan []byte)
  go HandleStream(dest, flow)
  return flow
}

func HandleStream(dest net.IP, que chan []byte) {
  for {
    req := <-que
    ConditionalForward(req, dest)
  }
}

var (
  packet4Conn net.PacketConn
  raw4Conn *ipv4.RawConn
  ipv4Layer layers.IPv4
  ipv4Parser *gopacket.DecodingLayerParser
)

func SetupSockets() {
  var err error
  ipv4Parser = gopacket.NewDecodingLayerParser(layers.LayerTypeIPv4, &ipv4Layer)

  packet4Conn, err = net.ListenPacket("ipv4", "127.0.0.1")
  if err != nil {
    panic(err)
  }
  raw4Conn, err = ipv4.NewRawConn(packet4Conn)
  if err != nil {
{'fraction_non_alphanumeric': 0.1286654697785757, 'fraction_numerical': 0.026929982046678635, 'mean_word_length': 3.75, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1599564', 'n_tokens_mistral': 665, 'n_tokens_neox': 649, 'n_words': 125}