<commit_before>package mrt

import (
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	monpb "github.com/CSUNetSec/netsec-protobufs/bgpmon"
	pbcom "github.com/CSUNetSec/netsec-protobufs/common"
	pbbgp "github.com/CSUNetSec/netsec-protobufs/protocol/bgp"
	"github.com/CSUNetSec/protoparse"
	pp "github.com/CSUNetSec/protoparse"
	bgp "github.com/CSUNetSec/protoparse/protocol/bgp"
	rib "github.com/CSUNetSec/protoparse/protocol/rib"
	util "github.com/CSUNetSec/protoparse/util"
	"net"
	"time"
)

const (
	MRT_HEADER_LEN    = 12
	BGP4MP            = 16
	BGP4MP_ET         = 17
	MESSAGE           = 1
	MESSAGE_AS4       = 4
	MESSAGE_LOCAL     = 7
	MESSAGE_AS4_LOCAL = 7
	TABLE_DUMP        = 12
	TABLE_DUMP_V2     = 13
	PEER_INDEX_TABLE  = 1
)

type MrtBufferStack struct {
	MrthBuf   protoparse.PbVal `json:"mrt_header,omitempty"`
	Bgp4mpbuf protoparse.PbVal `json:"bgp4mp_header,omitempty"`
	Bgphbuf   protoparse.PbVal `json:"bgp_header,omitempty"`
	Bgpupbuf  protoparse.PbVal `json:"bgp_update,omitempty"`

	Ribbuf protoparse.PbVal `json:"rib_entry,omitempty"`
}

func (mbs *MrtBufferStack) GetRawMessage() []byte {
	return mbs.MrthBuf.(*mrtHhdrBuf).buf
}

func (mbs *MrtBufferStack) IsRibStack() bool {
	return mbs.Ribbuf != nil
}

func MrtToBGPCapture(data []byte) (*monpb.BGPCapture, error) {
	mrth := NewMrtHdrBuf(data)
	bgp4h, errmrt := mrth.Parse()
	if errmrt != nil {
		return nil, fmt.Errorf("Failed parsing MRT header:%s\n", errmrt)
	}
	bgph, errbgph := bgp4h.Parse()
	if errbgph != nil {
		return nil, fmt.Errorf("Failed parsing BGP4MP header:%s\n", errbgph)
	}
	bgpup, errbgpup := bgph.Parse()
	if errbgpup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Header:%s\n", errbgpup)
	}
	_, errup := bgpup.Parse()
	if errup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Update:%s\n", errup)
	}
	capture := new(monpb.BGPCapture)
	bgphpb := bgp4h.(pp.BGP4MPHeaderer).GetHeader()
	mrtpb := mrth.GetHeader()
	capture.Timestamp = mrtpb.Timestamp
	capture.PeerAs = bgphpb.PeerAs
	capture.LocalAs = bgphpb.LocalAs
	capture.InterfaceIndex = bgphpb.InterfaceIndex
	capture.AddressFamily = bgphpb.AddressFamily
	capture.PeerIp = bgphpb.PeerIp
	capture.LocalIp = bgphpb.LocalIp
	capture.Update = bgpup.(pp.BGPUpdater).GetUpdate()
	return capture, nil
}
<commit_msg>creating an mrt to bgpcaturev2 function until the interface stabilises<commit_after>package mrt

import (
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	monpb "github.com/CSUNetSec/netsec-protobufs/bgpmon"
	monpb2 "github.com/CSUNetSec/netsec-protobufs/bgpmon/v2"
	pbcom "github.com/CSUNetSec/netsec-protobufs/common"
	pbbgp "github.com/CSUNetSec/netsec-protobufs/protocol/bgp"
	"github.com/CSUNetSec/protoparse"
	pp "github.com/CSUNetSec/protoparse"
	bgp "github.com/CSUNetSec/protoparse/protocol/bgp"
	rib "github.com/CSUNetSec/protoparse/protocol/rib"
	util "github.com/CSUNetSec/protoparse/util"
	"net"
	"time"
)

const (
	MRT_HEADER_LEN    = 12
	BGP4MP            = 16
	BGP4MP_ET         = 17
	MESSAGE           = 1
	MESSAGE_AS4       = 4
	MESSAGE_LOCAL     = 7
	MESSAGE_AS4_LOCAL = 7
	TABLE_DUMP        = 12
	TABLE_DUMP_V2     = 13
	PEER_INDEX_TABLE  = 1
)

func MrtToBGPCapture(data []byte) (*monpb.BGPCapture, error) {
	mrth := NewMrtHdrBuf(data)
	bgp4h, errmrt := mrth.Parse()
	if errmrt != nil {
		return nil, fmt.Errorf("Failed parsing MRT header:%s\n", errmrt)
	}
	bgph, errbgph := bgp4h.Parse()
	if errbgph != nil {
		return nil, fmt.Errorf("Failed parsing BGP4MP header:%s\n", errbgph)
	}
	bgpup, errbgpup := bgph.Parse()
	if errbgpup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Header:%s\n", errbgpup)
	}
	_, errup := bgpup.Parse()
	if errup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Update:%s\n", errup)
	}
	capture := new(monpb.BGPCapture)
	bgphpb := bgp4h.(pp.BGP4MPHeaderer).GetHeader()
	mrtpb := mrth.GetHeader()
	capture.Timestamp = mrtpb.Timestamp
	capture.PeerAs = bgphpb.PeerAs
	capture.LocalAs = bgphpb.LocalAs
	capture.InterfaceIndex = bgphpb.InterfaceIndex
	capture.AddressFamily = bgphpb.AddressFamily
	capture.PeerIp = bgphpb.PeerIp
	capture.LocalIp = bgphpb.LocalIp
	capture.Update = bgpup.(pp.BGPUpdater).GetUpdate()
	return capture, nil
}

func MrtToBGPCapturev2(data []byte) (*monpb2.BGPCapture, error) {
	mrth := NewMrtHdrBuf(data)
	bgp4h, errmrt := mrth.Parse()
	if errmrt != nil {
		return nil, fmt.Errorf("Failed parsing MRT header:%s\n", errmrt)
	}
	bgph, errbgph := bgp4h.Parse()
	if errbgph != nil {
		return nil, fmt.Errorf("Failed parsing BGP4MP header:%s\n", errbgph)
	}
	bgpup, errbgpup := bgph.Parse()
	if errbgpup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Header:%s\n", errbgpup)
	}
	_, errup := bgpup.Parse()
	if errup != nil {
		return nil, fmt.Errorf("Failed parsing BGP Update:%s\n", errup)
	}
	capture := new(monpb2.BGPCapture)
	bgphpb := bgp4h.(pp.BGP4MPHeaderer).GetHeader()
	mrtpb := mrth.GetHeader()
	capture.Timestamp = mrtpb.Timestamp
	capture.PeerAs = bgphpb.PeerAs
	capture.LocalAs = bgphpb.LocalAs
	capture.InterfaceIndex = bgphpb.InterfaceIndex
	capture.AddressFamily = bgphpb.AddressFamily
	capture.PeerIp = bgphpb.PeerIp
	capture.LocalIp = bgphpb.LocalIp
	capture.Update = bgpup.(pp.BGPUpdater).GetUpdate()
	return capture, nil
}

{'fraction_non_alphanumeric': 0.12879078694817658, 'fraction_numerical': 0.012092130518234165, 'mean_word_length': 4.263636363636364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5068616', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2125, 'n_words': 383}