<commit_before>//  ---------------------------------------------------------------------------
//
//  net.go
//
//  Copyright (c) 2013, <NAME>. 
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

// Network message (max length 32KB)
//		flags 
//			11: compressed
//			12: encrypted
//			13: reserved
//			14: reserved
//			15: reserved
//			16: reserved
//
// [0-1]     msgtype (bits 1-10 for 1024 unique msg types), flags (bits 11-16)
// [2-3]     msgsize (uint16)
// [4-32767] payload is msg size - 2
package net

// Stdlib imports.
import (
	"sync"
)

// Message header constants.
const (
	HEADER_LEN_B    = 4
	MAX_MSG_TYPE    = 1023
	MAX_NET_MSG_LEN = 32 * 1024
)

// Message flag bitwise offsets.
const (
	msgCompressedOffset = 11
	msgEncryptedOffset  = 12
)

// Msg flag masks.
const (
	msgTypeMask  = 0xFC00
	msgFlagsMask = 0x03FF
)

// Network id pool.
var netId uint32

// Routing map and synchronization.
var (
	routeMutex sync.RWMutex	
	routeMap   map[uint16]*TcpProtocol
)

// CompressionProvider specifies the interface which network protocols will
// use to compress/decompress network messages. All outgoing messages flow
// through Compress(). Only messages received with the compression header bit
// set will flow through Decompress().
type CompressionProvider interface {
	Compress(msg *NetMsg) error
	Decompress(msg *NetMsg) error
}

// CryptoProvider specifies the interface which network protocols will use
// to encrypt and decrypt network messages. All outgoing messages flow
// through Encrypt(). Only messages received with the encrypted header bit
// set will flow through Decrypt().
type CryptoProvider interface {
	Decrypt(msg *NetMsg) error
	Encrypt(msg *NetMsg) error
}

// AccessProvider specifies the interface which network protocols will use
// to authorize messages for sending or processing. All incoming and outgoing
// messages flow through Authorize() and are immediately dropped if it returns
// false.
type AccessProvider interface {
	Authorize(msg *NetMsg) (bool, error)
}

// MsgProcessor specifies the entry and exit points of the network system which
// network protcols use to accept and distribute incoming messages as well as
// accept and disseminate outgoing messages to the correct endpoints.
type MsgProcessor interface {
	ProcessMsg(msg *NetMsg) error
	SendMsg(id uint32, msg *NetMsg)
}

// GetMsgCompressedFlag retrieves bit 11 of the message header, which is used
// to specify whether the message data itself is compressed or not.
func GetMsgCompressedFlag(header uint16) bool {
	return (header & (1 << msgCompressedOffset)) != 0
}

// GetMsgEncryptedFlag retrieves bit 12 of the message header, which is used
// to specify whether the message data itself is encrypted or not.
func GetMsgEncryptedFlag(header uint16) bool {
	return (header & (1 << msgEncryptedOffset)) != 0
}

// GetMsgHeader retrieves the first 2 byte header of raw line data. 
// Packed into the header is the message type signature and flags specifying 
// whether the data payload is compressed and/or encrypted.
func GetMsgHeader(msgData []byte) uint16 {
	if len(msgData) < 2 {
		panic("msgData buffer less than 2 bytes")
	}

	header := uint16(msgData[0]) << 8 | uint16(msgData[1])
	
	return header
}

// GetMsgSig retrieves the message type signature out of a raw message header.
func GetMsgSig(header uint16) uint16 {
	sig := header &^ msgTypeMask

	return sig
}

// GetMsgSize retrieves the data size property from raw line data.
func GetMsgSize(msgData []byte) uint16 {
	if len(msgData) < 4 {<commit_msg>broke a test - adding a func back in<commit_after>//  ---------------------------------------------------------------------------
//
//  net.go
//
//  Copyright (c) 2013, <NAME>. 
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

// Network message (max length 32KB)
//		flags 
//			11: compressed
//			12: encrypted
//			13: reserved
//			14: reserved
//			15: reserved
//			16: reserved
//
// [0-1]     msgtype (bits 1-10 for 1024 unique msg types), flags (bits 11-16)
// [2-3]     msgsize (uint16)
// [4-32767] payload is msg size - 2
package net

// Stdlib imports.
import (
	"sync"
)

// Message header constants.
const (
	HEADER_LEN_B    = 4
	MAX_MSG_TYPE    = 1023
	MAX_NET_MSG_LEN = 32 * 1024
)

// Message flag bitwise offsets.
const (
	msgCompressedOffset = 11
	msgEncryptedOffset  = 12
)

// Msg flag masks.
const (
	msgTypeMask  = 0xFC00
	msgFlagsMask = 0x03FF
)

// Network id pool.
var netId uint32

// Routing map and synchronization.
var (
	routeMutex sync.RWMutex	
	routeMap   map[uint16]*TcpProtocol
)

// CompressionProvider specifies the interface which network protocols will
// use to compress/decompress network messages. All outgoing messages flow
// through Compress(). Only messages received with the compression header bit
// set will flow through Decompress().
type CompressionProvider interface {
	Compress(msg *NetMsg) error
	Decompress(msg *NetMsg) error
}

// CryptoProvider specifies the interface which network protocols will use
// to encrypt and decrypt network messages. All outgoing messages flow
// through Encrypt(). Only messages received with the encrypted header bit
// set will flow through Decrypt().
type CryptoProvider interface {
	Decrypt(msg *NetMsg) error
	Encrypt(msg *NetMsg) error
}

// AccessProvider specifies the interface which network protocols will use
// to authorize messages for sending or processing. All incoming and outgoing
// messages flow through Authorize() and are immediately dropped if it returns
// false.
type AccessProvider interface {
	Authorize(msg *NetMsg) (bool, error)
}

// MsgProcessor specifies the entry and exit points of the network system which
// network protcols use to accept and distribute incoming messages as well as
// accept and disseminate outgoing messages to the correct endpoints.
type MsgProcessor interface {
	ProcessMsg(msg *NetMsg) error
	SendMsg(id uint32, msg *NetMsg)
}

// GetMsgCompressedFlag retrieves bit 11 of the message header, which is used
// to specify whether the message data itself is compressed or not.
func GetMsgCompressedFlag(header uint16) bool {
	return (header & (1 << msgCompressedOffset)) != 0
}

// GetMsgEncryptedFlag retrieves bit 12 of the message header, which is used
// to specify whether the message data itself is encrypted or not.
func GetMsgEncryptedFlag(header uint16) bool {
	return (header & (1 << msgEncryptedOffset)) != 0
}

// GetMsgHeader retrieves the first 2 byte header of raw line data. 
// Packed into the header is the message type signature and flags specifying 
// whether the data payload is compressed and/or encrypted.
func GetMsgHeader(msgData []byte) uint16 {
	if len(msgData) < 2 {
		panic("msgData buffer less than 2 bytes")
	}

	header := uint16(msgData[0]) << 8 | uint16(msgData[1])
	
	return header
}

// GetMsgPayload returns the payload portion of a raw message buffer.
func GetMsgPayload(msgData []byte) []byte {
  	if len(msgData) < 4 {
     	panic("msgData buffer less than 2 bytes")
   	}

   	size := GetMsgSize(msgData)
 
   	return msgData[4:size + 4]
 }

// GetMsgSig retrieves the message type signature out of a raw message header.
func GetMsgSig(header uint16) uint16 {
	sig := header &^ msgTypeMask

	return sig
}

// GetMsgSize retrieves the data size property from raw line data.
func GetMsgSize(msgData []byte) uint16 {
	if len(msgData) < 4 {
{'fraction_non_alphanumeric': 0.10821881606765328, 'fraction_numerical': 0.026955602536997886, 'mean_word_length': 3.2403361344537815, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1404544', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2153, 'n_words': 899}