<commit_before>#ifndef _CH_FRB_IO_INTERNALS_HPP
#define _CH_FRB_IO_INTERNALS_HPP

#if (__cplusplus < 201103) && !defined(__GXX_EXPERIMENTAL_CXX0X__)
#error "This source file needs to be compiled with C++11 support (g++ -std=c++11)"
#endif

#include <cmath>
#include <random>
#include <cstring>
#include <sstream>
#include <iostream>
#include <stdexcept>

#include <errno.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>

#include "ch_frb_io.hpp"


// Branch predictor hint
#ifndef _unlikely
#define _unlikely(cond)  (__builtin_expect(cond,0))
#endif


namespace ch_frb_io {
#if 0
}; // pacify emacs c-mode
#endif


// -------------------------------------------------------------------------------------------------


struct intensity_packet {
    // "Header" fields
    uint32_t  protocol_version;
    int16_t   data_nbytes;
    uint16_t  fpga_counts_per_sample;
    uint64_t  fpga_count;
    uint16_t  nbeams;
    uint16_t  nfreq_coarse;
    uint16_t  nupfreq;
    uint16_t  ntsamp;

    // "Data" fields
    uint16_t  *beam_ids;   // 1D array of length nbeams
    uint16_t  *freq_ids;   // 1D array of length nfreq_coarse
    float     *scales;     // 2D array of shape (nbeam, nfreq_coarse)
    float     *offsets;    // 2D array of shape (nbeam, nfreq_coarse)
    uint8_t   *data;       // array of shape (nbeam, nfreq_coarse, nupfreq, ntsamp)

    // FIXME rethink the member function names below, since they're not very intuitive

    // Returns true if packet is good, false if bad
    // Note: no checking of freq_ids is performed!
    bool read(const uint8_t *src, int src_nbytes);

    // Returns packet_nbytes (not data_nbytes)
    int write(uint8_t *dst) const;


    // The semantics of encode() aren't particularly intuitive, so we document it carefully here!
    //
    // It is assumed that the caller has initialized the "header" fields and the data pointers 'beam_ids', 'freq_ids'.
    // The other three data pointers (scales, offsets, data) are not assumed initialized, and encode() will initialize them
    // to point to subregions of the 'dst' array.
    //
    // The 'dst' array should point to an allocated but uninitialized memory region which will be filled by encode_packet().
    // Note that encode_packet() doesn't check for overflows (or do any arugment checking at all!)
    //
    // The 'intensity' and 'weights' pointers should point to arrays of logical shape (nbeams, nfreq_coarse, nupfreq, ntsamp).
    //
    // The stride arguments are defined so that the intensity array element with logical indices (b,f,u,t) is stored at
    // memory location
    //
    //    intensity + b*beam_stride + (f*nupfreq+u)*freq_stride + t
    //
    // and likewise for the weights.

    void encode(uint8_t *dst, const float *intensity, const float *weights, int beam_stride, int freq_stride, float wt_cutoff);
};


inline int header_size(int nbeams, int nfreq_coarse)<commit_msg>add comment explaining checks in intensity_packet::read()<commit_after>#ifndef _CH_FRB_IO_INTERNALS_HPP
#define _CH_FRB_IO_INTERNALS_HPP

#if (__cplusplus < 201103) && !defined(__GXX_EXPERIMENTAL_CXX0X__)
#error "This source file needs to be compiled with C++11 support (g++ -std=c++11)"
#endif

#include <cmath>
#include <random>
#include <cstring>
#include <sstream>
#include <iostream>
#include <stdexcept>

#include <errno.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>

#include "ch_frb_io.hpp"


// Branch predictor hint
#ifndef _unlikely
#define _unlikely(cond)  (__builtin_expect(cond,0))
#endif


namespace ch_frb_io {
#if 0
}; // pacify emacs c-mode
#endif


// -------------------------------------------------------------------------------------------------


struct intensity_packet {
    // "Header" fields
    uint32_t  protocol_version;
    int16_t   data_nbytes;
    uint16_t  fpga_counts_per_sample;
    uint64_t  fpga_count;
    uint16_t  nbeams;
    uint16_t  nfreq_coarse;
    uint16_t  nupfreq;
    uint16_t  ntsamp;

    // "Data" fields
    uint16_t  *beam_ids;   // 1D array of length nbeams
    uint16_t  *freq_ids;   // 1D array of length nfreq_coarse
    float     *scales;     // 2D array of shape (nbeam, nfreq_coarse)
    float     *offsets;    // 2D array of shape (nbeam, nfreq_coarse)
    uint8_t   *data;       // array of shape (nbeam, nfreq_coarse, nupfreq, ntsamp)

    // FIXME rethink the member function names below, since they're not very intuitive

    //
    // Returns true if packet is good, false if bad
    //
    // Includes the following checks:
    //   - protocol version == 1
    //   - dimensions (nbeams, nfreq_coarse, nupfreq, ntsamp) are not large enough to lead to overflows
    //   - packet and data byte counts are correct
    //
    // Does not check the following:
    //   - any checks on beam ids or coarse_freq_ids
    //   - ntsamp is a power of two
    //   - nbeams, nfreq_coarse, nupfreq, ntsamp, fpga_counts_per_sample are all > 0
    //   - fpga_count is a multiple of (fpga_counts_per_sample * ntsamp)
    //
    bool read(const uint8_t *src, int src_nbytes);

    // Returns packet_nbytes (not data_nbytes)
    int write(uint8_t *dst) const;


    // The semantics of encode() aren't particularly intuitive, so we document it carefully here!
    //
    // It is assumed that the caller has initialized the "header" fields and the data pointers 'beam_ids', 'freq_ids'.
    // The other three data pointers (scales, offsets, data) are not assumed initialized, and encode() will initialize them
    // to point to subregions of the 'dst' array.
    //
    // The 'dst' array should point to an allocated but uninitialized memory region which will be filled by encode_packet().
    // Note that encode_packet() doesn't check for overflows (or do any arugment checking at all!)
    //
    // The 'intensity' and 'weights' pointers should point to arrays of logical shape (nbeams, nfreq_coarse, nupfreq, ntsamp).
    //
    // The stride arguments are defined so that the intensity array element with logical indices (b,f,u,t) is stored at
    // memory location
    //
    //    intensity + b*beam_stride + (f*nupfreq+u)*freq_stride + t
    //
    // and likewise for the weights.

    void encode(uint8_t *dst, const float *intensity, const float *weights, int beam_stride, int freq_stride, float wt_cutoff);
};


inline int header_size(int nbeams, int nfreq_coarse)
{'fraction_non_alphanumeric': 0.12385249727711219, 'fraction_numerical': 0.013069861521705306, 'mean_word_length': 2.8261904761904764, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1178129', 'n_tokens_mistral': 2176, 'n_tokens_neox': 2056, 'n_words': 695}