<commit_before>
#include "common/ByteArray.h"
#include "common/QuickVec.h"
#include "common/Object.h"

#include <vorbis/vorbisfile.h>

#include <string>


namespace lumen {


        static size_t ogg_read_func(void* ptr, size_t size, size_t nmemb, void* datasource);
        static int ogg_seek_func(void* datasource, ogg_int64_t offset, int whence);
        static int ogg_close_func(void* datasource);
        static long ogg_tell_func(void* datasource);

        class OGG_file_source : public Object {
            public:
                FILE*               file_source;
                std::string         source_name;
                OggVorbis_File*     ogg_file;
                vorbis_info*        info;
                vorbis_comment*     comments;
                off_t               offset;
                off_t               length;

            OGG_file_source() : offset(0), length(0) {

                ogg_file = new OggVorbis_File();

            } //OGG_file_source

            ~OGG_file_source() {

                ov_clear(ogg_file);

            } //~
        };

        std::string ogg_error_string(int code) {
            switch(code)
            {
                case OV_EREAD:
                    return std::string("failed to read from media");
                case OV_ENOTVORBIS:
                    return std::string("not vorbis data");
                case OV_EVERSION:
                    return std::string("vorbis version mismatch");
                case OV_EBADHEADER:
                    return std::string("invalid vorbis header");
                case OV_EFAULT:
                    return std::string("internal logic fault (bug or heap/stack corruption.");
                default:
                    return std::string("unknown ogg error");
            }
        }

        #define OGG_BUFFER_LENGTH 4096
        // 0 for Little-Endian, 1 for Big-Endian
        #ifdef HXCPP_BIG_ENDIAN
            #define OGG_BUFFER_READ_TYPE 1
        #else
            #define OGG_BUFFER_READ_TYPE 0
        #endif

        void audio_load_ogg_bytes(
            QuickVec<unsigned char> &out_buffer, const char* _id, 
            int* channels, long* rate, long* bitrate_upper, long* bitrate_nominal, 
            long* bitrate_lower, long* bitrate_window 
        ) {

            ov_callbacks open_callbacks;
            OGG_file_source* ogg_source = new OGG_file_source();
            ogg_source->source_name = std::string(_id);

            #ifdef ANDROID
        
                //mInfo = AndroidGetAssetFD(path.c_str());
                //oggFile = fdopen(mInfo.fd, "rb");
                //fseek(oggFile, mInfo.offset, 0);

            #endif

            open_callbacks.read_func =  &lumen::ogg_read_func;
            open_callbacks.seek_func =  &lumen::ogg_seek_func;
            open_callbacks.close_func = &lumen::ogg_close_func;
            open_callbacks.tell_func =  &lumen::ogg_tell_func;

            ogg_source->file_source = fopen(_id, "rb");
            if(!ogg_source->file_source) {
                printf("%s %s\n", "error opening file:", _id);
                return;
            }
            
            fseek(ogg_source->file_source, 0, SEEK_END);
            ogg_source->length = ftell(ogg_source->file_source);
            fseek(ogg_source->file_source, 0, SEEK_SET);

            int result = ov_open_callbacks(ogg_source, ogg_source->ogg_file, NULL, 0, open_callbacks);

            if(result < 0) {
            
                fclose(ogg_source->file_source);
                delete ogg_source;
                
                std::string s = ogg_error_string(result);
                printf("%s result:%d   code:%s \n", "ogg stream failed to open!?", result, s.c_str());
                
                return;

            } //result < 0

            ogg_source->info = ov_info(ogg_source->ogg_file, -1);
            ogg_source->comments = ov_comment(ogg_source->ogg_file, -1);

            ogg_int64_t total_length = ov_pcm_total( ogg_source->ogg_file, -1 ) * ogg_source->info->channels * (2);

                printf("version         %d \n",     ogg_source->info->version);
                printf("channels        %d \n",     ogg_source->info->channels);
                printf("rate (hz)       %lu \n",    ogg_source->info->rate);
                printf("bitrate upper   %lu \n",    ogg_source->info->bitrate_upper);
                printf("bitrate nominal %lu \n",    ogg_source->info->bitrate_nominal);
                printf("bitrate lower   %lu \n",    ogg_source->info->bitrate_lower);
                printf("bitrate window  %lu \n",    ogg_source->info->bitrate_window);
                printf("vendor          %s \n",     ogg_source->comments->vendor);
                printf("length          %ld \n",    ogg_source->length);
                printf("uncompressed    %lld \n",   (long long)total_length);
                
            for(int i = 0; i < ogg_source->comments->comments; i++) {
                printf("\t%s\n",  ogg_source->comments->user_comments[i]);
            }

            *channels           = ogg_source->info->channels;<commit_msg>crazy type warnings differ between linux and osx gcc, heh<commit_after>
#include "common/ByteArray.h"
#include "common/QuickVec.h"
#include "common/Object.h"

#include <vorbis/vorbisfile.h>

#include <string>


namespace lumen {


        static size_t ogg_read_func(void* ptr, size_t size, size_t nmemb, void* datasource);
        static int ogg_seek_func(void* datasource, ogg_int64_t offset, int whence);
        static int ogg_close_func(void* datasource);
        static long ogg_tell_func(void* datasource);

        class OGG_file_source : public Object {
            public:
                FILE*               file_source;
                std::string         source_name;
                OggVorbis_File*     ogg_file;
                vorbis_info*        info;
                vorbis_comment*     comments;
                off_t               offset;
                off_t               length;

            OGG_file_source() : offset(0), length(0) {

                ogg_file = new OggVorbis_File();

            } //OGG_file_source

            ~OGG_file_source() {

                ov_clear(ogg_file);

            } //~
        };

        std::string ogg_error_string(int code) {
            switch(code)
            {
                case OV_EREAD:
                    return std::string("failed to read from media");
                case OV_ENOTVORBIS:
                    return std::string("not vorbis data");
                case OV_EVERSION:
                    return std::string("vorbis version mismatch");
                case OV_EBADHEADER:
                    return std::string("invalid vorbis header");
                case OV_EFAULT:
                    return std::string("internal logic fault (bug or heap/stack corruption.");
                default:
                    return std::string("unknown ogg error");
            }
        }

        #define OGG_BUFFER_LENGTH 4096
        // 0 for Little-Endian, 1 for Big-Endian
        #ifdef HXCPP_BIG_ENDIAN
            #define OGG_BUFFER_READ_TYPE 1
        #else
            #define OGG_BUFFER_READ_TYPE 0
        #endif

        void audio_load_ogg_bytes(
            QuickVec<unsigned char> &out_buffer, const char* _id, 
            int* channels, long* rate, long* bitrate_upper, long* bitrate_nominal, 
            long* bitrate_lower, long* bitrate_window 
        ) {

            ov_callbacks open_callbacks;
            OGG_file_source* ogg_source = new OGG_file_source();
            ogg_source->source_name = std::string(_id);

            #ifdef ANDROID
        
                //mInfo = AndroidGetAssetFD(path.c_str());
                //oggFile = fdopen(mInfo.fd, "rb");
                //fseek(oggFile, mInfo.offset, 0);

            #endif

            open_callbacks.read_func =  &lumen::ogg_read_func;
            open_callbacks.seek_func =  &lumen::ogg_seek_func;
            open_callbacks.close_func = &lumen::ogg_close_func;
            open_callbacks.tell_func =  &lumen::ogg_tell_func;

            ogg_source->file_source = fopen(_id, "rb");
            if(!ogg_source->file_source) {
                printf("%s %s\n", "error opening file:", _id);
                return;
            }
            
            fseek(ogg_source->file_source, 0, SEEK_END);
            ogg_source->length = ftell(ogg_source->file_source);
            fseek(ogg_source->file_source, 0, SEEK_SET);

            int result = ov_open_callbacks(ogg_source, ogg_source->ogg_file, NULL, 0, open_callbacks);

            if(result < 0) {
            
                fclose(ogg_source->file_source);
                delete ogg_source;
                
                std::string s = ogg_error_string(result);
                printf("%s result:%d   code:%s \n", "ogg stream failed to open!?", result, s.c_str());
                
                return;

            } //result < 0

            ogg_source->info = ov_info(ogg_source->ogg_file, -1);
            ogg_source->comments = ov_comment(ogg_source->ogg_file, -1);

            ogg_int64_t total_length = ov_pcm_total( ogg_source->ogg_file, -1 ) * ogg_source->info->channels * (2);

                printf("version         %d \n",     ogg_source->info->version);
                printf("channels        %d \n",     ogg_source->info->channels);
                printf("rate (hz)       %lu \n",    ogg_source->info->rate);
                printf("bitrate upper   %lu \n",    ogg_source->info->bitrate_upper);
                printf("bitrate nominal %lu \n",    ogg_source->info->bitrate_nominal);
                printf("bitrate lower   %lu \n",    ogg_source->info->bitrate_lower);
                printf("bitrate window  %lu \n",    ogg_source->info->bitrate_window);
                printf("vendor          %s \n",     ogg_source->comments->vendor);
                printf("length          %lld \n",    (long long)ogg_source->length);
                printf("uncompressed    %lld \n",   (long long)total_length);
                
            for(int i = 0; i < ogg_source->comments->comments; i++) {
                printf("\t%s\n",  ogg_source->comments->user_comments[i]);
            }

            *channels           = ogg_source->info->channels;
{'fraction_non_alphanumeric': 0.10625726462611391, 'fraction_numerical': 0.0048430840759395586, 'mean_word_length': 1.6159108183430453, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2543419', 'n_tokens_mistral': 3172, 'n_tokens_neox': 2937, 'n_words': 599}