<commit_before>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/time.h>

#include "../readstat.h"
#include "../readstat_iconv.h"

#include "../stata/readstat_dta.h"

#include "test_types.h"
#include "test_error.h"
#include "test_readstat.h"
#include "test_buffer.h"
#include "test_read.h"
#include "test_write.h"

#define MAX_TESTS_PER_GROUP 20

#define RT_FORMAT_TEST_TIMESTAMPS  (RT_FORMAT_DTA_105_AND_NEWER | RT_FORMAT_SPSS | RT_FORMAT_SAS7BDAT)

typedef struct rt_test_group_s {
    char             label[80];
    rt_test_file_t   tests[MAX_TESTS_PER_GROUP];
} rt_test_group_t;

rt_test_group_t _test_groups[] = {
    {
        .label = "Notes",
        .tests = {
            {
                .label = "Short notes",
                .test_formats = RT_FORMAT_DTA_105_AND_NEWER | RT_FORMAT_SPSS,

                .notes_count = 2,
                .notes = {
                    "This is a note",
                    "This is another note"
                }
            },

            {
                .label = "Long notes",
                .write_error = READSTAT_ERROR_NOTE_IS_TOO_LONG,
                .test_formats = RT_FORMAT_SPSS,

                .notes_count = 1,
                .notes = {
                    "This is a note that is longer than the 80-byte line length of "
                    "the SPSS document record, and will produce an error."
                }
            }
        }
    },

    {
        .label = "Compression tests",
        .tests = {
            {
                .label = "SAV row compression",
                .test_formats = RT_FORMAT_SAV_COMP_ROWS,
                .rows = 3,
                .columns = {
                    {
                        .name = "VAR1",
                        .type = READSTAT_TYPE_DOUBLE,
                        .label = "Double-precision variable",
                        .values = {
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .is_system_missing = 1 }
                        }
                    },

                    {
                        .name = "VAR2",
                        .type = READSTAT_TYPE_STRING,
                        .label = "String variable",
                        .values = {
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->        <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblah" } }
                        }
                    }
                }
            },

            {
                .label = "SAS7BDAT RLE compression",
                .test_formats = RT_FORMAT_SAS7BDAT_COMP_ROWS,
                .rows = 10,
                .columns = {
                    {
                        .name = "VAR1",
                        .type = READSTAT_TYPE_DOUBLE,
                        .label = "Double-precision variable",
                        .values = {
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .is_system_missing = 1 },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },

                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } }
                        }
                    },

                    {
                        .name = "VAR2",
                        .type = READSTAT_TYPE_STRING,
                        .label = "String variable",
                        .values = {
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->        <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->                             <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahblahblah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahblahblahblahblahblahbba" } },

                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahsafhuweyeoyraewayfeawopyfhewuhafeywfdhsfdsaf" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "atsyms->@@@@@@@@<-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "atsyms->@@@@@@@@@@@@@@@@@@@@@@@@@@@@@<-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "Fchars->FFFFFFFF<-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "Fchars->FFFFFFFFFFFFFFFFFFFFFFFFFFFFF<-- here" } }
                        }
                    }
                }
            }
        }
    },

    {
        .label = "Long strings and string refs",
        .tests = {
            {<commit_msg>Improve RLE test coverage<commit_after>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/time.h>

#include "../readstat.h"
#include "../readstat_iconv.h"

#include "../stata/readstat_dta.h"

#include "test_types.h"
#include "test_error.h"
#include "test_readstat.h"
#include "test_buffer.h"
#include "test_read.h"
#include "test_write.h"

#define MAX_TESTS_PER_GROUP 20

#define RT_FORMAT_TEST_TIMESTAMPS  (RT_FORMAT_DTA_105_AND_NEWER | RT_FORMAT_SPSS | RT_FORMAT_SAS7BDAT)

typedef struct rt_test_group_s {
    char             label[80];
    rt_test_file_t   tests[MAX_TESTS_PER_GROUP];
} rt_test_group_t;

rt_test_group_t _test_groups[] = {
    {
        .label = "Notes",
        .tests = {
            {
                .label = "Short notes",
                .test_formats = RT_FORMAT_DTA_105_AND_NEWER | RT_FORMAT_SPSS,

                .notes_count = 2,
                .notes = {
                    "This is a note",
                    "This is another note"
                }
            },

            {
                .label = "Long notes",
                .write_error = READSTAT_ERROR_NOTE_IS_TOO_LONG,
                .test_formats = RT_FORMAT_SPSS,

                .notes_count = 1,
                .notes = {
                    "This is a note that is longer than the 80-byte line length of "
                    "the SPSS document record, and will produce an error."
                }
            }
        }
    },

    {
        .label = "Compression tests",
        .tests = {
            {
                .label = "SAV row compression",
                .test_formats = RT_FORMAT_SAV_COMP_ROWS,
                .rows = 3,
                .columns = {
                    {
                        .name = "VAR1",
                        .type = READSTAT_TYPE_DOUBLE,
                        .label = "Double-precision variable",
                        .values = {
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .is_system_missing = 1 }
                        }
                    },

                    {
                        .name = "VAR2",
                        .type = READSTAT_TYPE_STRING,
                        .label = "String variable",
                        .values = {
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->        <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblah" } }
                        }
                    }
                }
            },

            {
                .label = "SAS7BDAT RLE compression",
                .test_formats = RT_FORMAT_SAS7BDAT_COMP_ROWS,
                .rows = 10,
                .columns = {
                    {
                        .name = "VAR1",
                        .type = READSTAT_TYPE_DOUBLE,
                        .label = "Double-precision variable",
                        .values = {
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .is_system_missing = 1 },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },

                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 100.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } },
                            { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0.0 } }
                        }
                    },

                    {
                        .name = "VAR2",
                        .type = READSTAT_TYPE_STRING,
                        .label = "String variable",
                        .values = {
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->        <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "spaces->                             <-- here" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahblahblah" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahblahblahblahblahblahbba" } },

                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "blahblahblahsafhuweyeoyraewayfeawopyfhewuhafeywfdhsfdsaf" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "atsyms->@@@@@@@@<--FFFFFFFFFFFFFFFFFFFFFFFFFF" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "atsyms->@@@@@@@@@@@@@@@@@@@@@@@@@@@@@<--FFFFF" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "jiafojdsaufwejfiewnfiabfiuaewbfiuwhfeiuwfuienawuifnwauiefnhfuiwheufhwfuiewfjwuifewuif" } },
                            { .type = READSTAT_TYPE_STRING, .v = { .string_value = "Fchars->GGGGGGGGGGGGGGGGGGGGGGGGGGGGG<-- here" } }
                        }
                    }
                }
            }
        }
    },

    {
        .label = "Long strings and string refs",
        .tests = {
            {
{'fraction_non_alphanumeric': 0.11885801423246357, 'fraction_numerical': 0.009488309047780414, 'mean_word_length': 1.1072831131738665, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6404557', 'n_tokens_mistral': 3650, 'n_tokens_neox': 3455, 'n_words': 594}