<commit_before>//---------------------------------------------------------------------------------------------------------------------------------
// File: core_results.cpp
//
// Contents: Result sets
//
// Microsoft Drivers 4.1 for PHP for SQL Server
// Copyright(c) Microsoft Corporation
// All rights reserved.
// MIT License
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the ""Software""), 
//  to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
//  and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions :
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
//  IN THE SOFTWARE.
//---------------------------------------------------------------------------------------------------------------------------------

#include "core_sqlsrv.h"

#include <functional>
#include <iostream>
#include <sstream>

using namespace core;

// conversion matrix
// each entry holds a function that can perform the conversion or NULL which means the conversion isn't supported
// this is initialized the first time the buffered result set is created.
sqlsrv_buffered_result_set::conv_matrix_t sqlsrv_buffered_result_set::conv_matrix;

namespace {

// *** internal types ***

#pragma warning(disable:4200)

// *** internal constants ***

const int INITIAL_FIELD_STRING_LEN = 256;    // base allocation size when retrieving a string field

// *** internal functions ***

// return an integral type rounded up to a certain number
template <int align, typename T>
T align_to( T number )
{
    DEBUG_SQLSRV_ASSERT( (number + align) > number, "Number to align overflowed" ); 
    return ((number % align) == 0) ? number : (number + align - (number % align));
}

// return a pointer address aligned to a certain address boundary
template <int align, typename T>
T* align_to( T* ptr )
{
    size_t p_value = (size_t) ptr;
    return align_to<align, size_t>( p_value );
}

// set the nth bit of the bitstream starting at ptr
void set_bit( void* ptr, unsigned int bit )
{
    unsigned char* null_bits = reinterpret_cast<unsigned char*>( ptr );
    null_bits += bit >> 3;
    *null_bits |= 1 << ( 7 - ( bit & 0x7 ));
}

// retrieve the nth bit from the bitstream starting at ptr
bool get_bit( void* ptr, unsigned int bit )
{
    unsigned char* null_bits = reinterpret_cast<unsigned char*>( ptr );
    null_bits += bit >> 3;
    return ((*null_bits & (1 << ( 7 - ( bit & 0x07 )))) != 0);
}

// read in LOB field during buffered result creation
SQLPOINTER read_lob_field( sqlsrv_stmt* stmt, SQLUSMALLINT field_index, sqlsrv_buffered_result_set::meta_data& meta,
                           zend_long mem_used TSRMLS_DC );

// dtor for each row in the cache
void cache_row_dtor(zval* data);

// convert a number to a string using locales
// There is an extra copy here, but given the size is short (usually <20 bytes) and the complications of
// subclassing a new streambuf just to avoid the copy, it's easier to do the copy
template <typename Char, typename Number>
SQLRETURN number_to_string( Number* number_data, _Out_ void* buffer, SQLLEN buffer_length, _Out_ SQLLEN* out_buffer_length,
                            sqlsrv_error_auto_ptr& last_error )
{
	// get to display size by removing the null terminator from buffer length
	size_t display_size = ( buffer_length - sizeof( Char )) / sizeof( Char );

    std::basic_ostringstream<Char> os;
	// use the display size to determine the sql type. And if it is a double, set the precision accordingly
	// the display sizes are set by the ODBC driver based on the precision of the sql type
	// otherwise we can just use the default precision as long will not be truncated
	size_t real_display_size = 14;
	size_t float_display_size = 24;
	size_t real_precision = 7;
	size_t float_precision = 15;
	// this is the case of sql type float(24) or real
	if ( display_size == real_display_size ) {
		os.precision( real_precision );
	}
	// this is the case of sql type float(53)
	else if ( display_size == float_display_size ) {
		os.precision( float_precision );
	}
    std::locale loc;
    os.imbue( loc );
    std::use_facet< std::num_put< Char > >( loc ).put( std::basic_ostream<Char>::_Iter( os.rdbuf() ), os, ' ', *number_data );
    std::basic_string<Char>& str_num = os.str();

    if( os.fail() ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "IMSSP", (SQLCHAR*) "Failed to convert number to string", -1 );
        return SQL_ERROR;
    }

    if( str_num.size() * sizeof(Char) > (size_t) buffer_length ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "HY090", (SQLCHAR*) "Buffer length too small to hold number as string", -1 );
        return SQL_ERROR;
    }

    *out_buffer_length = str_num.size() * sizeof( Char ); // str_num.size() already include the NULL terminator
    memcpy_s( buffer, buffer_length, str_num.c_str(), *out_buffer_length );

    return SQL_SUCCESS;
}

template <typename Number, typename Char>
SQLRETURN string_to_number( Char* string_data, SQLLEN str_len, _Out_ void* buffer, SQLLEN buffer_length, 
                            _Out_ SQLLEN* out_buffer_length, sqlsrv_error_auto_ptr& last_error )
{
    Number* number_data = reinterpret_cast<Number*>( buffer ); 
    std::locale loc;    // default locale should match system
    std::basic_istringstream<Char> is;
    is.str( string_data );
    is.imbue( loc );
    std::ios_base::iostate st = 0;

    std::use_facet< std::num_get< Char > >( loc ).get( std::basic_istream<Char>::_Iter( is.rdbuf( ) ), 
                                                       std::basic_istream<Char>::_Iter(0), is, st, *number_data );

    if( st & std::ios_base::failbit ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "22003", (SQLCHAR*) "Numeric value out of range", 103 );
        return SQL_ERROR;        
    }

    *out_buffer_length = sizeof( Number );

    return SQL_SUCCESS;
}

// "closure" for the hash table destructor
struct row_dtor_closure {

    sqlsrv_buffered_result_set* results;
    BYTE* row_data;

    row_dtor_closure( sqlsrv_buffered_result_set* st, BYTE* row ) :
        results( st ), row_data( row )
    {
    }                      
};

sqlsrv_error* odbc_get_diag_rec( sqlsrv_stmt* odbc, SQLSMALLINT record_number )
{
    SQLWCHAR wsql_state[ SQL_SQLSTATE_BUFSIZE ];
    SQLWCHAR wnative_message[ SQL_MAX_MESSAGE_LENGTH + 1 ];
    SQLINTEGER native_code;
    SQLSMALLINT wnative_message_len = 0;

    SQLRETURN r = SQLGetDiagRecW( SQL_HANDLE_STMT, odbc->handle(), record_number, wsql_state, &native_code, wnative_message, 
                                  SQL_MAX_MESSAGE_LENGTH + 1, &wnative_message_len );
    if( !SQL_SUCCEEDED( r ) || r == SQL_NO_DATA ) {
        return NULL;
    }

    // convert the error into the encoding of the context
    SQLSRV_ENCODING enc = odbc->encoding();
    if( enc == SQLSRV_ENCODING_DEFAULT ) {
        enc = odbc->conn->encoding();
    }

    // convert the error into the encoding of the context
    sqlsrv_malloc_auto_ptr<SQLCHAR> sql_state;
    SQLLEN sql_state_len = 0;
    if (!convert_string_from_utf16( enc, wsql_state, sizeof(wsql_state), (char**)&sql_state, sql_state_len )) {
        return NULL;
    }
    
    sqlsrv_malloc_auto_ptr<SQLCHAR> native_message;
    SQLLEN native_message_len = 0;
    if (!convert_string_from_utf16( enc, wnative_message, wnative_message_len, (char**)&native_message, native_message_len )) {
        return NULL;
    }

    return new (sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( (SQLCHAR*) sql_state, (SQLCHAR*) native_message, 
                                                                      native_code );
}

}   // namespace

// base class result set

sqlsrv_result_set::sqlsrv_result_set( sqlsrv_stmt* stmt ) :
    odbc( stmt )
{
}


// ODBC result set
// This object simply wraps ODBC function calls

sqlsrv_odbc_result_set::sqlsrv_odbc_result_set( sqlsrv_stmt* stmt ) : 
    sqlsrv_result_set( stmt )
{
}

sqlsrv_odbc_result_set::~sqlsrv_odbc_result_set( void )
{
}

SQLRETURN sqlsrv_odbc_result_set::fetch( SQLSMALLINT orientation, SQLLEN offset TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLFetchScroll( odbc, orientation, offset TSRMLS_CC );
}

SQLRETURN sqlsrv_odbc_result_set::get_data( SQLUSMALLINT field_index, SQLSMALLINT target_type,
                                            _Out_ SQLPOINTER buffer, SQLLEN buffer_length, _Out_ SQLLEN* out_buffer_length,
                                            bool handle_warning TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLGetData( odbc, field_index, target_type, buffer, buffer_length, out_buffer_length, handle_warning TSRMLS_CC );
}

SQLRETURN sqlsrv_odbc_result_set::get_diag_field( SQLSMALLINT record_number, SQLSMALLINT diag_identifier, 
                                                  _Out_ SQLPOINTER diag_info_buffer, SQLSMALLINT buffer_length,
                                                  _Out_ SQLSMALLINT* out_buffer_length TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLGetDiagField( odbc, record_number, diag_identifier, diag_info_buffer, buffer_length, 
                                  out_buffer_length TSRMLS_CC );
}

sqlsrv_error* sqlsrv_odbc_result_set::get_diag_rec( SQLSMALLINT record_number )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return odbc_get_diag_rec( odbc, record_number );
}

SQLLEN sqlsrv_odbc_result_set::row_count( TSRMLS_D )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLRowCount( odbc TSRMLS_CC );
}


// Buffered result set
// This class holds a result set in memory

sqlsrv_buffered_result_set::sqlsrv_buffered_result_set( sqlsrv_stmt* stmt TSRMLS_DC ) :
    sqlsrv_result_set( stmt ),
    cache(NULL),
    col_count(0),
    meta(NULL),
    current(0),
    last_field_index(-1),
    read_so_far(0)
{
    // 10 is an arbitrary number for now for the initial size of the cache
    ALLOC_HASHTABLE( cache );
    core::sqlsrv_zend_hash_init( *stmt, cache, 10 /* # of buckets */, cache_row_dtor /*dtor*/, 0 /*persistent*/ TSRMLS_CC );
    col_count = core::SQLNumResultCols( stmt TSRMLS_CC );
    // there is no result set to buffer
    if( col_count == 0 ) {
        return;
    }

    SQLULEN null_bytes = ( col_count / 8 ) + 1; // number of bits to reserve at the beginning of each row for NULL flags
    meta = static_cast<sqlsrv_buffered_result_set::meta_data*>( sqlsrv_malloc( col_count * 
                                                                               sizeof( sqlsrv_buffered_result_set::meta_data )));

    // set up the conversion matrix if this is the first time we're called
    if( conv_matrix.size() == 0 ) {

        conv_matrix[ SQL_C_CHAR ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::system_to_wide_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_binary_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::string_to_double;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::string_to_long;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_binary_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::wide_to_system_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::wstring_to_double;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::wstring_to_long;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::binary_to_system_string;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::binary_to_wide_string;
        conv_matrix[ SQL_C_LONG ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::long_to_double;
        conv_matrix[ SQL_C_LONG ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::to_long;
        conv_matrix[ SQL_C_LONG ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_long;
        conv_matrix[ SQL_C_LONG ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::long_to_system_string;
        conv_matrix[ SQL_C_LONG ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::long_to_wide_string;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::to_double;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_double;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::double_to_system_string;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::double_to_long;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::double_to_wide_string;
    }

    // get the meta data and calculate the size of a row buffer
    SQLULEN offset = null_bytes;
    for( SQLSMALLINT i = 0; i < col_count; ++i ) {

        core::SQLDescribeCol( stmt, i + 1, NULL, 0, NULL, &meta[i].type, &meta[i].length, &meta[i].scale, NULL TSRMLS_CC );

        offset = align_to<4>( offset );
        meta[i].offset = offset;

        switch( meta[i].type ) {

            // these types are the display size
            case SQL_BIGINT:
            case SQL_DECIMAL:
            case SQL_GUID:
            case SQL_NUMERIC:
                core::SQLColAttribute( stmt, i + 1, SQL_DESC_DISPLAY_SIZE, NULL, 0, NULL,
                                       reinterpret_cast<SQLLEN*>( &meta[i].length ) TSRMLS_CC );
                meta[i].length += sizeof( char ) + sizeof( SQLULEN ); // null terminator space
                offset += meta[i].length;
                break;

            // these types are the column size
            case SQL_BINARY:
            case SQL_CHAR:
            case SQL_SS_UDT:
            case SQL_VARBINARY:
            case SQL_VARCHAR:
                // var* field types are length prefixed
                if( meta[i].length == sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN ) {
                    offset += sizeof( void* );
                }
                else {
                    meta[i].length += sizeof( SQLULEN ) + sizeof( char ); // length plus null terminator space
                    offset += meta[i].length;
                }
                break;

            case SQL_WCHAR:
            case SQL_WVARCHAR:
                if( meta[i].length == sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN ) {
                    offset += sizeof( void* );
                }
                else {
                    meta[i].length *= sizeof( WCHAR );
                    meta[i].length += sizeof( SQLULEN ) + sizeof( WCHAR ); // length plus null terminator space
                    offset += meta[i].length;
                }
                break;

            // these types are LOBs
            case SQL_LONGVARBINARY:
            case SQL_LONGVARCHAR:
            case SQL_WLONGVARCHAR:
            case SQL_SS_XML:
                meta[i].length = sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN;
                offset += sizeof( void* );
                break;

            // these types are the ISO date size
            case SQL_DATETIME:
            case SQL_TYPE_DATE:
            case SQL_SS_TIME2:
            case SQL_SS_TIMESTAMPOFFSET:
            case SQL_TYPE_TIMESTAMP:
                core::SQLColAttribute( stmt, i + 1, SQL_DESC_DISPLAY_SIZE, NULL, 0, NULL, 
                                       reinterpret_cast<SQLLEN*>( &meta[i].length ) TSRMLS_CC );
                meta[i].length += sizeof(char) + sizeof( SQLULEN );  // null terminator space
                offset += meta[i].length;
                break;

            // these types are the native size
            case SQL_BIT:
            case SQL_INTEGER:
            case SQL_SMALLINT:
            case SQL_TINYINT:
                meta[i].length = sizeof( long );
                offset += meta[i].length;
                break;

            case SQL_REAL:
            case SQL_FLOAT:
                meta[i].length = sizeof( double );
                offset += meta[i].length;
                break;

            default:
                SQLSRV_ASSERT( false, "Unknown type in sqlsrv_buffered_query::sqlsrv_buffered_query" );
                break;
        }

        switch( meta[i].type ) {

            case SQL_BIGINT:
            case SQL_CHAR:
            case SQL_DATETIME:
            case SQL_DECIMAL:
            case SQL_GUID:
            case SQL_NUMERIC:
            case SQL_LONGVARCHAR:
            case SQL_TYPE_DATE:
            case SQL_SS_TIME2:
            case SQL_SS_TIMESTAMPOFFSET:
            case SQL_SS_XML:
            case SQL_TYPE_TIMESTAMP:
            case SQL_VARCHAR:
                meta[i].c_type = SQL_C_CHAR;
                break;

            case SQL_SS_UDT:
            case SQL_LONGVARBINARY:
            case SQL_BINARY:
            case SQL_VARBINARY:
                meta[i].c_type = SQL_C_BINARY;
                break;<commit_msg>revert core_results to 365220a<commit_after>//---------------------------------------------------------------------------------------------------------------------------------
// File: core_results.cpp
//
// Contents: Result sets
//
// Microsoft Drivers 4.1 for PHP for SQL Server
// Copyright(c) Microsoft Corporation
// All rights reserved.
// MIT License
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the ""Software""), 
//  to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
//  and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions :
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
//  IN THE SOFTWARE.
//---------------------------------------------------------------------------------------------------------------------------------

#include "core_sqlsrv.h"

#include <functional>
#include <iostream>
#include <sstream>

using namespace core;

// conversion matrix
// each entry holds a function that can perform the conversion or NULL which means the conversion isn't supported
// this is initialized the first time the buffered result set is created.
sqlsrv_buffered_result_set::conv_matrix_t sqlsrv_buffered_result_set::conv_matrix;

namespace {

// *** internal types ***

#pragma warning(disable:4200)

// *** internal constants ***

const int INITIAL_FIELD_STRING_LEN = 256;    // base allocation size when retrieving a string field

// *** internal functions ***

// return an integral type rounded up to a certain number
template <int align, typename T>
T align_to( T number )
{
    DEBUG_SQLSRV_ASSERT( (number + align) > number, "Number to align overflowed" ); 
    return ((number % align) == 0) ? number : (number + align - (number % align));
}

// return a pointer address aligned to a certain address boundary
template <int align, typename T>
T* align_to( T* ptr )
{
    size_t p_value = (size_t) ptr;
    return align_to<align, size_t>( p_value );
}

// set the nth bit of the bitstream starting at ptr
void set_bit( void* ptr, unsigned int bit )
{
    unsigned char* null_bits = reinterpret_cast<unsigned char*>( ptr );
    null_bits += bit >> 3;
    *null_bits |= 1 << ( 7 - ( bit & 0x7 ));
}

// retrieve the nth bit from the bitstream starting at ptr
bool get_bit( void* ptr, unsigned int bit )
{
    unsigned char* null_bits = reinterpret_cast<unsigned char*>( ptr );
    null_bits += bit >> 3;
    return ((*null_bits & (1 << ( 7 - ( bit & 0x07 )))) != 0);
}

// read in LOB field during buffered result creation
SQLPOINTER read_lob_field( sqlsrv_stmt* stmt, SQLUSMALLINT field_index, sqlsrv_buffered_result_set::meta_data& meta,
                           zend_long mem_used TSRMLS_DC );

// dtor for each row in the cache
void cache_row_dtor(zval* data);

// convert a number to a string using locales
// There is an extra copy here, but given the size is short (usually <20 bytes) and the complications of
// subclassing a new streambuf just to avoid the copy, it's easier to do the copy
template <typename Char, typename Number>
SQLRETURN number_to_string( Number* number_data, _Out_ void* buffer, SQLLEN buffer_length, _Out_ SQLLEN* out_buffer_length,
                            sqlsrv_error_auto_ptr& last_error )
{
	// get to display size by removing the null terminator from buffer length
	size_t display_size = ( buffer_length - sizeof( Char )) / sizeof( Char );

    std::basic_ostringstream<Char> os;
	// use the display size to determine the sql type. And if it is a double, set the precision accordingly
	// the display sizes are set by the ODBC driver based on the precision of the sql type
	// otherwise we can just use the default precision as long will not be truncated
	size_t real_display_size = 14;
	size_t float_display_size = 24;
	size_t real_precision = 7;
	size_t float_precision = 15;
	// this is the case of sql type float(24) or real
	if ( display_size == real_display_size ) {
		os.precision( real_precision );
	}
	// this is the case of sql type float(53)
	else if ( display_size == float_display_size ) {
		os.precision( float_precision );
	}
    std::locale loc;
    os.imbue( loc );
    std::use_facet< std::num_put< Char > >( loc ).put( std::basic_ostream<Char>::_Iter( os.rdbuf() ), os, ' ', *number_data );
    std::basic_string<Char>& str_num = os.str();

    if( os.fail() ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "IMSSP", (SQLCHAR*) "Failed to convert number to string", -1 );
        return SQL_ERROR;
    }

    if( str_num.size() * sizeof(Char) > (size_t) buffer_length ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "HY090", (SQLCHAR*) "Buffer length too small to hold number as string", -1 );
        return SQL_ERROR;
    }

    *out_buffer_length = str_num.size() * sizeof( Char ); // str_num.size() already include the NULL terminator
    memcpy_s( buffer, buffer_length, str_num.c_str(), *out_buffer_length );

    return SQL_SUCCESS;
}

template <typename Number, typename Char>
SQLRETURN string_to_number( Char* string_data, SQLLEN str_len, _Out_ void* buffer, SQLLEN buffer_length, 
                            _Out_ SQLLEN* out_buffer_length, sqlsrv_error_auto_ptr& last_error )
{
    Number* number_data = reinterpret_cast<Number*>( buffer ); 
    std::locale loc;    // default locale should match system
    std::basic_istringstream<Char> is;
    is.str( string_data );
    is.imbue( loc );
    std::ios_base::iostate st = 0;

    std::use_facet< std::num_get< Char > >( loc ).get( std::basic_istream<Char>::_Iter( is.rdbuf( ) ), 
                                                       std::basic_istream<Char>::_Iter(0), is, st, *number_data );

    if( st & std::ios_base::failbit ) {
        last_error = new ( sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( 
            (SQLCHAR*) "22003", (SQLCHAR*) "Numeric value out of range", 103 );
        return SQL_ERROR;        
    }

    *out_buffer_length = sizeof( Number );

    return SQL_SUCCESS;
}

// "closure" for the hash table destructor
struct row_dtor_closure {

    sqlsrv_buffered_result_set* results;
    BYTE* row_data;

    row_dtor_closure( sqlsrv_buffered_result_set* st, BYTE* row ) :
        results( st ), row_data( row )
    {
    }                      
};

sqlsrv_error* odbc_get_diag_rec( sqlsrv_stmt* odbc, SQLSMALLINT record_number )
{
    SQLWCHAR wsql_state[ SQL_SQLSTATE_BUFSIZE ];
    SQLWCHAR wnative_message[ SQL_MAX_MESSAGE_LENGTH + 1 ];
    SQLINTEGER native_code;
    SQLSMALLINT wnative_message_len = 0;

    SQLRETURN r = SQLGetDiagRecW( SQL_HANDLE_STMT, odbc->handle(), record_number, wsql_state, &native_code, wnative_message, 
                                  SQL_MAX_MESSAGE_LENGTH + 1, &wnative_message_len );
    if( !SQL_SUCCEEDED( r ) || r == SQL_NO_DATA ) {
        return NULL;
    }

    // convert the error into the encoding of the context
    SQLSRV_ENCODING enc = odbc->encoding();
    if( enc == SQLSRV_ENCODING_DEFAULT ) {
        enc = odbc->conn->encoding();
    }

    // convert the error into the encoding of the context
    sqlsrv_malloc_auto_ptr<SQLCHAR> sql_state;
    SQLLEN sql_state_len = 0;
    if (!convert_string_from_utf16( enc, wsql_state, sizeof(wsql_state), (char**)&sql_state, sql_state_len )) {
        return NULL;
    }
    
    sqlsrv_malloc_auto_ptr<SQLCHAR> native_message;
    SQLLEN native_message_len = 0;
    if (!convert_string_from_utf16( enc, wnative_message, wnative_message_len, (char**)&native_message, native_message_len )) {
        return NULL;
    }

    return new (sqlsrv_malloc( sizeof( sqlsrv_error ))) sqlsrv_error( (SQLCHAR*) sql_state, (SQLCHAR*) native_message, 
                                                                      native_code );
}

}   // namespace

// base class result set

sqlsrv_result_set::sqlsrv_result_set( sqlsrv_stmt* stmt ) :
    odbc( stmt )
{
}


// ODBC result set
// This object simply wraps ODBC function calls

sqlsrv_odbc_result_set::sqlsrv_odbc_result_set( sqlsrv_stmt* stmt ) : 
    sqlsrv_result_set( stmt )
{
}

sqlsrv_odbc_result_set::~sqlsrv_odbc_result_set( void )
{
}

SQLRETURN sqlsrv_odbc_result_set::fetch( SQLSMALLINT orientation, SQLLEN offset TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLFetchScroll( odbc, orientation, offset TSRMLS_CC );
}

SQLRETURN sqlsrv_odbc_result_set::get_data( SQLUSMALLINT field_index, SQLSMALLINT target_type,
                                            _Out_ SQLPOINTER buffer, SQLLEN buffer_length, _Out_ SQLLEN* out_buffer_length,
                                            bool handle_warning TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLGetData( odbc, field_index, target_type, buffer, buffer_length, out_buffer_length, handle_warning TSRMLS_CC );
}

SQLRETURN sqlsrv_odbc_result_set::get_diag_field( SQLSMALLINT record_number, SQLSMALLINT diag_identifier, 
                                                  _Out_ SQLPOINTER diag_info_buffer, SQLSMALLINT buffer_length,
                                                  _Out_ SQLSMALLINT* out_buffer_length TSRMLS_DC )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLGetDiagField( odbc, record_number, diag_identifier, diag_info_buffer, buffer_length, 
                                  out_buffer_length TSRMLS_CC );
}

sqlsrv_error* sqlsrv_odbc_result_set::get_diag_rec( SQLSMALLINT record_number )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return odbc_get_diag_rec( odbc, record_number );
}

SQLLEN sqlsrv_odbc_result_set::row_count( TSRMLS_D )
{
    SQLSRV_ASSERT( odbc != NULL, "Invalid statement handle" );
    return core::SQLRowCount( odbc TSRMLS_CC );
}


// Buffered result set
// This class holds a result set in memory

sqlsrv_buffered_result_set::sqlsrv_buffered_result_set( sqlsrv_stmt* stmt TSRMLS_DC ) :
    sqlsrv_result_set( stmt ),
    cache(NULL),
    col_count(0),
    meta(NULL),
    current(0),
    last_field_index(-1),
    read_so_far(0)
{
    // 10 is an arbitrary number for now for the initial size of the cache
    ALLOC_HASHTABLE( cache );
    core::sqlsrv_zend_hash_init( *stmt, cache, 10 /* # of buckets */, cache_row_dtor /*dtor*/, 0 /*persistent*/ TSRMLS_CC );
    col_count = core::SQLNumResultCols( stmt TSRMLS_CC );
    // there is no result set to buffer
    if( col_count == 0 ) {
        return;
    }

    SQLULEN null_bytes = ( col_count / 8 ) + 1; // number of bits to reserve at the beginning of each row for NULL flags
    meta = static_cast<sqlsrv_buffered_result_set::meta_data*>( sqlsrv_malloc( col_count * 
                                                                               sizeof( sqlsrv_buffered_result_set::meta_data )));

    // set up the conversion matrix if this is the first time we're called
    if( conv_matrix.size() == 0 ) {

        conv_matrix[ SQL_C_CHAR ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::system_to_wide_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_binary_string;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::string_to_double;
        conv_matrix[ SQL_C_CHAR ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::string_to_long;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_binary_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::wide_to_system_string;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::wstring_to_double;
        conv_matrix[ SQL_C_WCHAR ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::wstring_to_long;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_same_string;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::binary_to_system_string;
        conv_matrix[ SQL_C_BINARY ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::binary_to_wide_string;
        conv_matrix[ SQL_C_LONG ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::long_to_double;
        conv_matrix[ SQL_C_LONG ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::to_long;
        conv_matrix[ SQL_C_LONG ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_long;
        conv_matrix[ SQL_C_LONG ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::long_to_system_string;
        conv_matrix[ SQL_C_LONG ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::long_to_wide_string;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_DOUBLE ] = &sqlsrv_buffered_result_set::to_double;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_BINARY ] = &sqlsrv_buffered_result_set::to_double;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_CHAR ] = &sqlsrv_buffered_result_set::double_to_system_string;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_LONG ] = &sqlsrv_buffered_result_set::double_to_long;
        conv_matrix[ SQL_C_DOUBLE ][ SQL_C_WCHAR ] = &sqlsrv_buffered_result_set::double_to_wide_string;
    }

    SQLSRV_ENCODING encoding = (( stmt->encoding() == SQLSRV_ENCODING_DEFAULT ) ? stmt->conn->encoding() :
        stmt->encoding());

    // get the meta data and calculate the size of a row buffer
    SQLULEN offset = null_bytes;
    for( SQLSMALLINT i = 0; i < col_count; ++i ) {

        core::SQLDescribeCol( stmt, i + 1, NULL, 0, NULL, &meta[i].type, &meta[i].length, &meta[i].scale, NULL TSRMLS_CC );

        offset = align_to<4>( offset );
        meta[i].offset = offset;

        switch( meta[i].type ) {

            // these types are the display size
            case SQL_BIGINT:
            case SQL_DECIMAL:
            case SQL_GUID:
            case SQL_NUMERIC:
                core::SQLColAttribute( stmt, i + 1, SQL_DESC_DISPLAY_SIZE, NULL, 0, NULL,
                                       reinterpret_cast<SQLLEN*>( &meta[i].length ) TSRMLS_CC );
                meta[i].length += sizeof( char ) + sizeof( SQLULEN ); // null terminator space
                offset += meta[i].length;
                break;
            case SQL_CHAR:
            case SQL_VARCHAR:
                if ( meta[i].length == sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN ) {
                    offset += sizeof( void* );
                }
                else {
                    // If encoding is set to UTF-8, the following types are not necessarily column size.
                    // We need to call SQLGetData with c_type SQL_C_WCHAR and set the size accordingly. 
                    if ( encoding == SQLSRV_ENCODING_UTF8 ) {
                        meta[i].length *= sizeof( WCHAR );
                        meta[i].length += sizeof( SQLULEN ) + sizeof( WCHAR ); // length plus null terminator space
                        offset += meta[i].length;
                    }
                    else {
                        meta[i].length += sizeof( SQLULEN ) + sizeof( char ); // length plus null terminator space
                        offset += meta[i].length;
                    }
                }
                break;

            // these types are the column size
            case SQL_BINARY:
            case SQL_SS_UDT:
            case SQL_VARBINARY:
                // var* field types are length prefixed
                if( meta[i].length == sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN ) {
                    offset += sizeof( void* );
                }
                else {
                    meta[i].length += sizeof( SQLULEN ) + sizeof( char ); // length plus null terminator space
                    offset += meta[i].length;
                }
                break;

            case SQL_WCHAR:
            case SQL_WVARCHAR:
                if( meta[i].length == sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN ) {
                    offset += sizeof( void* );
                }
                else {
                    meta[i].length *= sizeof( WCHAR );
                    meta[i].length += sizeof( SQLULEN ) + sizeof( WCHAR ); // length plus null terminator space
                    offset += meta[i].length;
                }
                break;

            // these types are LOBs
            case SQL_LONGVARBINARY:
            case SQL_LONGVARCHAR:
            case SQL_WLONGVARCHAR:
            case SQL_SS_XML:
                meta[i].length = sqlsrv_buffered_result_set::meta_data::SIZE_UNKNOWN;
                offset += sizeof( void* );
                break;

            // these types are the ISO date size
            case SQL_DATETIME:
            case SQL_TYPE_DATE:
            case SQL_SS_TIME2:
            case SQL_SS_TIMESTAMPOFFSET:
            case SQL_TYPE_TIMESTAMP:
                core::SQLColAttribute( stmt, i + 1, SQL_DESC_DISPLAY_SIZE, NULL, 0, NULL, 
                                       reinterpret_cast<SQLLEN*>( &meta[i].length ) TSRMLS_CC );
                meta[i].length += sizeof(char) + sizeof( SQLULEN );  // null terminator space
                offset += meta[i].length;
                break;

            // these types are the native size
            case SQL_BIT:
            case SQL_INTEGER:
            case SQL_SMALLINT:
            case SQL_TINYINT:
                meta[i].length = sizeof( long );
                offset += meta[i].length;
                break;

            case SQL_REAL:
            case SQL_FLOAT:
                meta[i].length = sizeof( double );
                offset += meta[i].length;
                break;

            default:
                SQLSRV_ASSERT( false, "Unknown type in sqlsrv_buffered_query::sqlsrv_buffered_query" );
                break;
        }

        switch( meta[i].type ) {

            case SQL_BIGINT:
            case SQL_DATETIME:
            case SQL_DECIMAL:
            case SQL_GUID:
            case SQL_NUMERIC:
            case SQL_TYPE_DATE:
            case SQL_SS_TIME2:
            case SQL_SS_TIMESTAMPOFFSET:
            case SQL_SS_XML:
            case SQL_TYPE_TIMESTAMP:
                meta[i].c_type = SQL_C_CHAR;
                break;
                
            case SQL_CHAR:
            case SQL_VARCHAR:
            case SQL_LONGVARCHAR:
                // If encoding is set to UTF-8, the following types are not necessarily column size.
                // We need to call SQLGetData with c_type SQL_C_WCHAR and set the size accordingly. 
                if ( encoding == SQLSRV_ENCODING_UTF8 ) {
                    meta[i].c_type = SQL_C_WCHAR;
                }
                else {
                    meta[i].c_type = SQL_C_CHAR;
                }
                break;
                
            case SQL_SS_UDT:
            case SQL_LONGVARBINARY:
            case SQL_BINARY:
            case SQL_VARBINARY:
                meta[i].c_type = SQL_C_BINARY;
                break;
{'fraction_non_alphanumeric': 0.10315556375715076, 'fraction_numerical': 0.004587035035457253, 'mean_word_length': 2.4289071680376026, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6414023', 'n_tokens_mistral': 11988, 'n_tokens_neox': 11492, 'n_words': 3056}