<commit_before>#include <sys/types.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include "SyncManagerI.h"
#include "Constants.h"
#include "SyncJSONReader.h"
#include "SyncUtil.h"
#include "SyncBlob.h"

extern char* fetch_remote_data(char *url);
extern int push_remote_data(char* url, char* data, size_t data_size, char* contentType);
extern void delete_db_session(const char *url_string);
extern char *get_database();
extern char *get_client_id();
extern void lock_sync_mutex();
extern void unlock_sync_mutex();

#if defined(_WIN32_WCE)
extern void delete_winmo_session(const char *url_string);
extern char *get_winmo_session_size(const char *url_string);
#define stricmp _stricmp
#endif

static sqlite3_stmt *op_list_source_ids_statement = NULL;
static sqlite3_stmt *source_params_statement = NULL;
static sqlite3_stmt *ob_count_statement = NULL;
static sqlite3_stmt *client_id_statement = NULL;
static sqlite3_stmt *client_id_insert_statement = NULL;
static sqlite3_stmt *client_db_statement = NULL;
static sqlite3_stmt *sync_status_statement = NULL;
static sqlite3_stmt *session_db_statement = NULL;
static sqlite3_stmt *del_session_db_statement = NULL;
static sqlite3_stmt *del_all_client_info_statement = NULL;
static sqlite3_stmt *del_all_ob_val_statement = NULL;
static sqlite3_stmt *session_url_source_statement=NULL;
static sqlite3_stmt *set_session_db_statement=NULL;
static sqlite3_stmt *update_token_db_statement=NULL;

void finalize_sync_util_statements() {
	if (op_list_source_ids_statement) {
		sqlite3_finalize(op_list_source_ids_statement);
		op_list_source_ids_statement = NULL;
	}
	if (source_params_statement) {
		sqlite3_finalize(source_params_statement);
		source_params_statement = NULL;
	}
	if (ob_count_statement) {
		sqlite3_finalize(ob_count_statement);
		ob_count_statement = NULL;
	}
	if (client_id_statement) {
		sqlite3_finalize(client_id_statement);
		client_id_statement = NULL;
	}
	if (client_id_insert_statement) {
		sqlite3_finalize(client_id_insert_statement);
		client_id_insert_statement = NULL;
	}
	if (client_db_statement) {
		sqlite3_finalize(client_db_statement);
		client_db_statement = NULL;
	}
	if (sync_status_statement) {
		sqlite3_finalize(sync_status_statement);
		sync_status_statement = NULL;
	}
	if (session_db_statement) {
		sqlite3_finalize(session_db_statement);
		session_db_statement = NULL;
	}
	if (del_session_db_statement) {
		sqlite3_finalize(del_session_db_statement);
		del_session_db_statement = NULL;
	}
	if (del_all_ob_val_statement) {
		sqlite3_finalize(del_all_ob_val_statement);
		del_all_ob_val_statement = NULL;
	}
	if (del_all_client_info_statement) {
		sqlite3_finalize(del_all_client_info_statement);
		del_all_client_info_statement = NULL;
	}
	if (session_url_source_statement) {
		sqlite3_finalize(session_url_source_statement);
		session_url_source_statement = NULL;
	}
	if (set_session_db_statement) {
		sqlite3_finalize(set_session_db_statement);
		set_session_db_statement = NULL;
	}
	if (update_token_db_statement) {
		sqlite3_finalize(update_token_db_statement);
		update_token_db_statement = NULL;
	}

}

/**
 * This global buffer will store source url between requests
 * It can be used by other routines to understand what is the original source url. To avoid parsing etc...
 */
static char g_source_url[1024] = {0};

void save_source_url(const char* source_url)
{
	int size = 0; 
	
	memset( g_source_url, 0, sizeof(g_source_url));
	
	if ( source_url )
	{
		size = strlen(source_url);
		
		if ( (size + 1 < sizeof(g_source_url) ) )
			memcpy( (void*)g_source_url, (const void*)source_url, size);
	}
}

const char* load_source_url()
{
	return g_source_url;
}

static void processToken( sqlite_uint64 token, pSource src );
/* 
 * Pulls the latest object_values list 
 * for a given source and populates a list
 * of sync objects in memory and the database.
 */
int fetch_remote_changes(sqlite3 *database, char *client_id, pSource src, char *params) {
	pSyncObject *list;
	char url_string[4096];
	int j, nTotal = 0;//,i,source_length;
	char *json_string;
	char *type = NULL;
	int success=0, size_deleted=0, size_inserted=0, repeat=1;
	double start=0, duration = 0;
    SyncHeader header;
    int nTry = 0;

    header._count = -1;
    header._token = 0;

    if ( !src->_source_url || strlen(src->_source_url) == 0 )
        return nTotal;

	save_source_url(src->_source_url);
	start = time(NULL);

    list = malloc(MAX_SYNC_OBJECTS*sizeof(pSyncObject));
    if ( list )
    {
        do {
	        if (params && strlen(params) > 0) {
				// Don't repeat if we're calling ask method
				repeat=0;
		        sprintf(url_string, 
				        "%s%s%s&client_id=%s&question=%s&p_size=%d", 
				        src->_source_url,
				        SYNC_ASK_ACTION,
				        SYNC_SOURCE_FORMAT, 
				        client_id,
				        params, SYNC_PAGE_SIZE );
	        } else {
		        sprintf(url_string, 
				        "%s%s&client_id=%s&p_size=%d", 
				        src->_source_url,
				        SYNC_SOURCE_FORMAT, 
				        client_id, SYNC_PAGE_SIZE);
	        }
            if ( header._token != 0 ){
                char szToken[30];
                sprintf(szToken, "&ack_token=%<PASSWORD>",  header._token );
                strcat(url_string,szToken);
            }

	        json_string = fetch_remote_data(url_string);
	        if(json_string && strlen(json_string) > 0) {
		        struct json_object* json_to_free = 0;
		        int available = parse_json_list(list, json_string, MAX_SYNC_OBJECTS, &json_to_free, &header);
		        printf("Parsed %i records from sync source...\n", available);
                nTotal += available;
		        if(available > 0) {

                    processToken( header._token, src );

			        for(j = 0; j < available; j++) {
				        list[j]->_database = database;
				        type = list[j]->_db_operation;<commit_msg>fixing uint64 formatting for iphone<commit_after>#include <sys/types.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include "SyncManagerI.h"
#include "Constants.h"
#include "SyncJSONReader.h"
#include "SyncUtil.h"
#include "SyncBlob.h"

extern char* fetch_remote_data(char *url);
extern int push_remote_data(char* url, char* data, size_t data_size, char* contentType);
extern void delete_db_session(const char *url_string);
extern char *get_database();
extern char *get_client_id();
extern void lock_sync_mutex();
extern void unlock_sync_mutex();

#if defined(_WIN32_WCE)
extern void delete_winmo_session(const char *url_string);
extern char *get_winmo_session_size(const char *url_string);
#define stricmp _stricmp
#endif

static sqlite3_stmt *op_list_source_ids_statement = NULL;
static sqlite3_stmt *source_params_statement = NULL;
static sqlite3_stmt *ob_count_statement = NULL;
static sqlite3_stmt *client_id_statement = NULL;
static sqlite3_stmt *client_id_insert_statement = NULL;
static sqlite3_stmt *client_db_statement = NULL;
static sqlite3_stmt *sync_status_statement = NULL;
static sqlite3_stmt *session_db_statement = NULL;
static sqlite3_stmt *del_session_db_statement = NULL;
static sqlite3_stmt *del_all_client_info_statement = NULL;
static sqlite3_stmt *del_all_ob_val_statement = NULL;
static sqlite3_stmt *session_url_source_statement=NULL;
static sqlite3_stmt *set_session_db_statement=NULL;
static sqlite3_stmt *update_token_db_statement=NULL;

void finalize_sync_util_statements() {
	if (op_list_source_ids_statement) {
		sqlite3_finalize(op_list_source_ids_statement);
		op_list_source_ids_statement = NULL;
	}
	if (source_params_statement) {
		sqlite3_finalize(source_params_statement);
		source_params_statement = NULL;
	}
	if (ob_count_statement) {
		sqlite3_finalize(ob_count_statement);
		ob_count_statement = NULL;
	}
	if (client_id_statement) {
		sqlite3_finalize(client_id_statement);
		client_id_statement = NULL;
	}
	if (client_id_insert_statement) {
		sqlite3_finalize(client_id_insert_statement);
		client_id_insert_statement = NULL;
	}
	if (client_db_statement) {
		sqlite3_finalize(client_db_statement);
		client_db_statement = NULL;
	}
	if (sync_status_statement) {
		sqlite3_finalize(sync_status_statement);
		sync_status_statement = NULL;
	}
	if (session_db_statement) {
		sqlite3_finalize(session_db_statement);
		session_db_statement = NULL;
	}
	if (del_session_db_statement) {
		sqlite3_finalize(del_session_db_statement);
		del_session_db_statement = NULL;
	}
	if (del_all_ob_val_statement) {
		sqlite3_finalize(del_all_ob_val_statement);
		del_all_ob_val_statement = NULL;
	}
	if (del_all_client_info_statement) {
		sqlite3_finalize(del_all_client_info_statement);
		del_all_client_info_statement = NULL;
	}
	if (session_url_source_statement) {
		sqlite3_finalize(session_url_source_statement);
		session_url_source_statement = NULL;
	}
	if (set_session_db_statement) {
		sqlite3_finalize(set_session_db_statement);
		set_session_db_statement = NULL;
	}
	if (update_token_db_statement) {
		sqlite3_finalize(update_token_db_statement);
		update_token_db_statement = NULL;
	}

}

/**
 * This global buffer will store source url between requests
 * It can be used by other routines to understand what is the original source url. To avoid parsing etc...
 */
static char g_source_url[1024] = {0};

void save_source_url(const char* source_url)
{
	int size = 0; 
	
	memset( g_source_url, 0, sizeof(g_source_url));
	
	if ( source_url )
	{
		size = strlen(source_url);
		
		if ( (size + 1 < sizeof(g_source_url) ) )
			memcpy( (void*)g_source_url, (const void*)source_url, size);
	}
}

const char* load_source_url()
{
	return g_source_url;
}

static void processToken( sqlite_uint64 token, pSource src );
/* 
 * Pulls the latest object_values list 
 * for a given source and populates a list
 * of sync objects in memory and the database.
 */
int fetch_remote_changes(sqlite3 *database, char *client_id, pSource src, char *params) {
	pSyncObject *list;
	char url_string[4096];
	int j, nTotal = 0;//,i,source_length;
	char *json_string;
	char *type = NULL;
	int success=0, size_deleted=0, size_inserted=0, repeat=1;
	double start=0, duration = 0;
    SyncHeader header;
    int nTry = 0;

    header._count = -1;
    header._token = 0;

    if ( !src->_source_url || strlen(src->_source_url) == 0 )
        return nTotal;

	save_source_url(src->_source_url);
	start = time(NULL);

    list = malloc(MAX_SYNC_OBJECTS*sizeof(pSyncObject));
    if ( list )
    {
        do {
	        if (params && strlen(params) > 0) {
				// Don't repeat if we're calling ask method
				repeat=0;
		        sprintf(url_string, 
				        "%s%s%s&client_id=%s&question=%s&p_size=%d", 
				        src->_source_url,
				        SYNC_ASK_ACTION,
				        SYNC_SOURCE_FORMAT, 
				        client_id,
				        params, SYNC_PAGE_SIZE );
	        } else {
		        sprintf(url_string, 
				        "%s%s&client_id=%s&p_size=%d", 
				        src->_source_url,
				        SYNC_SOURCE_FORMAT, 
				        client_id, SYNC_PAGE_SIZE);
	        }
            if ( header._token != 0 ){
                char szToken[30];
#if defined(_WIN32_WCE)
                sprintf(szToken, "&ack_token=%<PASSWORD>",  header._token );
#else
				sprintf(szToken, "&ack_token=%llu",  header._token );
#endif
                strcat(url_string,szToken);
            }

	        json_string = fetch_remote_data(url_string);
	        if(json_string && strlen(json_string) > 0) {
		        struct json_object* json_to_free = 0;
		        int available = parse_json_list(list, json_string, MAX_SYNC_OBJECTS, &json_to_free, &header);
		        printf("Parsed %i records from sync source...\n", available);
                nTotal += available;
		        if(available > 0) {

                    processToken( header._token, src );

			        for(j = 0; j < available; j++) {
				        list[j]->_database = database;
				        type = list[j]->_db_operation;
{'fraction_non_alphanumeric': 0.09880720751205482, 'fraction_numerical': 0.011335758396074782, 'mean_word_length': 4.128850325379609, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2961808', 'n_tokens_mistral': 4379, 'n_tokens_neox': 4022, 'n_words': 826}