<commit_before>/* ***** BEGIN LICENSE BLOCK *****
 *
 * BBN Address and AS Number PKI Database/repository software
 * Version 1.0
 *
 * US government users are permitted unrestricted rights as
 * defined in the FAR.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT
 * WARRANTY OF ANY KIND, either express or implied.
 *
 * Copyright (C) Raytheon BBN Technologies Corp. 2007.  All Rights Reserved.
 *
 * Contributor(s):  <NAME>
 *
 * ***** END LICENSE BLOCK ***** */

/*
  $Id: query.c 857 2009-09-30 15:27:40Z dmontana $
*/

/************************
 * Get the next round of RTR data into the database
 ***********************/

#include "err.h"
#include "scmf.h"
#include "querySupport.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>

// number of hours to retain incremental updates
// should be at least 24 + the maximum time between updates, since need
//   to always have not just all those within the last 24 hours but also
//   one more beyond these
#define RETENTION_HOURS_DEFAULT 96

static scm      *scmp = NULL;
static scmcon   *connection = NULL;
static scmsrcha *roaSrch = NULL;
static scmtab   *roaTable = NULL;
static scmtab   *nonceTable = NULL;
static scmtab   *fullTable = NULL;
static scmtab   *updateTable = NULL;
static scmsrcha *snSrch = NULL;

// serial number of this and previous update
static uint prevSerialNum, currSerialNum, lastSerialNum;

static void setupSnQuery(scm *scmp) {
	snSrch = newsrchscm(NULL, 1, 0, 1);
	addcolsrchscm(snSrch, "serial_num", SQL_C_ULONG, 8);
	snSrch->wherestr = NULL;
	updateTable = findtablescm(scmp, "rtr_update");
	if (updateTable == NULL) printf("Cannot find table rtr_update\n");
}

/* helper function for getLastSerialNumber */
static int setLastSN(scmcon *conp, scmsrcha *s, int numLine) {
	(void)conp;
	(void)numLine;
	lastSerialNum = *((uint *) (s->vec[0].valptr));
	return -1;    // stop after first row
}

/****
 * find the serial number from the most recent update
 ****/
static uint getLastSerialNumber(scmcon *connect, scm *scmp) {
	lastSerialNum = 0;
	if (snSrch == NULL) setupSnQuery(scmp);
	searchscm (connect, updateTable, snSrch, NULL, setLastSN,
			   SCM_SRCH_DOVALUE_ALWAYS | SCM_SRCH_BREAK_VERR,
			   "create_time desc");
	return lastSerialNum;
}

/*****
 * allows overriding of retention time for data via environment variable
 *****/
static int retentionHours() {
	if (getenv("RTR_RETENTION_HOURS") != NULL)
		return atoi(getenv("RTR_RETENTION_HOURS"));
	return RETENTION_HOURS_DEFAULT;
}


/******
 * callback that writes the data from a ROA into the update table
 *   if the ROA is valid
 *****/
static int writeROAData(scmcon *conp, scmsrcha *s, int numLine) {
	uint asn = *((uint *)s->vec[0].valptr);
	char *ptr = (char *)s->vec[1].valptr, *end;
	char msg[1024];
	conp = conp; numLine = numLine;

	if (! checkValidity((char *)s->vec[2].valptr, 0, scmp, connection)) return -1;
	while ((end = strstr(ptr, ", ")) != NULL) {
		end[0] = '\0';
		end[1] = '\0';
		snprintf(msg, sizeof(msg),
				 "insert ignore into %s values (%u, %u, \"%s\");",
				 fullTable->tabname, currSerialNum, asn, ptr);
		statementscm_no_data(connection, msg);
		ptr = end + 2;
	}
	if (ptr[0] != '\0') {
		snprintf(msg, sizeof(msg),
				 "insert ignore into %s values (%u, %u, \"%s\");",
				 fullTable->tabname, currSerialNum, asn, ptr);
		statementscm_no_data(connection, msg);
	}
	return 1;
}


int main(int argc, char **argv) {
	char msg[1024];
	int sta;
	uint nonce_count;
	int first_time = 0;

	if (argc != 2)
	{
		fprintf(stderr, "Usage: %s <staleness spec file>\n", argv[0]);
		return EXIT_FAILURE;
	}

	// initialize the database connection
	scmp = initscm();
	checkErr(scmp == NULL, "Cannot initialize database schema\n");
	connection = connectscm (scmp->dsn, msg, sizeof(msg));
	checkErr(connection == NULL, "Cannot connect to database: %s\n", msg);

	nonceTable = findtablescm(scmp, "rtr_nonce");
	checkErr(nonceTable == NULL, "Cannot find table rtr_nonce\n");

	sta = newhstmt(connection);
	checkErr(!SQLOK(sta), "Can't create a new statement handle\n");
	sta = statementscm(connection, "SELECT COUNT(*) FROM rtr_nonce;");
	checkErr(sta < 0, "Can't query rtr_nonce\n");
	sta = getuintscm(connection, &nonce_count);
	pophstmt(connection);
	checkErr(sta < 0, "Can't get results of querying rtr_nonce\n");
	if (nonce_count != 1) {
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_nonce;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_update;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_full;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_incremental;");
		statementscm_no_data(connection, "INSERT INTO rtr_nonce (cache_nonce) VALUES (FLOOR(RAND() * (1 << 16)));");
		first_time = 1;
	}

	// find the last serial number
	prevSerialNum = getLastSerialNumber(connection, scmp);
	currSerialNum = (prevSerialNum == UINT_MAX) ? 0 : (prevSerialNum + 1);

	// setup up the query if this is the first time
	// note that the where string is set to only select valid roa's, where
    //   the definition of valid is given by the staleness specs
	if (roaSrch == NULL) {
		QueryField *field;
		roaSrch = newsrchscm(NULL, 3, 0, 1);
		field = findField("asn");
		addcolsrchscm(roaSrch, "asn", field->sqlType, field->maxSize);
		field = findField("ip_addrs");
		addcolsrchscm(roaSrch, "ip_addrs", field->sqlType, field->maxSize);
		field = findField("ski");
		addcolsrchscm(roaSrch, "ski", field->sqlType, field->maxSize);
		roaSrch->wherestr[0] = 0;
		parseStalenessSpecsFile(argv[1]);
		addQueryFlagTests(roaSrch->wherestr, 0);
		roaTable = findtablescm(scmp, "roa");
		checkErr(roaTable == NULL, "Cannot find table roa\n");
		fullTable = findtablescm(scmp, "rtr_full");
		checkErr(fullTable == NULL, "Cannot find table rtr_full\n");
	}

	// write all the data into the database (done writing "full")<commit_msg>delete unfinished updates before trying again<commit_after>/* ***** BEGIN LICENSE BLOCK *****
 *
 * BBN Address and AS Number PKI Database/repository software
 * Version 1.0
 *
 * US government users are permitted unrestricted rights as
 * defined in the FAR.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT
 * WARRANTY OF ANY KIND, either express or implied.
 *
 * Copyright (C) Raytheon BBN Technologies Corp. 2007.  All Rights Reserved.
 *
 * Contributor(s):  <NAME>
 *
 * ***** END LICENSE BLOCK ***** */

/*
  $Id: query.c 857 2009-09-30 15:27:40Z dmontana $
*/

/************************
 * Get the next round of RTR data into the database
 ***********************/

#include "err.h"
#include "scmf.h"
#include "querySupport.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>

// number of hours to retain incremental updates
// should be at least 24 + the maximum time between updates, since need
//   to always have not just all those within the last 24 hours but also
//   one more beyond these
#define RETENTION_HOURS_DEFAULT 96

static scm      *scmp = NULL;
static scmcon   *connection = NULL;
static scmsrcha *roaSrch = NULL;
static scmtab   *roaTable = NULL;
static scmtab   *nonceTable = NULL;
static scmtab   *fullTable = NULL;
static scmtab   *updateTable = NULL;
static scmsrcha *snSrch = NULL;

// serial number of this and previous update
static uint prevSerialNum, currSerialNum, lastSerialNum;

static void setupSnQuery(scm *scmp) {
	snSrch = newsrchscm(NULL, 1, 0, 1);
	addcolsrchscm(snSrch, "serial_num", SQL_C_ULONG, 8);
	snSrch->wherestr = NULL;
	updateTable = findtablescm(scmp, "rtr_update");
	if (updateTable == NULL) printf("Cannot find table rtr_update\n");
}

/* helper function for getLastSerialNumber */
static int setLastSN(scmcon *conp, scmsrcha *s, int numLine) {
	(void)conp;
	(void)numLine;
	lastSerialNum = *((uint *) (s->vec[0].valptr));
	return -1;    // stop after first row
}

/****
 * find the serial number from the most recent update
 ****/
static uint getLastSerialNumber(scmcon *connect, scm *scmp) {
	lastSerialNum = 0;
	if (snSrch == NULL) setupSnQuery(scmp);
	searchscm (connect, updateTable, snSrch, NULL, setLastSN,
			   SCM_SRCH_DOVALUE_ALWAYS | SCM_SRCH_BREAK_VERR,
			   "create_time desc");
	return lastSerialNum;
}

/*****
 * allows overriding of retention time for data via environment variable
 *****/
static int retentionHours() {
	if (getenv("RTR_RETENTION_HOURS") != NULL)
		return atoi(getenv("RTR_RETENTION_HOURS"));
	return RETENTION_HOURS_DEFAULT;
}


/******
 * callback that writes the data from a ROA into the update table
 *   if the ROA is valid
 *****/
static int writeROAData(scmcon *conp, scmsrcha *s, int numLine) {
	uint asn = *((uint *)s->vec[0].valptr);
	char *ptr = (char *)s->vec[1].valptr, *end;
	char msg[1024];
	conp = conp; numLine = numLine;

	if (! checkValidity((char *)s->vec[2].valptr, 0, scmp, connection)) return -1;
	while ((end = strstr(ptr, ", ")) != NULL) {
		end[0] = '\0';
		end[1] = '\0';
		snprintf(msg, sizeof(msg),
				 "insert ignore into %s values (%u, %u, \"%s\");",
				 fullTable->tabname, currSerialNum, asn, ptr);
		statementscm_no_data(connection, msg);
		ptr = end + 2;
	}
	if (ptr[0] != '\0') {
		snprintf(msg, sizeof(msg),
				 "insert ignore into %s values (%u, %u, \"%s\");",
				 fullTable->tabname, currSerialNum, asn, ptr);
		statementscm_no_data(connection, msg);
	}
	return 1;
}


int main(int argc, char **argv) {
	char msg[1024];
	int sta;
	uint nonce_count;
	int first_time = 0;

	if (argc != 2)
	{
		fprintf(stderr, "Usage: %s <staleness spec file>\n", argv[0]);
		return EXIT_FAILURE;
	}

	// initialize the database connection
	scmp = initscm();
	checkErr(scmp == NULL, "Cannot initialize database schema\n");
	connection = connectscm (scmp->dsn, msg, sizeof(msg));
	checkErr(connection == NULL, "Cannot connect to database: %s\n", msg);

	nonceTable = findtablescm(scmp, "rtr_nonce");
	checkErr(nonceTable == NULL, "Cannot find table rtr_nonce\n");

	sta = newhstmt(connection);
	checkErr(!SQLOK(sta), "Can't create a new statement handle\n");
	sta = statementscm(connection, "SELECT COUNT(*) FROM rtr_nonce;");
	checkErr(sta < 0, "Can't query rtr_nonce\n");
	sta = getuintscm(connection, &nonce_count);
	pophstmt(connection);
	checkErr(sta < 0, "Can't get results of querying rtr_nonce\n");
	if (nonce_count != 1) {
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_nonce;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_update;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_full;");
		statementscm_no_data(connection, "TRUNCATE TABLE rtr_incremental;");
		statementscm_no_data(connection, "INSERT INTO rtr_nonce (cache_nonce) VALUES (FLOOR(RAND() * (1 << 16)));");
		first_time = 1;
	}

	// delete any updates that weren't completed
	statementscm_no_data(connection,
		 "delete rtr_incremental\n"
		 "from rtr_incremental\n"
		 "left join rtr_update on rtr_incremental.serial_num = rtr_update.serial_num\n"
		 "where rtr_update.serial_num is null;");
	statementscm_no_data(connection,
		 "delete rtr_full\n"
		 "from rtr_full\n"
		 "left join rtr_update on rtr_full.serial_num = rtr_update.serial_num\n"
		 "where rtr_update.serial_num is null;");

	// find the last serial number
	prevSerialNum = getLastSerialNumber(connection, scmp);
	currSerialNum = (prevSerialNum == UINT_MAX) ? 0 : (prevSerialNum + 1);

	// setup up the query if this is the first time
	// note that the where string is set to only select valid roa's, where
    //   the definition of valid is given by the staleness specs
	if (roaSrch == NULL) {
		QueryField *field;
		roaSrch = newsrchscm(NULL, 3, 0, 1);
		field = findField("asn");
		addcolsrchscm(roaSrch, "asn", field->sqlType, field->maxSize);
		field = findField("ip_addrs");
		addcolsrchscm(roaSrch, "ip_addrs", field->sqlType, field->maxSize);
		field = findField("ski");
		addcolsrchscm(roaSrch, "ski", field->sqlType, field->maxSize);
		roaSrch->wherestr[0] = 0;
		parseStalenessSpecsFile(argv[1]);
		addQueryFlagTests(roaSrch->wherestr, 0);
		roaTable = findtablescm(scmp, "roa");
		checkErr(roaTable == NULL, "Cannot find table roa\n");
		fullTable = findtablescm(scmp, "rtr_full");
		checkErr(fullTable == NULL, "Cannot find table rtr_full\n");
	}

	// write all the data into the database (done writing "full")
{'fraction_non_alphanumeric': 0.1458401572224042, 'fraction_numerical': 0.012446773665247298, 'mean_word_length': 4.53375623017671, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2523959', 'n_tokens_mistral': 4679, 'n_tokens_neox': 4312, 'n_words': 1250}