<commit_before>/*
 * link-generator.c
 *
 * Generate random corpora from dictionaries.
 * February 2021
 */

#include <argp.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>

#include "../link-grammar/link-includes.h"

/* Argument parsing for the generator */
typedef struct
{
	const char* language;
	int sentence_length;
	int corpus_size;
} gen_parameters;

static struct argp_option options[] =
{
	{"length", 'l', 0, 0, "Sentence length."},
	{"size", 's', 0, 0, "Corpus size."},
	{"version", 'v', 0, 0, "Print version and exit."},
	{ 0 }
};

static error_t parse_opt(int key, char *arg, struct argp_state *state)
{
	gen_parameters* gp = state->input;
	switch (key)
	{
		case 'l': gp->sentence_length = atoi(arg); break;
		case 's': gp->corpus_size = atoi(arg); break;

		case 'v':
		{
			printf("Version: %s\n", linkgrammar_get_version());
			printf("%s\n", linkgrammar_get_configuration());
			exit(0);
		}

		case ARGP_KEY_ARG: return 0;
		default: return ARGP_ERR_UNKNOWN;
	}
	return 0;
}

static char args_doc[] = "<language|dictionary>";
static struct argp argp = { options, parse_opt, args_doc, 0, 0, 0 };

int main (int argc, char* argv[])
{
	const char     *language = NULL;
	Dictionary      dict;
	Parse_Options   opts;
	Sentence        sent = NULL;

	/* Process options used by GNU programs. */
	gen_parameters parms;
	parms.language = "lt";
	parms.sentence_length = 6;
	parms.corpus_size = 50;
	argp_parse(&argp, argc, argv, 0, 0, &parms);

	if ((argc > 1) && (argv[1][0] != '-')) {
		/* The dictionary is the first argument if it doesn't begin with "-" */
		language = argv[1];
	}

	if (language && *language)
	{
		printf("#\n# Corpus for language \"%s\"\n", language);

		// Force the system into generation mode by appending "-generate"
		// to the langauge. XXX this seems hacky, need a better API.
		char lan[100]; // buffer overflow
		lan[0] = 0;
		strcat(lan, language);
		strcat(lan, "-generate");

		dict = dictionary_create_lang(lan);
		if (dict == NULL)
		{
			prt_error("Fatal error: Unable to open dictionary.\n");
			exit(-1);
		}
	}
	else
	{
		prt_error("Fatal error: A language must be specified!\n");
		exit(-1);
	}

	opts = parse_options_create();
	parse_options_set_linkage_limit(opts, 350);

	sent = sentence_create("6", dict);
	// sentence_split(sent, opts);<commit_msg>Convert the arguments to argp<commit_after>/*
 * link-generator.c
 *
 * Generate random corpora from dictionaries.
 * February 2021
 */

#include <argp.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>

#include "../link-grammar/link-includes.h"

/* Argument parsing for the generator */
typedef struct
{
	const char* language;
	int sentence_length;
	int corpus_size;
} gen_parameters;

static struct argp_option options[] =
{
	{"length", 'l', 0, 0, "Sentence length."},
	{"size", 's', 0, 0, "Corpus size."},
	{"version", 'v', 0, 0, "Print version and exit."},
	{ 0 }
};

static error_t parse_opt(int key, char *arg, struct argp_state *state)
{
	gen_parameters* gp = state->input;
	switch (key)
	{
		case 'l': gp->sentence_length = atoi(arg); break;
		case 's': gp->corpus_size = atoi(arg); break;

		case 'v':
		{
			printf("Version: %s\n", linkgrammar_get_version());
			printf("%s\n", linkgrammar_get_configuration());
			exit(0);
		}

		case ARGP_KEY_ARG: return 0;
		default: return ARGP_ERR_UNKNOWN;
	}
	return 0;
}

static char args_doc[] = "<language|dictionary>";
static struct argp argp = { options, parse_opt, args_doc, 0, 0, 0 };

int main (int argc, char* argv[])
{
	Dictionary      dict;
	Parse_Options   opts;
	Sentence        sent = NULL;

	/* Process options used by GNU programs. */
	gen_parameters parms;
	parms.language = "lt";
	parms.sentence_length = 6;
	parms.corpus_size = 50;
	argp_parse(&argp, argc, argv, 0, 0, &parms);

	printf("#\n# Corpus for language \"%s\"\n", parms.language);

	// Force the system into generation mode by appending "-generate"
	// to the langauge. XXX this seems hacky, need a better API.
	char lan[100]; // buffer overflow
	lan[0] = 0;
	strcat(lan, parms.language);
	strcat(lan, "-generate");

	dict = dictionary_create_lang(lan);
	if (dict == NULL)
	{
		prt_error("Fatal error: Unable to open dictionary.\n");
		exit(-1);
	}

	opts = parse_options_create();
	parse_options_set_linkage_limit(opts, 350);

	// Set the number of words in the sentence.
	// XXX this is a hacky API. Fix it.
	char slen[30];
	snprintf(slen, 30, "%d", parms.sentence_length);
	sent = sentence_create(slen, dict);

	// sentence_split(sent, opts);
{'fraction_non_alphanumeric': 0.15719360568383658, 'fraction_numerical': 0.015985790408525755, 'mean_word_length': 3.865010799136069, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1049469', 'n_tokens_mistral': 1762, 'n_tokens_neox': 1610, 'n_words': 466}