<commit_before>/***************************************************************************/
/* Copyright (c) 2004                                                      */
/* <NAME>, <NAME>, and <NAME>                      */
/* Copyright (c) 2008, 2014 <NAME>                                  */
/* All rights reserved                                                     */
/*                                                                         */
/* Use of the link grammar parsing system is subject to the terms of the   */
/* license set forth in the LICENSE file included with this software.      */
/* This license allows free redistribution and use in source and binary    */
/* forms, with or without modification, subject to certain conditions.     */
/*                                                                         */
/***************************************************************************/

 /****************************************************************************
 *
 *   This is a simple example of the link parser API.  It simulates most of
 *   the functionality of the original link grammar parser, allowing sentences
 *   to be typed in either interactively or in "batch" mode (if -batch is
 *   specified on the command line, and stdin is redirected to a file).
 *   The program:
 *     Opens up a dictionary
 *     Iterates:
 *        1. Reads from stdin to get an input string to parse
 *        2. Tokenizes the string to form a Sentence
 *        3. Tries to parse it with cost 0
 *        4. Tries to parse with increasing cost
 *     When a parse is found:
 *        1. Extracts each Linkage
 *        2. Passes it to process_some_linkages()
 *        3. Deletes linkage
 *     After parsing each Sentence is deleted by making a call to
 *     sentence_delete.
 *
 ****************************************************************************/

#include <errno.h>
#include <locale.h>
#include <stdlib.h>
#include <string.h>

/* Used for terminal resizing */
#ifndef _WIN32
#include <langinfo.h>
#include <termios.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#else
#include <windows.h>
#include <wchar.h>
#endif /* _WIN32 */

#ifdef _MSC_VER
#define LINK_GRAMMAR_DLL_EXPORT 0
#endif /* _MSC_VER */

#ifndef _WIN32
#define LAST_RESORT_LOCALE "en_US.UTF-8" /* Supposing POSIX systems */
#else
#define LAST_RESORT_LOCALE ""            /* Use user default locale */
#endif /* _WIN32 */

#include "parser-utilities.h"
#include "command-line.h"
#include "lg_readline.h"
#ifdef USE_VITERBI
#include "../viterbi/viterbi.h"
#endif /* USE_VITERBI */

#define DISPLAY_MAX 1024
#define COMMENT_CHAR '%'  /* input lines beginning with this are ignored */

static int batch_errors = 0;
static int verbosity = 0;
static char * debug = (char *)"";
static char * test = (char *)"";
static bool isatty_stdin, isatty_stdout;
#ifdef _WIN32
static bool running_under_cygwin = false;
#endif /* _WIN32 */

typedef enum
{
	UNGRAMMATICAL = '*',
	PARSE_WITH_DISJUNCT_COST_GT_0 = ':',  /* Not used anywhere, currently ... */
	NO_LABEL = ' '
} Label;


#ifdef _WIN32
/* Windows console (cmd.exe) input to utf8 */
static char* oem_to_utf8(char *instring)
{
	char * out;
	wchar_t *winput;
	size_t len;
	int cv;
	unsigned int consolecp;

	consolecp = GetConsoleOutputCP();

	/* Convert input string to wide chars. */
	len = strlen(instring) + 1;
	cv = MultiByteToWideChar(consolecp, 0, instring, len, NULL, 0);
	winput = (wchar_t*) malloc(cv * sizeof(wchar_t));
	cv = MultiByteToWideChar(consolecp, 0, instring, len, winput, cv);

	/* Convert wide chars to utf8. */
	cv = WideCharToMultiByte(CP_UTF8, 0, winput, len, NULL, 0, NULL, NULL);
	out = (char*) malloc(cv);
	cv = WideCharToMultiByte(CP_UTF8, 0, winput, len, out, cv, NULL, NULL);

	free(winput);

	return out;
}
#endif /* _WIN32 */

static char *
fget_input_string(FILE *in, FILE *out, bool check_return)
{
	static char input_string[MAX_INPUT];
	static bool input_pending = false;

	if ((in != stdin) || !isatty_stdin)
	{
		/* Here the input is not from a terminal. */
		if (check_return) return (char *)"x"; /* XXX One linkage per sentence. */
		if (fgets(input_string, MAX_INPUT, in))
		{
			char *lf = strchr(input_string, '\n');
			if (lf) *lf = '\0';
			return input_string;
		}
		return NULL;
	}

	/* If we are here, the input is from a terminal. */
	static char *pline;
	const char *prompt = (0 == verbosity)? "" : "linkparser> ";

	if (input_pending)
	{
		input_pending = false;
		return pline;
	}

#ifdef HAVE_EDITLINE
	#ifdef _WIN32
		#error __FILE__ ": Cannot use HAVE_EDITLINE "
		                "(the console already has line editing and history)."
	#endif /* _WIN32 */
	pline = lg_readline(prompt);
#else
	fprintf(out, prompt);
	fflush(out);
#ifdef _WIN32
	if (!running_under_cygwin)
		pline = get_console_line();
	else
		pline = fgets(input_string, MAX_INPUT, in);
#else
	pline = fgets(input_string, MAX_INPUT, in);
#endif /* _WIN32 */
#endif /* HAVE_EDITLINE */

	if (NULL == pline) return NULL;                /* EOF */
	if (check_return)
	{
		if ('\0' == pline[0]) return (char *)"\n"; /* Continue linkage display */
		input_pending = true;
		return (char *)"x";                        /* Stop linkage display */
	}

	return pline;
}

/**************************************************************************
*
*  This procedure displays a linkage graphically.  Since the diagrams
*  are passed as character strings, they need to be deleted with a
*  call to free.
*
**************************************************************************/<commit_msg>Check for long input lines.<commit_after>/***************************************************************************/
/* Copyright (c) 2004                                                      */
/* <NAME>, <NAME>, and <NAME>                      */
/* Copyright (c) 2008, 2014 <NAME>                                  */
/* All rights reserved                                                     */
/*                                                                         */
/* Use of the link grammar parsing system is subject to the terms of the   */
/* license set forth in the LICENSE file included with this software.      */
/* This license allows free redistribution and use in source and binary    */
/* forms, with or without modification, subject to certain conditions.     */
/*                                                                         */
/***************************************************************************/

 /****************************************************************************
 *
 *   This is a simple example of the link parser API.  It simulates most of
 *   the functionality of the original link grammar parser, allowing sentences
 *   to be typed in either interactively or in "batch" mode (if -batch is
 *   specified on the command line, and stdin is redirected to a file).
 *   The program:
 *     Opens up a dictionary
 *     Iterates:
 *        1. Reads from stdin to get an input string to parse
 *        2. Tokenizes the string to form a Sentence
 *        3. Tries to parse it with cost 0
 *        4. Tries to parse with increasing cost
 *     When a parse is found:
 *        1. Extracts each Linkage
 *        2. Passes it to process_some_linkages()
 *        3. Deletes linkage
 *     After parsing each Sentence is deleted by making a call to
 *     sentence_delete.
 *
 ****************************************************************************/

#include <errno.h>
#include <locale.h>
#include <stdlib.h>
#include <string.h>

/* Used for terminal resizing */
#ifndef _WIN32
#include <langinfo.h>
#include <termios.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#else
#include <windows.h>
#include <wchar.h>
#endif /* _WIN32 */

#ifdef _MSC_VER
#define LINK_GRAMMAR_DLL_EXPORT 0
#endif /* _MSC_VER */

#ifndef _WIN32
#define LAST_RESORT_LOCALE "en_US.UTF-8" /* Supposing POSIX systems */
#else
#define LAST_RESORT_LOCALE ""            /* Use user default locale */
#endif /* _WIN32 */

#include "parser-utilities.h"
#include "command-line.h"
#include "lg_readline.h"
#ifdef USE_VITERBI
#include "../viterbi/viterbi.h"
#endif /* USE_VITERBI */

#define DISPLAY_MAX 1024
#define COMMENT_CHAR '%'  /* input lines beginning with this are ignored */

static int batch_errors = 0;
static int verbosity = 0;
static char * debug = (char *)"";
static char * test = (char *)"";
static bool isatty_stdin, isatty_stdout;
#ifdef _WIN32
static bool running_under_cygwin = false;
#endif /* _WIN32 */

typedef enum
{
	UNGRAMMATICAL = '*',
	PARSE_WITH_DISJUNCT_COST_GT_0 = ':',  /* Not used anywhere, currently ... */
	NO_LABEL = ' '
} Label;


#ifdef _WIN32
/* Windows console (cmd.exe) input to utf8 */
static char* oem_to_utf8(char *instring)
{
	char * out;
	wchar_t *winput;
	size_t len;
	int cv;
	unsigned int consolecp;

	consolecp = GetConsoleOutputCP();

	/* Convert input string to wide chars. */
	len = strlen(instring) + 1;
	cv = MultiByteToWideChar(consolecp, 0, instring, len, NULL, 0);
	winput = (wchar_t*) malloc(cv * sizeof(wchar_t));
	cv = MultiByteToWideChar(consolecp, 0, instring, len, winput, cv);

	/* Convert wide chars to utf8. */
	cv = WideCharToMultiByte(CP_UTF8, 0, winput, len, NULL, 0, NULL, NULL);
	out = (char*) malloc(cv);
	cv = WideCharToMultiByte(CP_UTF8, 0, winput, len, out, cv, NULL, NULL);

	free(winput);

	return out;
}
#endif /* _WIN32 */

static char *
fget_input_string(FILE *in, FILE *out, bool check_return)
{
	static char input_string[MAX_INPUT];
	static bool input_pending = false;

	if ((in != stdin) || !isatty_stdin)
	{
		/* Here the input is not from a terminal. */
		if (check_return) return (char *)"x"; /* XXX One linkage per sentence. */
		if (fgets(input_string, MAX_INPUT, in))
		{
			char *lf = strchr(input_string, '\n');
			if (lf) *lf = '\0';
			return input_string;
		}
		return NULL;
	}

	/* If we are here, the input is from a terminal. */
	static char *pline;
	const char *prompt = (0 == verbosity)? "" : "linkparser> ";

	if (input_pending)
	{
		input_pending = false;
		return pline;
	}

#ifdef HAVE_EDITLINE
	#ifdef _WIN32
		#error __FILE__ ": Cannot use HAVE_EDITLINE "
		                "(the console already has line editing and history)."
	#endif /* _WIN32 */
	pline = lg_readline(prompt);
#else
	fprintf(out, prompt);
	fflush(out);
	input_string[MAX_INPUT-2] = '\0';
#ifdef _WIN32
	if (!running_under_cygwin)
		pline = get_console_line();
	else
		pline = fgets(input_string, MAX_INPUT, in);
#else
	pline = fgets(input_string, MAX_INPUT, in);
#endif /* _WIN32 */
#endif /* HAVE_EDITLINE */

	if (NULL == pline) return NULL;                /* EOF */
	if (('\0' != input_string[MAX_INPUT-2]) &&
	    ('\n' != input_string[MAX_INPUT-2]))
	{
		prt_error("Warning: Input line too long (>%d)\n", MAX_INPUT-1);
		/* TODO: Ignore it and its continuation part(s). */
	}
	if (check_return)
	{
		if ('\0' == pline[0]) return (char *)"\n"; /* Continue linkage display */
		input_pending = true;
		return (char *)"x";                        /* Stop linkage display */
	}

	return pline;
}

/**************************************************************************
*
*  This procedure displays a linkage graphically.  Since the diagrams
*  are passed as character strings, they need to be deleted with a
*  call to free.
*
**************************************************************************/
{'fraction_non_alphanumeric': 0.19019212211597508, 'fraction_numerical': 0.012632687077813844, 'mean_word_length': 2.889457523029683, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7117590', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3404, 'n_words': 1135}