<commit_before>/* $Id$ $Revision$ */
/* vim:set shiftwidth=4 ts=8: */

/**********************************************************
*      This software is part of the graphviz package      *
*                http://www.graphviz.org/                 *
*                                                         *
*            Copyright (c) 1994-2004 AT&T Corp.           *
*                and is licensed under the                *
*            Common Public License, Version 1.0           *
*                      by AT&T Corp.                      *
*                                                         *
*        Information and Software Systems Research        *
*              AT&T Research, Florham Park NJ             *
**********************************************************/


#include <stdarg.h>
#include "libgraph.h"
#include "parser.h"
#include "triefa.cP"
#include "agxbuf.h"

#ifdef DMALLOC
#include "dmalloc.h"
#endif

#define InfileName (InputFile?InputFile:"<unknown>")

static FILE *Lexer_fp;
static char *LexPtr, *TokenBuf;
static int LineBufSize;
static uchar In_comment;
static uchar Comment_start;
static uchar Start_html_string;
static int Line_number;
static char *InputFile;
static gets_f Lexer_gets;

  /* Reset line number.
   * Argument n is indexed from 1, so we decrement it.
   */
void agreadline(int n)
{
    Line_number = n - 1;
}

  /* (Re)set file:
   */
void agsetfile(char *f)
{
    InputFile = f;
    Line_number = 0;
}

void aglexinit(FILE * fp, gets_f mygets)
{
    Lexer_fp = fp;
    Lexer_gets = mygets;
    LexPtr = NULL;
    if (AG.linebuf == NULL) {
	LineBufSize = BUFSIZ;
	AG.linebuf = N_NEW(LineBufSize, char);
	TokenBuf = N_NEW(LineBufSize, char);
    }
    (Lexer_gets) (AG.linebuf, 0, fp);	/* reset mygets */
}

#define ISSPACE(c) (isspace(c) || iscntrl(c))
<commit_msg>Fix bug in previous version: technically, '\0' is a control character but we can't treat it as white space.<commit_after>/* $Id$ $Revision$ */
/* vim:set shiftwidth=4 ts=8: */

/**********************************************************
*      This software is part of the graphviz package      *
*                http://www.graphviz.org/                 *
*                                                         *
*            Copyright (c) 1994-2004 AT&T Corp.           *
*                and is licensed under the                *
*            Common Public License, Version 1.0           *
*                      by AT&T Corp.                      *
*                                                         *
*        Information and Software Systems Research        *
*              AT&T Research, Florham Park NJ             *
**********************************************************/


#include <stdarg.h>
#include "libgraph.h"
#include "parser.h"
#include "triefa.cP"
#include "agxbuf.h"

#ifdef DMALLOC
#include "dmalloc.h"
#endif

#define InfileName (InputFile?InputFile:"<unknown>")

static FILE *Lexer_fp;
static char *LexPtr, *TokenBuf;
static int LineBufSize;
static uchar In_comment;
static uchar Comment_start;
static uchar Start_html_string;
static int Line_number;
static char *InputFile;
static gets_f Lexer_gets;

  /* Reset line number.
   * Argument n is indexed from 1, so we decrement it.
   */
void agreadline(int n)
{
    Line_number = n - 1;
}

  /* (Re)set file:
   */
void agsetfile(char *f)
{
    InputFile = f;
    Line_number = 0;
}

void aglexinit(FILE * fp, gets_f mygets)
{
    Lexer_fp = fp;
    Lexer_gets = mygets;
    LexPtr = NULL;
    if (AG.linebuf == NULL) {
	LineBufSize = BUFSIZ;
	AG.linebuf = N_NEW(LineBufSize, char);
	TokenBuf = N_NEW(LineBufSize, char);
    }
    (Lexer_gets) (AG.linebuf, 0, fp);	/* reset mygets */
}

#define ISSPACE(c) ((c != 0) && ((isspace(c) || iscntrl(c))))

{'fraction_non_alphanumeric': 0.16870350690754515, 'fraction_numerical': 0.009032943676939426, 'mean_word_length': 2.058489033306255, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2310038', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1145, 'n_words': 318}