<commit_before>/*
 * Copyright (c) 2006-2013, Juniper Networks, Inc.
 * All rights reserved.
 * This SOFTWARE is licensed under the LICENSE provided in the
 * ../Copyright file. By downloading, installing, copying, or otherwise
 * using the SOFTWARE, you agree to be bound by the terms of that
 * LICENSE.
 */

#include <ctype.h>		/* isalnum() */

/*
 * This is a should-not-occur error string if you ever see it, we've
 * done something wrong (but it beats a core dump).
 */
#define UNKNOWN_EXPR "<<<<slax error>>>>"

extern int slaxLogIsEnabled;	/* Global logging output switch */
extern const char *slaxKeywordString[];
extern const char *slaxTokenNameFancy[];

/*
 * YYSTYPE defines our stack frame.  Since we're all about strings,
 * we use strings as a natural token stack element.
 */
#define YYSTYPE slax_string_t *	/* This is our bison stack frame */

/*
 * Our main data structure
 */
struct slax_data_s {
    int sd_errors;		/* Number of errors seen */
    FILE *sd_file;		/* File to read from */
    unsigned sd_flags;		/* Flags */
    int sd_parse;		/* Parsing mode (M_PARSE_*) */
    int sd_ttype;		/* Token type returned on next lexer call */
    int sd_last;		/* Last token type returned */
    char sd_filename[MAXPATHLEN]; /* Path of current file */
    int sd_line;		/* Line number */
    int sd_col;			/* Column number */
    int sd_start;		/* Next valid byte in sd_buf */
    int sd_cur;			/* Current byte in sd_buf */<commit_msg>record flags for the next lexer, so they aren't affected by the yacc<commit_after>/*
 * Copyright (c) 2006-2013, Juniper Networks, Inc.
 * All rights reserved.
 * This SOFTWARE is licensed under the LICENSE provided in the
 * ../Copyright file. By downloading, installing, copying, or otherwise
 * using the SOFTWARE, you agree to be bound by the terms of that
 * LICENSE.
 */

#include <ctype.h>		/* isalnum() */

/*
 * This is a should-not-occur error string if you ever see it, we've
 * done something wrong (but it beats a core dump).
 */
#define UNKNOWN_EXPR "<<<<slax error>>>>"

extern int slaxLogIsEnabled;	/* Global logging output switch */
extern const char *slaxKeywordString[];
extern const char *slaxTokenNameFancy[];

/*
 * YYSTYPE defines our stack frame.  Since we're all about strings,
 * we use strings as a natural token stack element.
 */
#define YYSTYPE slax_string_t *	/* This is our bison stack frame */

/*
 * Our main data structure
 */
struct slax_data_s {
    int sd_errors;		/* Number of errors seen */
    FILE *sd_file;		/* File to read from */
    unsigned sd_flags;		/* Flags */
    unsigned sd_flags_next;	/* Flags for the next yylex call */
    int sd_parse;		/* Parsing mode (M_PARSE_*) */
    int sd_ttype;		/* Token type returned on next lexer call */
    int sd_last;		/* Last token type returned */
    char sd_filename[MAXPATHLEN]; /* Path of current file */
    int sd_line;		/* Line number */
    int sd_col;			/* Column number */
    int sd_start;		/* Next valid byte in sd_buf */
    int sd_cur;			/* Current byte in sd_buf */
{'fraction_non_alphanumeric': 0.10758027143330023, 'fraction_numerical': 0.005296259516716319, 'mean_word_length': 3.078272604588394, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095069', 'n_tokens_mistral': 984, 'n_tokens_neox': 939, 'n_words': 382}