<commit_before>/*
 * TRE tree processor
 * Copyright (c) 2005-2008 <NAME> <<EMAIL>>
 *
 * Reading TRE expressions.
 */

#include "config.h"
#include "atom.h"
#include "list.h"
#include "io.h"
#include "read.h"
#include "string2.h"
#include "gc.h"
#include "thread.h"
#include "number.h"
#include "error.h"

#include <ctype.h>
#include <string.h>

/*
 * Token values.
 *
 * Don't forget to update is_symchar().
 */
#define TRETOKEN_EOF			0	/* end of file */
#define TRETOKEN_BRACKET_OPEN	1
#define TRETOKEN_BRACKET_CLOSE	2
#define TRETOKEN_DOT			3
#define TRETOKEN_DBLQUOTE		4
#define TRETOKEN_QUOTE			5
#define TRETOKEN_BACKQUOTE		6
#define TRETOKEN_QUASIQUOTE		7
#define TRETOKEN_QUASIQUOTE_SPLICE	8
#define TRETOKEN_FUNCTION		9
#define TRETOKEN_CHAR			10
#define TRETOKEN_HEXNUM			11
#define TRETOKEN_SYMBOL			12	/* Keep this at the end. */

#define TRETOKEN_IS_QUOTE(x) \
	(x >= TRETOKEN_QUOTE && x <= TRETOKEN_FUNCTION)

/*
 * Check if the next incoming character would be legal part of a symbol.
 */
bool
is_symchar (unsigned char c)
{
    return (c > ' ' && c != '(' && c != ')' && c != '\'' && 
			c != '`' && c != ',' && c != '"' && c != ';' &&
			c != '#' && c != 255);
}

/*
 * Read symbol.
 *
 * Returns the length of the symbol or 0 if a special character or end
 * of input is read.
 */
int
get_symbol (struct tre_stream *str, char *s, char *p)
{
    ulong  len = 0;
    char   * os = s;
    char   c;

    *s = 0;
    *p = -1;
again:
    treio_skip_spaces (str);

    /* Ignore comments. */
    while (str->last_char == ';') {
        do { /* Read until line end. */
			c = treio_getc (str);
		} while (c != 10 && c != -1);
        goto again;
    }

after_pname:
    /* Read until a non-symbol character is found. */
    while (1) {
        c = toupper (treio_getc (str));
        if (is_symchar (c)) {
	    	/* Take read symbol as package name. */
            if (c == ':') {
				if (*p != -1)
		    		treerror (treptr_invalid, "double package name");
				strcpy (p, os);
				len = 0;
				s = os;
				goto after_pname;
	    	}

            *s++ = c;
            len++;
	    	if (len > TRE_MAX_SYMLEN)
				treerror_internal (treptr_invalid,
			   					   "literal symbols must be no longer than %d chars",
			   					   TRE_MAX_SYMLEN);
            continue;
        }

        *s++ = 0;
        if (len > 0)
	    	treio_putback (str);

        return len;
    }
}<commit_msg>READ: Now reads packages properly.<commit_after>/*
 * TRE tree processor
 * Copyright (c) 2005-2008 <NAME> <<EMAIL>>
 *
 * Reading TRE expressions.
 */

#include "config.h"
#include "atom.h"
#include "list.h"
#include "io.h"
#include "read.h"
#include "string2.h"
#include "gc.h"
#include "thread.h"
#include "number.h"
#include "error.h"

#include <ctype.h>
#include <string.h>

/*
 * Token values.
 *
 * Don't forget to update is_symchar().
 */
#define TRETOKEN_EOF			0	/* end of file */
#define TRETOKEN_BRACKET_OPEN	1
#define TRETOKEN_BRACKET_CLOSE	2
#define TRETOKEN_DOT			3
#define TRETOKEN_DBLQUOTE		4
#define TRETOKEN_QUOTE			5
#define TRETOKEN_BACKQUOTE		6
#define TRETOKEN_QUASIQUOTE		7
#define TRETOKEN_QUASIQUOTE_SPLICE	8
#define TRETOKEN_FUNCTION		9
#define TRETOKEN_CHAR			10
#define TRETOKEN_HEXNUM			11
#define TRETOKEN_SYMBOL			12	/* Keep this at the end. */

#define TRETOKEN_IS_QUOTE(x) \
	(x >= TRETOKEN_QUOTE && x <= TRETOKEN_FUNCTION)

/*
 * Check if the next incoming character would be legal part of a symbol.
 */
bool
is_symchar (unsigned char c)
{
    return (c > ' ' && c != '(' && c != ')' && c != '\'' && 
			c != '`' && c != ',' && c != '"' && c != ';' &&
			c != '#' && c != 255);
}

/*
 * Read symbol.
 *
 * Returns the length of the symbol or 0 if a special character or end
 * of input is read.
 */
int
get_symbol (struct tre_stream *str, char *s, char *p)
{
    ulong  len = 0;
    char   * os = s;
    char   c;
	bool   got_package = FALSE;

    *s = 0;
    *p = -1;
again:
    treio_skip_spaces (str);

    /* Ignore comments. */
    while (str->last_char == ';') {
        do { /* Read until line end. */
			c = treio_getc (str);
		} while (c != 10 && c != -1);
        goto again;
    }

after_pname:
    /* Read until a non-symbol character is found. */
    while (1) {
        c = toupper (treio_getc (str));
        if (is_symchar (c)) {
	    	/* Take read symbol as package name. */
            if (c == ':') {
				if (got_package)
		    		treerror (treptr_invalid, "double package name");
				strcpy (p, os);
				len = 0;
				s = os;
				got_package = TRUE;
				goto after_pname;
	    	}

            *s++ = c;
            len++;
	    	if (len > TRE_MAX_SYMLEN)
				treerror_internal (treptr_invalid,
			   					   "literal symbols must be no longer than %d chars",
			   					   TRE_MAX_SYMLEN);
            continue;
        }

        *s++ = 0;
        if (len > 0)
	    	treio_putback (str);

        return len;
    }
}
{'fraction_non_alphanumeric': 0.14694381559991768, 'fraction_numerical': 0.01625848940111134, 'mean_word_length': 2.344803854094976, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3696990', 'n_tokens_mistral': 2024, 'n_tokens_neox': 1875, 'n_words': 526}