<commit_before>//******************************************************************************
// scanner.C
//
// scanner() is used by parser to return the elements of an expression one at
// a time.
// 
//******************************************************************************

#include "condor_common.h"
#include "condor_exprtype.h"
#include "condor_scanner.h" 

static const int 	MAXVARNAME = 256;

////////////////////////////////////////////////////////////////////////////////
// Implementation of class Token
////////////////////////////////////////////////////////////////////////////////

Token::Token()
{
	length = 0;
	type = NOT_KEYWORD;
	strVal = NULL;
	isString = FALSE;
}

Token::~Token()
{
	if(isString)
	{
		delete []strVal;
	}
}

////////////////////////////////////////////////////////////////////////////////
// This function reads input from a string, returns a token;
////////////////////////////////////////////////////////////////////////////////
void Scanner(char*& s, Token& t)
{ 
    char	str[MAXVARNAME];
    char*	tmp;		// working variable

    // skip white space
    t.length = 0;
    while(isspace(*s))
    {
        s = s + 1;
		t.length++;
    }

    if(!strncmp(s, "NULL", 4) && !isalpha(*(s+4)) && *(s+4) != '_')
    // NULL
    {
        s = s + 4;
		t.length = t.length + 4;
        t.type = LX_NULL;
		return;
    }
    if(!strncasecmp(s, "TRUE", 4) && !isalpha(*(s+4)) && *(s+4) != '_')
    // TRUE
    {
        s = s + 4;
		t.length = t.length + 4;
        t.intVal = 1;
        t.type = LX_BOOL;
		return;
    }
    if(!strncasecmp(s, "FALSE", 5) && !isalpha(*(s+5)) && *(s+5) != '_')
    // FALSE
    {
        s = s + 5;
		t.length = t.length + 5;
        t.intVal = 0;
        t.type = LX_BOOL;
		return;
    }
    if(isalpha(*s) || *s == '_')
    // token is a variable
    {
        tmp = str;
        // prefix, if any.
		if(!strncmp(s, "MY.", 3) && (isalpha(*(s+3)) || *(s+3) == '_'))
		{
			for(int i=0; i<3; i++)
			{
				*tmp = *s;
				s++;
				tmp++;
				t.length++;
			}
		}
		else if(!strncmp(s, "TARGET.", 7) && (isalpha(*(s+7)) || *(s+7) == '_'))
		{
			for(int i=0; i<7; i++)
			{
				*tmp = *s;
				s++;
				tmp++;
				t.length++;
			}
		}
		// the rest of the variable.
        while(isalnum(*s) || *s == '_')
		{ 
			*tmp = *s;
			s++;
            tmp++;
			t.length++;
        }
        *tmp = '\0';
        t.strVal = new char[strlen(str) + 1];
		strcpy(t.strVal, str);
        t.type = LX_VARIABLE;
		t.isString = TRUE;
		return;
    }
    if(isdigit(*s))
    // token is an integer or a floating point number
    {
		// count the length of the number
        tmp = s;
        while(isdigit(*tmp))
		{
			tmp++;
			t.length++;
		}
        if(*tmp == '.')
        // token is a floating point number
        {
			t.length++;
			for(tmp++; isdigit(*tmp); tmp++) t.length++;
            t.floatVal = (float)strtod(s, &s);
            t.type = LX_FLOAT; 
        }
		else
		// token is an integer number
		{
            t.intVal = strtol(s, &s, 10);
            t.type = LX_INTEGER; 
        }
		return;
    }
    if(*s == '"')
    // token is a string
    {
        s++;
		tmp = str;
		t.length++;
        while(*s != '"' && *s != '\0')
		{
			if(*s == '\\')
			{
				s++;
				t.length++; 
			} 
			*tmp = *s;
			s++;
            tmp++;
			t.length++;
        }
        if(*s == '\0')
		{
			 t.type = LX_ERROR;
			 t.length = 0;
			 return;
		}
        s++;
		t.length++;
        *tmp = '\0';
		t.strVal = new char[strlen(str) + 1];
		strcpy(t.strVal, str);
        t.type = LX_STRING; 
		t.isString = TRUE;
		return;
    }

    // token is an operator or a unit
	switch(*s)
	{
		case '(': t.type = LX_LPAREN;
			   s = s + 1;
			   t.length++;
			   break; <commit_msg>handle string constants larger than _POSIX_PATH_MAX<commit_after>//******************************************************************************
// scanner.C
//
// scanner() is used by parser to return the elements of an expression one at
// a time.
// 
//******************************************************************************

#include "condor_common.h"
#include "condor_exprtype.h"
#include "condor_scanner.h" 

static const int 	MAXVARNAME = 256;

////////////////////////////////////////////////////////////////////////////////
// Implementation of class Token
////////////////////////////////////////////////////////////////////////////////

Token::Token()
{
	length = 0;
	type = NOT_KEYWORD;
	strVal = NULL;
	isString = FALSE;
}

Token::~Token()
{
	if(isString)
	{
		delete []strVal;
	}
}

////////////////////////////////////////////////////////////////////////////////
// This function reads input from a string, returns a token;
////////////////////////////////////////////////////////////////////////////////
void Scanner(char*& s, Token& t)
{ 
    char	str[MAXVARNAME];
    char	*tmp, *tmp2;		// working variables

    // skip white space
    t.length = 0;
    while(isspace(*s))
    {
        s = s + 1;
		t.length++;
    }

    if(!strncmp(s, "NULL", 4) && !isalpha(*(s+4)) && *(s+4) != '_')
    // NULL
    {
        s = s + 4;
		t.length = t.length + 4;
        t.type = LX_NULL;
		return;
    }
    if(!strncasecmp(s, "TRUE", 4) && !isalpha(*(s+4)) && *(s+4) != '_')
    // TRUE
    {
        s = s + 4;
		t.length = t.length + 4;
        t.intVal = 1;
        t.type = LX_BOOL;
		return;
    }
    if(!strncasecmp(s, "FALSE", 5) && !isalpha(*(s+5)) && *(s+5) != '_')
    // FALSE
    {
        s = s + 5;
		t.length = t.length + 5;
        t.intVal = 0;
        t.type = LX_BOOL;
		return;
    }
    if(isalpha(*s) || *s == '_')
    // token is a variable
    {
        tmp = str;
        // prefix, if any.
		if(!strncmp(s, "MY.", 3) && (isalpha(*(s+3)) || *(s+3) == '_'))
		{
			for(int i=0; i<3; i++)
			{
				*tmp = *s;
				s++;
				tmp++;
				t.length++;
			}
		}
		else if(!strncmp(s, "TARGET.", 7) && (isalpha(*(s+7)) || *(s+7) == '_'))
		{
			for(int i=0; i<7; i++)
			{
				*tmp = *s;
				s++;
				tmp++;
				t.length++;
			}
		}
		// the rest of the variable.
        while(isalnum(*s) || *s == '_')
		{ 
			*tmp = *s;
			s++;
            tmp++;
			t.length++;
        }
        *tmp = '\0';
        t.strVal = new char[strlen(str) + 1];
		strcpy(t.strVal, str);
        t.type = LX_VARIABLE;
		t.isString = TRUE;
		return;
    }
    if(isdigit(*s))
    // token is an integer or a floating point number
    {
		// count the length of the number
        tmp = s;
        while(isdigit(*tmp))
		{
			tmp++;
			t.length++;
		}
        if(*tmp == '.')
        // token is a floating point number
        {
			t.length++;
			for(tmp++; isdigit(*tmp); tmp++) t.length++;
            t.floatVal = (float)strtod(s, &s);
            t.type = LX_FLOAT; 
        }
		else
		// token is an integer number
		{
            t.intVal = strtol(s, &s, 10);
            t.type = LX_INTEGER; 
        }
		return;
    }
    if(*s == '"')
    // token is a string
    {
        s++;
		tmp2 = tmp = new char[strlen(s) + 1]; // make sure to have enough space
		t.length++;
        while(*s != '"' && *s != '\0')
		{
			if(*s == '\\')
			{
				s++;
				t.length++; 
			} 
			*tmp = *s;
			s++;
            tmp++;
			t.length++;
        }
        if(*s == '\0')
		{
			 t.type = LX_ERROR;
			 t.length = 0;
			 delete [] tmp2;
			 return;
		}
        s++;
		t.length++;
        *tmp = '\0';
		t.strVal = new char[strlen(tmp2) + 1];
		strcpy(t.strVal, tmp2);
        t.type = LX_STRING; 
		t.isString = TRUE;
		delete [] tmp2;
		return;
    }

    // token is an operator or a unit
	switch(*s)
	{
		case '(': t.type = LX_LPAREN;
			   s = s + 1;
			   t.length++;
			   break; 
{'fraction_non_alphanumeric': 0.2997246623836371, 'fraction_numerical': 0.01219352301035794, 'mean_word_length': 1.4347270986275136, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4200514', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2700, 'n_words': 619}