<commit_before>using System;
using System.Text;
using System.Collections.Generic;

namespace CorvusAlba.MyLittleLispy.Runtime
{
    public class Parser
    {
	private IEnumerator<string> _enumerator;
	private readonly HashSet<char> _whitespaces = new HashSet<char>(new [] {' ', '\t', '\n'});
	
	private bool IsValidForIdentifier(char c)
	{
	    return !(c == '(' || c == ')') && !_whitespaces.Contains(c);
	}

	private IEnumerable<string> Tokenize(string script)
	{
	    var chars = script.ToCharArray();
	    var i = 0;
	    while (i < chars.Length)
	    {
		if (_whitespaces.Contains(chars[i]))
		{
		    i++;
		    continue;
		}
		
		var sb = new StringBuilder();
		if (chars[i] == '\"')
		{
		    while (chars[i] != '\"')
		    {
			sb.Append(chars[i]);
			i++;
		    }
		    sb.Append(chars[i]);
		    i++;
		}
		else if (IsValidForIdentifier(chars[i]))
		{
		    while (IsValidForIdentifier(chars[i]))
		    {
			sb.Append(chars[i]);
			i++;
		    }
		}
		else 
		{
		    sb.Append(chars[i]);
		    i++;
		}

		yield return sb.ToString();		
	    }	  
	}

	private Node Expression()
	{
	    Syntax.Assert(_enumerator.Current == "(");
	    _enumerator.MoveNext();

	    var nodes = new List<Node>();
	    while (_enumerator.Current != ")")
	    {
		nodes.Add(Atom());
	    }
	    Syntax.Assert(_enumerator.Current == ")");
	    _enumerator.MoveNext();

	    return new Expression(nodes);
	}

	private Node Atom()
	{
	    if (_enumerator.Current == "(")
	    {
		return Expression();
	    }

	    if (_enumerator.Current == "'")
	    {
		return Quote();
	    }

	    string rawValue = _enumerator.Current;
	    _enumerator.MoveNext();

	    if (rawValue == "#t")
	    {
		return new Constant(new Bool(true));
	    }

	    if (rawValue == "#f")
	    {
		return new Constant(new Bool(false));
	    }

	    if (rawValue.StartsWith("\""))
	    {
		return new Constant(new String(rawValue.Trim('\"')));
	    }		

	    int value;
	    if (int.TryParse(rawValue, out value))
	    {
		return new Constant(new Integer(value));<commit_msg>fix lexer and string constants<commit_after>using System;
using System.Text;
using System.Collections.Generic;

namespace CorvusAlba.MyLittleLispy.Runtime
{
    public class Parser
    {
	private IEnumerator<string> _enumerator;
	private readonly HashSet<char> _whitespaces = new HashSet<char>(new [] {' ', '\t', '\n', '\r'});
	
	private bool IsValidForIdentifier(char c)
	{
	    return !(c == '(' || c == ')' || c == '\'') && !_whitespaces.Contains(c);
	}

	private IEnumerable<string> Tokenize(string script)
	{
	    var chars = script.ToCharArray();
	    var i = 0;
	    while (i < chars.Length)
	    {
		if (_whitespaces.Contains(chars[i]))
		{
		    i++;
		    continue;
		}
		
		var sb = new StringBuilder();
		if (chars[i] == '\"')
		{
		    // WTF?!
		    sb.Append(chars[i]);
		    i++;
		    while (chars[i] != '\"')
		    {
			sb.Append(chars[i]);
			i++;
		    }
		    sb.Append(chars[i]);
		    i++;
		}
		else if (IsValidForIdentifier(chars[i]))
		{
		    while (i < chars.Length && IsValidForIdentifier(chars[i]))
		    {
			sb.Append(chars[i]);
			i++;
		    }
		}
		else 
		{
		    sb.Append(chars[i]);
		    i++;
		}
		yield return sb.ToString();		
	    }	  
	}

	private Node Expression()
	{
	    Syntax.Assert(_enumerator.Current == "(");
	    _enumerator.MoveNext();

	    var nodes = new List<Node>();
	    while (_enumerator.Current != ")")
	    {
		nodes.Add(Atom());
	    }
	    Syntax.Assert(_enumerator.Current == ")");
	    _enumerator.MoveNext();

	    return new Expression(nodes);
	}

	private Node Atom()
	{
	    if (_enumerator.Current == "(")
	    {
		return Expression();
	    }

	    if (_enumerator.Current == "'")
	    {
		return Quote();
	    }

	    string rawValue = _enumerator.Current;
	    _enumerator.MoveNext();

	    if (rawValue == "#t")
	    {
		return new Constant(new Bool(true));
	    }

	    if (rawValue == "#f")
	    {
		return new Constant(new Bool(false));
	    }

	    if (rawValue.StartsWith("\""))
	    {
		return new Constant(new String(rawValue.Substring(1, rawValue.Length - 2)));
	    }		

	    int value;
	    if (int.TryParse(rawValue, out value))
	    {
		return new Constant(new Integer(value));
{'fraction_non_alphanumeric': 0.16829558998808106, 'fraction_numerical': 0.0009535160905840286, 'mean_word_length': 2.45919208573784, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3392530', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1435, 'n_words': 280}