<commit_before>package org.yellcorp.lib.format.geo
{
import org.yellcorp.lib.format.FormatStringError;
import org.yellcorp.lib.format.geo.renderer.Literal;
import org.yellcorp.lib.format.geo.renderer.NumberRenderer;
import org.yellcorp.lib.format.geo.renderer.SignRenderer;
import org.yellcorp.lib.relexer.Lexer;
import org.yellcorp.lib.relexer.Token;


internal class GeoParser
{
    private static var lexer:Lexer = new Lexer(buildTokenExpression());
    private var renderers:Array;
    private var field:GeoFieldProperties;

    public function GeoParser()
    {
        field = new GeoFieldProperties();
    }

    public function parse(formatString:String):Array
    {
        var returnRenderers:Array;

        lexer.start(formatString);
        renderers = [ ];
        while (!lexer.atEnd)
        {
            parseToken();
        }
        returnRenderers = renderers.slice();
        renderers = null;
        return returnRenderers;
    }

    private function parseToken():void
    {
        var token:Token = lexer.nextToken();

        if (token.text == "%")
        {
            parseField();
        }
        else if (token.text)
        {
            renderers.push(new Literal(token.text));
        }
    }

    private function parseField():void
    {
        field.clear();
        parseWidth();
        parsePrecision();
        parseConversion();
    }

    private function parseWidth():void
    {
        var token:Token = lexer.nextToken();
        if (token.text)
        {
            if (token.text.charAt(0) == '0')
            {
                field.zeroPad = true;
            }
            field.minWidth = parseInt(token.text, 10);
        }
    }

    private function parsePrecision():void
    {
        var token:Token = lexer.nextToken();

        if (token.text)
        {
            field.precision = parseInt(token.text.substr(1), 10);
        }
    }

    private function parseConversion():void
    {
        var token:Token = lexer.nextToken();

        if (!token.text)
        {
            throw new FormatStringError(
                "Encountered field without conversion specifier",
                lexer.text, lexer.currentChar);
        }

        switch (token.text)
        {
        case 'd' :
            renderers.push(new NumberRenderer(field, 1, 0, true));
            break;

        case 'm' :
            renderers.push(new NumberRenderer(field, 60, 60, true));
            break;

        case 's' :
            renderers.push(new NumberRenderer(field, 3600, 60, false));
            break;

        case '-' :
            renderers.push(new SignRenderer("", "-"));
            break;

        case '+' :
            renderers.push(new SignRenderer("+", "-"));
            break;

        case 'o' :
            renderers.push(new SignRenderer("e", "w"));
            break;

        case 'O' :
            renderers.push(new SignRenderer("E", "W"));
            break;

        case 'a' :
            renderers.push(new SignRenderer("n", "s"));
            break;

        case 'A' :
            renderers.push(new SignRenderer("N", "S"));
            break;

        case '*' :
            renderers.push(new Literal(String.fromCharCode(0x00B0)));
            break;

        case "'" :
            renderers.push(new Literal(String.fromCharCode(0x2032)));
            break;

        case "''" :
            renderers.push(new Literal(String.fromCharCode(0x2033)));
            break;

        case '%' :
            renderers.push(new Literal("%"));
            break;
        }
    }

    private static function buildTokenExpression():RegExp
    {
        var width:String = "(\\d+)?";
        var precision:String = "(\\.\\d+)?";

        var singleConversions:String = "[dms\\-+oOaA*%]";
        var primeSequence:String = "''?";

        var expr:String =
            "(%)" +         // group 1
            width +         // 2
            precision +     // 3
            "(" +           // 4
                singleConversions +
            "|" +
                primeSequence +
            ")";<commit_msg>GeoParser now uses retokenizer and passes tests<commit_after>package org.yellcorp.lib.format.geo
{
import org.yellcorp.lib.format.geo.renderer.Literal;
import org.yellcorp.lib.format.geo.renderer.NumberRenderer;
import org.yellcorp.lib.format.geo.renderer.Renderer;
import org.yellcorp.lib.format.geo.renderer.SignRenderer;
import org.yellcorp.lib.string.retokenizer.Token;
import org.yellcorp.lib.string.retokenizer.Tokenizer;


internal class GeoParser
{
    private static var tokenRegex:RegExp;

    private var renderers:Array;
    private var tokenizer:Tokenizer;

    public function GeoParser()
    {
    }

    public function parse(formatString:String):Array
    {
        if (!tokenRegex)
        {
            tokenRegex = buildTokenRegex();
        }
        tokenizer = new Tokenizer(tokenRegex, formatString);
        renderers = [ ];
        while (!tokenizer.atEnd)
        {
            parseToken();
        }

        var returnRenderers:Array = renderers;
        renderers = null;
        return returnRenderers;
    }

    private function parseToken():void
    {
        var token:Token = tokenizer.next();

        if (!token)
        {
            return;
        }
        else if (!token.matched)
        {
            renderers.push(new Literal(token.text));
        }
        else
        {
            renderers.push(createRendererFromToken(token));
        }
    }

    private function createRendererFromToken(token:Token):Renderer
    {
        var width:String = token.getGroup(1);
        var precision:String = token.getGroup(2);
        var specifier:String = token.getGroup(3);

        var field:GeoFieldProperties = new GeoFieldProperties();

        if (width)
        {
            if (width.charAt(0) == '0')
            {
                field.zeroPad = true;
            }
            field.minWidth = parseInt(width, 10);
        }

        if (precision)
        {
            field.precision = parseInt(precision.substr(1), 10);
        }

        return createRendererFromSpecifier(specifier, field);
    }

    private function createRendererFromSpecifier(specifier:String, field:GeoFieldProperties):Renderer
    {
        switch (specifier)
        {
        case 'd' :
            return new NumberRenderer(field, 1, 0, true);
        case 'm' :
            return new NumberRenderer(field, 60, 60, true);
        case 's' :
            return new NumberRenderer(field, 3600, 60, false);
        case '-' :
            return new SignRenderer("", "-");
        case '+' :
            return new SignRenderer("+", "-");
        case 'o' :
            return new SignRenderer("e", "w");
        case 'O' :
            return new SignRenderer("E", "W");
        case 'a' :
            return new SignRenderer("n", "s");
        case 'A' :
            return new SignRenderer("N", "S");
        case '*' :
            return new Literal(String.fromCharCode(0x00B0));
        case "'" :
            return new Literal(String.fromCharCode(0x2032));
        case "''" :
            return new Literal(String.fromCharCode(0x2033));
        case '%' :
            return new Literal("%");
        default:
            throw new Error("Internal error: RegExp match an unhandled conversion specifier");
        }
    }

    private static function buildTokenRegex():RegExp
    {
        var width:String = "(\\d+)?";
        var precision:String = "(\\.\\d+)?";

        var singleCharSpecifiers:String = "[dms\\-+oOaA*%]";
        var primeSequence:String = "''?";

        var expr:String =
            "%" +
            width +         // 1
            precision +     // 2
            "(" +           // 3
                singleCharSpecifiers +
            "|" +
                primeSequence +
            ")";
{'fraction_non_alphanumeric': 0.12251402345741969, 'fraction_numerical': 0.009688934217236104, 'mean_word_length': 1.692175703500343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4222647', 'n_tokens_mistral': 2326, 'n_tokens_neox': 2200, 'n_words': 438}