<commit_before>from rupypy import ast

from .base import BaseRuPyPyTest


class TestParser(BaseRuPyPyTest):
    def test_int_constant(self, ec):
        assert ec.space.parse(ec, "1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1))
        ]))
        assert ec.space.parse(ec, "-1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(-1))
        ]))
        assert ec.space.parse(ec, "1_1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(11))
        ]))
        assert ec.space.parse(ec, "0d10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(10))
        ]))
        assert ec.space.parse(ec, "0xA") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(10))
        ]))
        assert ec.space.parse(ec, "0o10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(8))
        ]))
        assert ec.space.parse(ec, "0b10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(2))
        ]))

    def test_float(self, ec):
        assert ec.space.parse(ec, "0.2") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(0.2))
        ]))
        assert ec.space.parse(ec, "1E1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(10.0))
        ]))
        assert ec.space.parse(ec, "1e1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(10.0))
        ]))

    def test_binary_expression(self, ec):
        assert ec.space.parse(ec, "1+1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "1/1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("/", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "1===1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("===", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "2 % 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("%", ast.ConstantInt(2), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 =~ 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("=~", ast.ConstantInt(2), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 !~ 3") == ast.Main(ast.Block([
            ast.Statement(ast.Not(ast.BinOp("=~", ast.ConstantInt(2), ast.ConstantInt(3), 1)))
        ]))
        assert ec.space.parse(ec, "1 =~ /v/") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("=~", ast.ConstantInt(1), ast.ConstantRegexp("v"), 1))
        ]))
        assert ec.space.parse(ec, "2 & 3 | 5") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("|", ast.BinOp("&", ast.ConstantInt(2), ast.ConstantInt(3), 1), ast.ConstantInt(5), 1))
        ]))
        assert ec.space.parse(ec, "$a << []") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<<", ast.Global("$a"), ast.Array([]), 1))
        ]))
        assert ec.space.parse(ec, "3 >> 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp(">>", ast.ConstantInt(3), ast.ConstantInt(2), 1))
        ]))
        assert ec.space.parse(ec, "5 or 3") == ast.Main(ast.Block([
            ast.Statement(ast.Or(ast.ConstantInt(5),
                                        ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "puts 5 and 3") == ast.Main(ast.Block([
            ast.Statement(ast.And(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(5)],
                                                  None, 1),
                                         ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "x[0] == ?-") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("==",
                ast.Subscript(ast.Variable("x", 1), [ast.ConstantInt(0)], 1),
                ast.ConstantString("-"),
            1,))
        ]))
        assert ec.space.parse(ec, "@x-1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("-", ast.InstanceVariable("@x"), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, ":a <=> :a") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<=>", ast.ConstantSymbol("a"), ast.ConstantSymbol("a"), 1))
        ]))
        assert ec.space.parse(ec, ":a != ?-") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("!=", ast.ConstantSymbol("a"), ast.ConstantString("-"), 1))
        ]))
        assert ec.space.parse(ec, "1 ^ 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("^", ast.ConstantInt(1), ast.ConstantInt(2), 1))
        ]))
        assert ec.space.parse(ec, "1 ** 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("**", ast.ConstantInt(1), ast.ConstantInt(2), 1))
        ]))

    def test_multi_term_expr(self, ec):
        assert ec.space.parse(ec, "1 + 2 * 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.ConstantInt(1), ast.BinOp("*", ast.ConstantInt(2), ast.ConstantInt(3), 1), 1))
        ]))
        assert ec.space.parse(ec, "1 * 2 + 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.BinOp("*", ast.ConstantInt(1), ast.ConstantInt(2), 1), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 << 3 * 4") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<<", ast.ConstantInt(2), ast.BinOp("*", ast.ConstantInt(3), ast.ConstantInt(4), 1), 1))
        ]))

    def test_parens(self, ec):
        assert ec.space.parse(ec, "1 * (2 - 3)") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("*", ast.ConstantInt(1), ast.BinOp("-", ast.ConstantInt(2), ast.ConstantInt(3), 1), 1))
        ]))

    def test_multiple_statements_no_sep(self, ec):
        with self.raises("SyntaxError"):
            ec.space.parse(ec, "3 3")

    def test_multiple_statements(self, ec):
        r = ec.space.parse(ec, """
        1
        2
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
        ]))

    def test_multiple_statements_semicolon(self, ec):
        assert ec.space.parse(ec, "1; 2") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
        ]))

        assert ec.space.parse(ec, "1; 2; 3") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
            ast.Statement(ast.ConstantInt(3)),
        ]))

    def test_send(self, ec):
        assert ec.space.parse(ec, "puts 2") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "puts 1, 2") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(1), ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "puts(1, 2)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(1), ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s 10") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s.to_i") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1), "to_i", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s()") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(10)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(*10)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.Splat(ast.ConstantInt(10))], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(10, *x)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10), ast.Splat(ast.Variable("x", 1))], None, 1))
        ]))

    def test_assignment(self, ec):
        assert ec.space.parse(ec, "a = 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "a = b = 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.Assignment(ast.Variable("b", 1), ast.ConstantInt(3))))
        ]))

    def test_multi_assignment(self, ec):
        assert ec.space.parse(ec, "a.x, b[:idx], c::Const, d = 3") == ast.Main(ast.Block([
            ast.Statement(ast.MultiAssignment(
                [
                    ast.Send(ast.Variable("a", 1), "x", [], None, 1),
                    ast.Subscript(ast.Variable("b", 1), [ast.ConstantSymbol("idx")], 1),
                    ast.LookupConstant(ast.Variable("c", 1), "Const", 1),
                    ast.Variable("d", 1),
                ],
                ast.ConstantInt(3)
            ))
        ]))
        assert ec.space.parse(ec, "a = 2, 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.Array([ast.ConstantInt(2), ast.ConstantInt(3)])))
        ]))
        with self.raises("SyntaxError"):
            ec.space.parse(ec, "a, b += 3")

    def test_load_variable(self, ec):
        assert ec.space.parse(ec, "a") == ast.Main(ast.Block([
            ast.Statement(ast.Variable("a", 1))
        ]))

    def test_if_statement(self, ec):
        res = lambda lineno: ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.Send(ast.Self(lineno), "puts", [ast.ConstantInt(2)], None, lineno))
            ]), ast.Block([])))
        ]))
        assert ec.space.parse(ec, "if 3 then puts 2 end") == res(1)
        assert ec.space.parse(ec, """
        if 3
            puts 2
        end
        """) == res(3)
        assert ec.space.parse(ec, "if 3; puts 2 end") == res(1)
        assert ec.space.parse(ec, "if 3; end") == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([]), ast.Block([])))
        ]))
        r = ec.space.parse(ec, """
        if 0
            puts 2
            puts 3
            puts 4
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(0), ast.Block([
                ast.Statement(ast.Send(ast.Self(3), "puts", [ast.ConstantInt(2)], None, 3)),
                ast.Statement(ast.Send(ast.Self(4), "puts", [ast.ConstantInt(3)], None, 4)),
                ast.Statement(ast.Send(ast.Self(5), "puts", [ast.ConstantInt(4)], None, 5)),
            ]), ast.Block([])))
        ]))

    def test_else(self, ec):
        r = ec.space.parse(ec, """if 3 then 5 else 4 end""")
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ]), ast.Block([
                ast.Statement(ast.ConstantInt(4))
            ])))
        ]))

    def test_elsif(self, ec):
        r = ec.space.parse(ec, """
        if 3
            5
        elsif 4 == 2
            3
        elsif 3 == 1
            2
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ]), ast.Block([
                ast.Statement(ast.If(ast.BinOp("==", ast.ConstantInt(4), ast.ConstantInt(2), 4), ast.Block([
                    ast.Statement(ast.ConstantInt(3))
                ]), ast.Block([
                    ast.Statement(ast.If(ast.BinOp("==", ast.ConstantInt(3), ast.ConstantInt(1), 6), ast.Block([
                        ast.Statement(ast.ConstantInt(2))
                    ]), ast.Block([ast.Statement(ast.Variable("nil", -1))])))
                ])))
            ])))
        ]))

    def test_comparison_ops(self, ec):
        assert ec.space.parse(ec, "1 == 2; 1 < 2; 1 > 2; 1 != 2; 1 <= 2; 1 >= 2; 1 <=> 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("==", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp(">", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("!=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp(">=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<=>", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
        ]))

    def test_while(self, ec):
        expected = ast.Main(ast.Block([
            ast.Statement(ast.While(ast.Variable("true", 1), ast.Block([
                ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(5)], None, 1))
            ])))
        ]))
        assert ec.space.parse(ec, "while true do puts 5 end") == expected
        assert ec.space.parse(ec, "while true do; puts 5 end") == expected
        assert ec.space.parse(ec, "while true; puts 5 end") == expected
        assert ec.space.parse(ec, "while true; end") == ast.Main(ast.Block([
            ast.Statement(ast.While(ast.Variable("true", 1), ast.Block([
                ast.Statement(ast.Variable("nil", -1))
            ])))
        ]))

        res = ec.space.parse(ec, """
        i = 0
        while i < 10 do
            puts i
            puts 1
            puts i
            puts true
        end
        """)
        assert res == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("i", 2), ast.ConstantInt(0))),
            ast.Statement(ast.While(ast.BinOp("<", ast.Variable("i", 3), ast.ConstantInt(10), 3), ast.Block([
                ast.Statement(ast.Send(ast.Self(4), "puts", [ast.Variable("i", 4)], None, 4)),
                ast.Statement(ast.Send(ast.Self(5), "puts", [ast.ConstantInt(1)], None, 5)),
                ast.Statement(ast.Send(ast.Self(6), "puts", [ast.Variable("i", 6)], None, 6)),
                ast.Statement(ast.Send(ast.Self(7), "puts", [ast.Variable("true", 7)], None, 7)),
            ])))
        ]))

    def test_until(self, ec):
        r = ec.space.parse(ec, """
        until 3
            5
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.Until(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ])))
        ]))

    def test_return(self, ec):
        assert ec.space.parse(ec, "return 4") == ast.Main(ast.Block([
            ast.Return(ast.ConstantInt(4))
        ]))
        assert ec.space.parse(ec, "return") == ast.Main(ast.Block([
            ast.Return(ast.Variable("nil", 1))
        ]))
        assert ec.space.parse(ec, "return 3, 4, 5") == ast.Main(ast.Block([
            ast.Return(ast.Array([
                ast.ConstantInt(3),
                ast.ConstantInt(4),
                ast.ConstantInt(5),
            ]))
        ]))

    def test_array(self, ec):
        assert ec.space.parse(ec, "[]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([]))
        ]))

        assert ec.space.parse(ec, "[1, 2, 3]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.ConstantInt(1),
                ast.ConstantInt(2),
                ast.ConstantInt(3),
            ]))
        ]))

        assert ec.space.parse(ec, "[[1], [2], [3]]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.Array([ast.ConstantInt(1)]),
                ast.Array([ast.ConstantInt(2)]),
                ast.Array([ast.ConstantInt(3)]),
            ]))
        ]))

        assert ec.space.parse(ec, "[1, 2,]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.ConstantInt(1),
                ast.ConstantInt(2),
            ]))
        ]))

        r = ec.space.parse(ec, """
        [
            f()
        ]
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.Send(ast.Self(3), "f", [], None, 3),
            ])),
        ]))

    def test_subscript(self, ec):
        assert ec.space.parse(ec, "[1][0]") == ast.Main(ast.Block([
            ast.Statement(ast.Subscript(ast.Array([ast.ConstantInt(1)]), [ast.ConstantInt(0)], 1))
        ]))

        assert ec.space.parse(ec, "self[i]") == ast.Main(ast.Block([
            ast.Statement(ast.Subscript(ast.Variable("self", 1), [ast.Variable("i", 1)], 1))
        ]))

        assert ec.space.parse(ec, "self[i].to_s") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Subscript(ast.Variable("self", 1), [ast.Variable("i", 1)], 1), "to_s", [], None, 1))
        ]))<commit_msg>test for splat in array<commit_after>from rupypy import ast

from .base import BaseRuPyPyTest


class TestParser(BaseRuPyPyTest):
    def test_int_constant(self, ec):
        assert ec.space.parse(ec, "1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1))
        ]))
        assert ec.space.parse(ec, "-1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(-1))
        ]))
        assert ec.space.parse(ec, "1_1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(11))
        ]))
        assert ec.space.parse(ec, "0d10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(10))
        ]))
        assert ec.space.parse(ec, "0xA") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(10))
        ]))
        assert ec.space.parse(ec, "0o10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(8))
        ]))
        assert ec.space.parse(ec, "0b10") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(2))
        ]))

    def test_float(self, ec):
        assert ec.space.parse(ec, "0.2") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(0.2))
        ]))
        assert ec.space.parse(ec, "1E1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(10.0))
        ]))
        assert ec.space.parse(ec, "1e1") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantFloat(10.0))
        ]))

    def test_binary_expression(self, ec):
        assert ec.space.parse(ec, "1+1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "1/1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("/", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "1===1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("===", ast.ConstantInt(1), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, "2 % 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("%", ast.ConstantInt(2), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 =~ 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("=~", ast.ConstantInt(2), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 !~ 3") == ast.Main(ast.Block([
            ast.Statement(ast.Not(ast.BinOp("=~", ast.ConstantInt(2), ast.ConstantInt(3), 1)))
        ]))
        assert ec.space.parse(ec, "1 =~ /v/") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("=~", ast.ConstantInt(1), ast.ConstantRegexp("v"), 1))
        ]))
        assert ec.space.parse(ec, "2 & 3 | 5") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("|", ast.BinOp("&", ast.ConstantInt(2), ast.ConstantInt(3), 1), ast.ConstantInt(5), 1))
        ]))
        assert ec.space.parse(ec, "$a << []") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<<", ast.Global("$a"), ast.Array([]), 1))
        ]))
        assert ec.space.parse(ec, "3 >> 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp(">>", ast.ConstantInt(3), ast.ConstantInt(2), 1))
        ]))
        assert ec.space.parse(ec, "5 or 3") == ast.Main(ast.Block([
            ast.Statement(ast.Or(ast.ConstantInt(5),
                                        ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "puts 5 and 3") == ast.Main(ast.Block([
            ast.Statement(ast.And(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(5)],
                                                  None, 1),
                                         ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "x[0] == ?-") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("==",
                ast.Subscript(ast.Variable("x", 1), [ast.ConstantInt(0)], 1),
                ast.ConstantString("-"),
            1,))
        ]))
        assert ec.space.parse(ec, "@x-1") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("-", ast.InstanceVariable("@x"), ast.ConstantInt(1), 1))
        ]))
        assert ec.space.parse(ec, ":a <=> :a") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<=>", ast.ConstantSymbol("a"), ast.ConstantSymbol("a"), 1))
        ]))
        assert ec.space.parse(ec, ":a != ?-") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("!=", ast.ConstantSymbol("a"), ast.ConstantString("-"), 1))
        ]))
        assert ec.space.parse(ec, "1 ^ 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("^", ast.ConstantInt(1), ast.ConstantInt(2), 1))
        ]))
        assert ec.space.parse(ec, "1 ** 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("**", ast.ConstantInt(1), ast.ConstantInt(2), 1))
        ]))

    def test_multi_term_expr(self, ec):
        assert ec.space.parse(ec, "1 + 2 * 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.ConstantInt(1), ast.BinOp("*", ast.ConstantInt(2), ast.ConstantInt(3), 1), 1))
        ]))
        assert ec.space.parse(ec, "1 * 2 + 3") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("+", ast.BinOp("*", ast.ConstantInt(1), ast.ConstantInt(2), 1), ast.ConstantInt(3), 1))
        ]))
        assert ec.space.parse(ec, "2 << 3 * 4") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("<<", ast.ConstantInt(2), ast.BinOp("*", ast.ConstantInt(3), ast.ConstantInt(4), 1), 1))
        ]))

    def test_parens(self, ec):
        assert ec.space.parse(ec, "1 * (2 - 3)") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("*", ast.ConstantInt(1), ast.BinOp("-", ast.ConstantInt(2), ast.ConstantInt(3), 1), 1))
        ]))

    def test_multiple_statements_no_sep(self, ec):
        with self.raises("SyntaxError"):
            ec.space.parse(ec, "3 3")

    def test_multiple_statements(self, ec):
        r = ec.space.parse(ec, """
        1
        2
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
        ]))

    def test_multiple_statements_semicolon(self, ec):
        assert ec.space.parse(ec, "1; 2") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
        ]))

        assert ec.space.parse(ec, "1; 2; 3") == ast.Main(ast.Block([
            ast.Statement(ast.ConstantInt(1)),
            ast.Statement(ast.ConstantInt(2)),
            ast.Statement(ast.ConstantInt(3)),
        ]))

    def test_send(self, ec):
        assert ec.space.parse(ec, "puts 2") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "puts 1, 2") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(1), ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "puts(1, 2)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(1), ast.ConstantInt(2)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s 10") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s.to_i") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1), "to_i", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s()") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(10)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10)], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(*10)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.Splat(ast.ConstantInt(10))], None, 1))
        ]))
        assert ec.space.parse(ec, "2.to_s(10, *x)") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.ConstantInt(2), "to_s", [ast.ConstantInt(10), ast.Splat(ast.Variable("x", 1))], None, 1))
        ]))

    def test_assignment(self, ec):
        assert ec.space.parse(ec, "a = 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.ConstantInt(3)))
        ]))
        assert ec.space.parse(ec, "a = b = 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.Assignment(ast.Variable("b", 1), ast.ConstantInt(3))))
        ]))

    def test_multi_assignment(self, ec):
        assert ec.space.parse(ec, "a.x, b[:idx], c::Const, d = 3") == ast.Main(ast.Block([
            ast.Statement(ast.MultiAssignment(
                [
                    ast.Send(ast.Variable("a", 1), "x", [], None, 1),
                    ast.Subscript(ast.Variable("b", 1), [ast.ConstantSymbol("idx")], 1),
                    ast.LookupConstant(ast.Variable("c", 1), "Const", 1),
                    ast.Variable("d", 1),
                ],
                ast.ConstantInt(3)
            ))
        ]))
        assert ec.space.parse(ec, "a = 2, 3") == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("a", 1), ast.Array([ast.ConstantInt(2), ast.ConstantInt(3)])))
        ]))
        with self.raises("SyntaxError"):
            ec.space.parse(ec, "a, b += 3")

    def test_load_variable(self, ec):
        assert ec.space.parse(ec, "a") == ast.Main(ast.Block([
            ast.Statement(ast.Variable("a", 1))
        ]))

    def test_if_statement(self, ec):
        res = lambda lineno: ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.Send(ast.Self(lineno), "puts", [ast.ConstantInt(2)], None, lineno))
            ]), ast.Block([])))
        ]))
        assert ec.space.parse(ec, "if 3 then puts 2 end") == res(1)
        assert ec.space.parse(ec, """
        if 3
            puts 2
        end
        """) == res(3)
        assert ec.space.parse(ec, "if 3; puts 2 end") == res(1)
        assert ec.space.parse(ec, "if 3; end") == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([]), ast.Block([])))
        ]))
        r = ec.space.parse(ec, """
        if 0
            puts 2
            puts 3
            puts 4
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(0), ast.Block([
                ast.Statement(ast.Send(ast.Self(3), "puts", [ast.ConstantInt(2)], None, 3)),
                ast.Statement(ast.Send(ast.Self(4), "puts", [ast.ConstantInt(3)], None, 4)),
                ast.Statement(ast.Send(ast.Self(5), "puts", [ast.ConstantInt(4)], None, 5)),
            ]), ast.Block([])))
        ]))

    def test_else(self, ec):
        r = ec.space.parse(ec, """if 3 then 5 else 4 end""")
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ]), ast.Block([
                ast.Statement(ast.ConstantInt(4))
            ])))
        ]))

    def test_elsif(self, ec):
        r = ec.space.parse(ec, """
        if 3
            5
        elsif 4 == 2
            3
        elsif 3 == 1
            2
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.If(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ]), ast.Block([
                ast.Statement(ast.If(ast.BinOp("==", ast.ConstantInt(4), ast.ConstantInt(2), 4), ast.Block([
                    ast.Statement(ast.ConstantInt(3))
                ]), ast.Block([
                    ast.Statement(ast.If(ast.BinOp("==", ast.ConstantInt(3), ast.ConstantInt(1), 6), ast.Block([
                        ast.Statement(ast.ConstantInt(2))
                    ]), ast.Block([ast.Statement(ast.Variable("nil", -1))])))
                ])))
            ])))
        ]))

    def test_comparison_ops(self, ec):
        assert ec.space.parse(ec, "1 == 2; 1 < 2; 1 > 2; 1 != 2; 1 <= 2; 1 >= 2; 1 <=> 2") == ast.Main(ast.Block([
            ast.Statement(ast.BinOp("==", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp(">", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("!=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp(">=", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
            ast.Statement(ast.BinOp("<=>", ast.ConstantInt(1), ast.ConstantInt(2), 1)),
        ]))

    def test_while(self, ec):
        expected = ast.Main(ast.Block([
            ast.Statement(ast.While(ast.Variable("true", 1), ast.Block([
                ast.Statement(ast.Send(ast.Self(1), "puts", [ast.ConstantInt(5)], None, 1))
            ])))
        ]))
        assert ec.space.parse(ec, "while true do puts 5 end") == expected
        assert ec.space.parse(ec, "while true do; puts 5 end") == expected
        assert ec.space.parse(ec, "while true; puts 5 end") == expected
        assert ec.space.parse(ec, "while true; end") == ast.Main(ast.Block([
            ast.Statement(ast.While(ast.Variable("true", 1), ast.Block([
                ast.Statement(ast.Variable("nil", -1))
            ])))
        ]))

        res = ec.space.parse(ec, """
        i = 0
        while i < 10 do
            puts i
            puts 1
            puts i
            puts true
        end
        """)
        assert res == ast.Main(ast.Block([
            ast.Statement(ast.Assignment(ast.Variable("i", 2), ast.ConstantInt(0))),
            ast.Statement(ast.While(ast.BinOp("<", ast.Variable("i", 3), ast.ConstantInt(10), 3), ast.Block([
                ast.Statement(ast.Send(ast.Self(4), "puts", [ast.Variable("i", 4)], None, 4)),
                ast.Statement(ast.Send(ast.Self(5), "puts", [ast.ConstantInt(1)], None, 5)),
                ast.Statement(ast.Send(ast.Self(6), "puts", [ast.Variable("i", 6)], None, 6)),
                ast.Statement(ast.Send(ast.Self(7), "puts", [ast.Variable("true", 7)], None, 7)),
            ])))
        ]))

    def test_until(self, ec):
        r = ec.space.parse(ec, """
        until 3
            5
        end
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.Until(ast.ConstantInt(3), ast.Block([
                ast.Statement(ast.ConstantInt(5))
            ])))
        ]))

    def test_return(self, ec):
        assert ec.space.parse(ec, "return 4") == ast.Main(ast.Block([
            ast.Return(ast.ConstantInt(4))
        ]))
        assert ec.space.parse(ec, "return") == ast.Main(ast.Block([
            ast.Return(ast.Variable("nil", 1))
        ]))
        assert ec.space.parse(ec, "return 3, 4, 5") == ast.Main(ast.Block([
            ast.Return(ast.Array([
                ast.ConstantInt(3),
                ast.ConstantInt(4),
                ast.ConstantInt(5),
            ]))
        ]))

    def test_array(self, ec):
        assert ec.space.parse(ec, "[]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([]))
        ]))

        assert ec.space.parse(ec, "[1, 2, 3]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.ConstantInt(1),
                ast.ConstantInt(2),
                ast.ConstantInt(3),
            ]))
        ]))

        assert ec.space.parse(ec, "[[1], [2], [3]]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.Array([ast.ConstantInt(1)]),
                ast.Array([ast.ConstantInt(2)]),
                ast.Array([ast.ConstantInt(3)]),
            ]))
        ]))

        assert ec.space.parse(ec, "[1, 2,]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.ConstantInt(1),
                ast.ConstantInt(2),
            ]))
        ]))

        r = ec.space.parse(ec, """
        [
            f()
        ]
        """)
        assert r == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.Send(ast.Self(3), "f", [], None, 3),
            ])),
        ]))
        assert ec.space.parse(ec, "[1, *2, *3]") == ast.Main(ast.Block([
            ast.Statement(ast.Array([
                ast.ConstantInt(1),
                ast.Splat(ast.ConstantInt(2)),
                ast.Splat(ast.ConstantInt(3)),
            ]))
        ]))

    def test_subscript(self, ec):
        assert ec.space.parse(ec, "[1][0]") == ast.Main(ast.Block([
            ast.Statement(ast.Subscript(ast.Array([ast.ConstantInt(1)]), [ast.ConstantInt(0)], 1))
        ]))

        assert ec.space.parse(ec, "self[i]") == ast.Main(ast.Block([
            ast.Statement(ast.Subscript(ast.Variable("self", 1), [ast.Variable("i", 1)], 1))
        ]))

        assert ec.space.parse(ec, "self[i].to_s") == ast.Main(ast.Block([
            ast.Statement(ast.Send(ast.Subscript(ast.Variable("self", 1), [ast.Variable("i", 1)], 1), "to_s", [], None, 1))
        ]))
{'fraction_non_alphanumeric': 0.2002077982048544, 'fraction_numerical': 0.023377298046119658, 'mean_word_length': 2.033884948778566, 'pattern_counts': {'":': 4, '<': 39, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3178401', 'n_tokens_mistral': 12072, 'n_tokens_neox': 11386, 'n_words': 1848}