<commit_before><?php

require __DIR__ . '/../lib/bootstrap.php';

ini_set('xdebug.max_nesting_level', 2000);

// Disable XDebug var_dump() output truncation
ini_set('xdebug.var_display_max_children', -1);
ini_set('xdebug.var_display_max_data', -1);
ini_set('xdebug.var_display_max_depth', -1);

list($operations, $files) = parseArgs($argv);

/* Dump nodes by default */
if (empty($operations)) {
    $operations[] = 'dump';
}

if (empty($files)) {
    showHelp("Must specify at least one file.");
}

$parser = new PhpParser\Parser(new PhpParser\Lexer\Emulative);
$dumper = new PhpParser\NodeDumper;
$prettyPrinter = new PhpParser\PrettyPrinter\Standard;
$serializer = new PhpParser\Serializer\XML;

$traverser = new PhpParser\NodeTraverser();
$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);

foreach ($files as $file) {
    if (!file_exists($file)) {
        die("File $file does not exist.\n");
    }

    echo "====> File $file:\n";

    $code = file_get_contents($file);
    try {
        $stmts = $parser->parse($code);
    } catch (PhpParser\Error $e) {
        die("==> Parse Error: {$e->getMessage()}\n");
    }

    foreach ($operations as $operation) {
        if ('dump' === $operation) {
            echo "==> Node dump:\n";
            echo $dumper->dump($stmts), "\n";
        } elseif ('pretty-print' === $operation) {
            echo "==> Pretty print:\n";
            echo $prettyPrinter->prettyPrintFile($stmts), "\n";
        } elseif ('serialize-xml' === $operation) {
            echo "==> Serialized XML:\n";
            echo $serializer->serialize($stmts), "\n";
        } elseif ('var-dump' === $operation) {
            echo "==> var_dump():\n";
            var_dump($stmts);
        } elseif ('resolve-names' === $operation) {
            echo "==> Resolved names.\n";
            $stmts = $traverser->traverse($stmts);
        }
    }
}

function showHelp($error) {
    die($error . "\n\n" .
        <<<OUTPUT
Usage:

    php php-parse.php [operations] file1.php [file2.php ...]

Operations is a list of the following options (--dump by default):

    --dump           -d  Dump nodes using NodeDumper
    --pretty-print   -p  Pretty print file using PrettyPrinter\Standard
    --serialize-xml      Serialize nodes using Serializer\XML
    --var-dump           var_dump() nodes (for exact structure)
    --resolve-names  -N  Resolve names using NodeVisitor\NameResolver

Example:

    php php-parse.php -d -p -N -d file.php

    Dumps nodes, pretty prints them, then resolves names and dumps them again.
OUTPUT
    );
}

function parseArgs($args) {
    $operations = array();
    $files = array();

    array_shift($args);
    $parseOptions = true;
    foreach ($args as $arg) {<commit_msg>Add ability to pass code directly to php-parse.php<commit_after><?php

require __DIR__ . '/../lib/bootstrap.php';

ini_set('xdebug.max_nesting_level', 2000);

// Disable XDebug var_dump() output truncation
ini_set('xdebug.var_display_max_children', -1);
ini_set('xdebug.var_display_max_data', -1);
ini_set('xdebug.var_display_max_depth', -1);

list($operations, $files) = parseArgs($argv);

/* Dump nodes by default */
if (empty($operations)) {
    $operations[] = 'dump';
}

if (empty($files)) {
    showHelp("Must specify at least one file.");
}

$parser = new PhpParser\Parser(new PhpParser\Lexer\Emulative);
$dumper = new PhpParser\NodeDumper;
$prettyPrinter = new PhpParser\PrettyPrinter\Standard;
$serializer = new PhpParser\Serializer\XML;

$traverser = new PhpParser\NodeTraverser();
$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);

foreach ($files as $file) {
    if (strpos($file, '<?php') === 0) {
        $code = $file;
        echo "====> Code $code\n";
    } else {
        if (!file_exists($file)) {
            die("File $file does not exist.\n");
        }

        $code = file_get_contents($file);
        echo "====> File $file:\n";
    }

    try {
        $stmts = $parser->parse($code);
    } catch (PhpParser\Error $e) {
        die("==> Parse Error: {$e->getMessage()}\n");
    }

    foreach ($operations as $operation) {
        if ('dump' === $operation) {
            echo "==> Node dump:\n";
            echo $dumper->dump($stmts), "\n";
        } elseif ('pretty-print' === $operation) {
            echo "==> Pretty print:\n";
            echo $prettyPrinter->prettyPrintFile($stmts), "\n";
        } elseif ('serialize-xml' === $operation) {
            echo "==> Serialized XML:\n";
            echo $serializer->serialize($stmts), "\n";
        } elseif ('var-dump' === $operation) {
            echo "==> var_dump():\n";
            var_dump($stmts);
        } elseif ('resolve-names' === $operation) {
            echo "==> Resolved names.\n";
            $stmts = $traverser->traverse($stmts);
        }
    }
}

function showHelp($error) {
    die($error . "\n\n" .
        <<<OUTPUT
Usage:

    php php-parse.php [operations] file1.php [file2.php ...]

The file arguments can also be replaced with a code string:

    php php-parse.php [operations] "<?php code"

Operations is a list of the following options (--dump by default):

    --dump           -d  Dump nodes using NodeDumper
    --pretty-print   -p  Pretty print file using PrettyPrinter\Standard
    --serialize-xml      Serialize nodes using Serializer\XML
    --var-dump           var_dump() nodes (for exact structure)
    --resolve-names  -N  Resolve names using NodeVisitor\NameResolver

Example:

    php php-parse.php -d -p -N -d file.php

    Dumps nodes, pretty prints them, then resolves names and dumps them again.
OUTPUT
    );
}

function parseArgs($args) {
    $operations = array();
    $files = array();

    array_shift($args);
    $parseOptions = true;
    foreach ($args as $arg) {
{'fraction_non_alphanumeric': 0.1617954070981211, 'fraction_numerical': 0.0033054975643702156, 'mean_word_length': 2.723445595854922, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499614', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1844, 'n_words': 455}