<commit_before><?php

final class DivinerPHPAtomizer extends DivinerAtomizer {

  protected function newAtom($type) {
    return parent::newAtom($type)->setLanguage('php');
  }


  protected function executeAtomize($file_name, $file_data) {
    $future = xhpast_get_parser_future($file_data);
    $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
      $file_data,
      $future->resolve());

    $atoms = array();

    $root = $tree->getRootNode();

    $func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
    foreach ($func_decl as $func) {

      $name = $func->getChildByIndex(2);

      $atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION)
        ->setName($name->getConcreteString())
        ->setLine($func->getLineNumber())
        ->setFile($file_name);

      $this->findAtomDocblock($atom, $func);

      $this->parseParams($atom, $func);
      $this->parseReturnType($atom, $func);

      $atoms[] = $atom;
    }

    $class_types = array(
      DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION',
      DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION',
    );
    foreach ($class_types as $atom_type => $node_type) {
      $class_decls = $root->selectDescendantsOfType($node_type);
      foreach ($class_decls as $class) {
        $name = $class->getChildByIndex(1, 'n_CLASS_NAME');

        $atom = $this->newAtom($atom_type)
          ->setName($name->getConcreteString())
          ->setFile($file_name)
          ->setLine($class->getLineNumber());

        // TODO: Parse "abstract" and "final".

        // If this exists, it is n_EXTENDS_LIST.
        $extends = $class->getChildByIndex(2);
        $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
        foreach ($extends_class as $parent_class) {
          $atom->addExtends(
            $this->newRef(
              DivinerAtom::TYPE_CLASS,
              $parent_class->getConcreteString()));
        }

        // If this exists, it is n_IMPLEMENTS_LIST.
        $implements = $class->getChildByIndex(3);
        $iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME');
        foreach ($iface_names as $iface_name) {
          $atom->addExtends(
            $this->newRef(
              DivinerAtom::TYPE_INTERFACE,
              $iface_name->getConcreteString()));
        }

        $this->findAtomDocblock($atom, $class);

        $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');<commit_msg>In Diviner, parse "abstract" and "final" in PHP classes, interfaces and methods<commit_after><?php

final class DivinerPHPAtomizer extends DivinerAtomizer {

  protected function newAtom($type) {
    return parent::newAtom($type)->setLanguage('php');
  }


  protected function executeAtomize($file_name, $file_data) {
    $future = xhpast_get_parser_future($file_data);
    $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
      $file_data,
      $future->resolve());

    $atoms = array();

    $root = $tree->getRootNode();

    $func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
    foreach ($func_decl as $func) {

      $name = $func->getChildByIndex(2);

      $atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION)
        ->setName($name->getConcreteString())
        ->setLine($func->getLineNumber())
        ->setFile($file_name);

      $this->findAtomDocblock($atom, $func);

      $this->parseParams($atom, $func);
      $this->parseReturnType($atom, $func);

      $atoms[] = $atom;
    }

    $class_types = array(
      DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION',
      DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION',
    );
    foreach ($class_types as $atom_type => $node_type) {
      $class_decls = $root->selectDescendantsOfType($node_type);
      foreach ($class_decls as $class) {
        $name = $class->getChildByIndex(1, 'n_CLASS_NAME');

        $atom = $this->newAtom($atom_type)
          ->setName($name->getConcreteString())
          ->setFile($file_name)
          ->setLine($class->getLineNumber());

        // This parses "final" and "abstract".
        $attributes = $class->getChildByIndex(0, 'n_CLASS_ATTRIBUTES');
        foreach ($attributes->selectDescendantsOfType('n_STRING') as $attr) {
          $atom->setProperty($attr->getConcreteString(), true);
        }

        // If this exists, it is n_EXTENDS_LIST.
        $extends = $class->getChildByIndex(2);
        $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
        foreach ($extends_class as $parent_class) {
          $atom->addExtends(
            $this->newRef(
              DivinerAtom::TYPE_CLASS,
              $parent_class->getConcreteString()));
        }

        // If this exists, it is n_IMPLEMENTS_LIST.
        $implements = $class->getChildByIndex(3);
        $iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME');
        foreach ($iface_names as $iface_name) {
          $atom->addExtends(
            $this->newRef(
              DivinerAtom::TYPE_INTERFACE,
              $iface_name->getConcreteString()));
        }

        $this->findAtomDocblock($atom, $class);

        $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
{'fraction_non_alphanumeric': 0.14539143848823757, 'fraction_numerical': 0.0017354415734670266, 'mean_word_length': 3.05234375, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3952489', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1635, 'n_words': 251}