<commit_before><?php

/*
 * Copyright 2011 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * Parse XHP (or PHP) source files into @{class:DivinerAtom}s.
 */
class DivinerXHPEngine extends DivinerEngine {

  private $trees;

  public function buildFileContentHashes() {
    $files = array();
    $root = $this->getConfiguration()->getProjectRoot();

    $finder = new FileFinder($root);
    $finder
      ->excludePath('*/.*')
      ->withSuffix('php')
      ->withType('f')
      ->setGenerateChecksums(true);

    foreach ($finder->find() as $path => $hash) {
      $path = Filesystem::readablePath($path, $root);
      $files[$path] = $hash;
    }

    return $files;
  }

  public function willParseFiles(array $file_map) {
    $futures = array();
    foreach ($file_map as $file => $data) {
      $futures[$file] = xhpast_get_parser_future($data);
    }

    foreach (Futures($futures)->limit(8) as $file => $future) {
      $this->trees[$file] = XHPASTTree::newFromDataAndResolvedExecFuture(
        $file_map[$file],
        $future->resolve());
    }
  }

  public function parseFile($file, $data) {

    $tree = $this->trees[$file]->getRootNode();

    $atoms = array();

    $func_decl = $tree->selectDescendantsOfType('n_FUNCTION_DECLARATION');
    foreach ($func_decl as $func) {
      $name = $func->getChildByIndex(2);

      $atom = new DivinerFunctionAtom();
      $atom->setName($name->getConcreteString());
      $atom->setLine($func->getLineNumber());
      $atom->setFile($file);

      $this->findAtomDocblock($atom, $func);

      $params = $func
        ->getChildByIndex(3)
        ->selectDescendantsOfType('n_DECLARATION_PARAMETER');
      $this->parseParams($atom, $params);
      $this->parseReturnType($atom, $func);

      $atoms[] = $atom;
    }

    $class_decls = $tree->selectDescendantsOfType('n_CLASS_DECLARATION');
    foreach ($class_decls as $class) {
      $name = $class->getChildByIndex(1);

      $atom = new DivinerClassAtom();
      $atom->setName($name->getConcreteString());
      $atom->setLine($class->getLineNumber());
      $atom->setFile($file);

      $extends = $class->getChildByIndex(2);
      $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
      foreach ($extends_class as $parent_class) {
        $atom->addParentClass($parent_class->getConcreteString());
      }

      $this->findAtomDocblock($atom, $class);

      $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
      foreach ($methods as $method) {
        $matom = new DivinerMethodAtom();
        $this->findAtomDocblock($matom, $method);

        $attribute_list = $method->getChildByIndex(0);
        $attributes = $attribute_list->selectDescendantsOfType('n_STRING');
        if ($attributes) {
          foreach ($attributes as $attribute) {
            $matom->setAttribute(strtolower($attribute->getConcreteString()));
          }
        } else {
          $matom->setAttribute('public');
        }

        $params = $method
          ->getChildByIndex(3)
          ->selectDescendantsOfType('n_DECLARATION_PARAMETER');
        $this->parseParams($matom, $params);

        $matom->setName($method->getChildByIndex(2)->getConcreteString());
        $matom->setFile($file);
        $matom->setLine($method->getLineNumber());

        if ($matom->getName() == '__construct') {
          $matom->setReturnTypeAttributes(array(
            'doctype' => 'this',
          ));
        } else {
          $this->parseReturnType($matom, $method);
        }
        $atom->addMethod($matom);
      }

      $atoms[] = $atom;
    }

    $file_atom = new DivinerFileAtom();
    $file_atom->setName($file);
    $file_atom->setFile($file);
    foreach ($atoms as $atom) {
      $file_atom->addChild($atom);
    }

    $this->trees[$file]->dispose();
    unset($this->trees[$file]);

    return array($file_atom);
  }


  private function findAtomDocblock(DivinerAtom $atom, XHPASTNode $node) {
    $token = $node->getDocblockToken();
    if ($token) {
      $atom->setRawDocblock($token->getValue());
      return true;
    } else {
      return false;
    }
  }

  private function parseParams(DivinerAtom $atom, XHPASTNodeList $params) {
    $metadata = $atom->getDocblockMetadata();
    $docs = idx($metadata, 'param', '');
    if ($docs) {
      $docs = explode("\n", $docs);
    }

    foreach ($params as $param) {
      $name = $param->getChildByIndex(1);
      $dict = array(
        'type'    => $param->getChildByIndex(0)->getConcreteString(),
        'default' => $param->getChildByIndex(2)->getConcreteString(),
      );
      if ($docs) {
        $doc = array_shift($docs);
        if ($doc) {
          $split = preg_split('/\s+/', trim($doc), $limit = 2);
          if (!empty($split[0])) {
            $dict['doctype'] = $split[0];
          }
          if (!empty($split[1])) {
            $dict['docs'] = $split[1];
          }
        }
      }
      $atom->addParameter($name->getConcreteString(), $dict);<commit_msg>Fix an XHPASTNodeList typehint in Diviner<commit_after><?php

/*
 * Copyright 2011 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * Parse XHP (or PHP) source files into @{class:DivinerAtom}s.
 */
class DivinerXHPEngine extends DivinerEngine {

  private $trees;

  public function buildFileContentHashes() {
    $files = array();
    $root = $this->getConfiguration()->getProjectRoot();

    $finder = new FileFinder($root);
    $finder
      ->excludePath('*/.*')
      ->withSuffix('php')
      ->withType('f')
      ->setGenerateChecksums(true);

    foreach ($finder->find() as $path => $hash) {
      $path = Filesystem::readablePath($path, $root);
      $files[$path] = $hash;
    }

    return $files;
  }

  public function willParseFiles(array $file_map) {
    $futures = array();
    foreach ($file_map as $file => $data) {
      $futures[$file] = xhpast_get_parser_future($data);
    }

    foreach (Futures($futures)->limit(8) as $file => $future) {
      $this->trees[$file] = XHPASTTree::newFromDataAndResolvedExecFuture(
        $file_map[$file],
        $future->resolve());
    }
  }

  public function parseFile($file, $data) {

    $tree = $this->trees[$file]->getRootNode();

    $atoms = array();

    $func_decl = $tree->selectDescendantsOfType('n_FUNCTION_DECLARATION');
    foreach ($func_decl as $func) {
      $name = $func->getChildByIndex(2);

      $atom = new DivinerFunctionAtom();
      $atom->setName($name->getConcreteString());
      $atom->setLine($func->getLineNumber());
      $atom->setFile($file);

      $this->findAtomDocblock($atom, $func);

      $params = $func
        ->getChildByIndex(3)
        ->selectDescendantsOfType('n_DECLARATION_PARAMETER');
      $this->parseParams($atom, $params);
      $this->parseReturnType($atom, $func);

      $atoms[] = $atom;
    }

    $class_decls = $tree->selectDescendantsOfType('n_CLASS_DECLARATION');
    foreach ($class_decls as $class) {
      $name = $class->getChildByIndex(1);

      $atom = new DivinerClassAtom();
      $atom->setName($name->getConcreteString());
      $atom->setLine($class->getLineNumber());
      $atom->setFile($file);

      $extends = $class->getChildByIndex(2);
      $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
      foreach ($extends_class as $parent_class) {
        $atom->addParentClass($parent_class->getConcreteString());
      }

      $this->findAtomDocblock($atom, $class);

      $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
      foreach ($methods as $method) {
        $matom = new DivinerMethodAtom();
        $this->findAtomDocblock($matom, $method);

        $attribute_list = $method->getChildByIndex(0);
        $attributes = $attribute_list->selectDescendantsOfType('n_STRING');
        if ($attributes) {
          foreach ($attributes as $attribute) {
            $matom->setAttribute(strtolower($attribute->getConcreteString()));
          }
        } else {
          $matom->setAttribute('public');
        }

        $params = $method
          ->getChildByIndex(3)
          ->selectDescendantsOfType('n_DECLARATION_PARAMETER');
        $this->parseParams($matom, $params);

        $matom->setName($method->getChildByIndex(2)->getConcreteString());
        $matom->setFile($file);
        $matom->setLine($method->getLineNumber());

        if ($matom->getName() == '__construct') {
          $matom->setReturnTypeAttributes(array(
            'doctype' => 'this',
          ));
        } else {
          $this->parseReturnType($matom, $method);
        }
        $atom->addMethod($matom);
      }

      $atoms[] = $atom;
    }

    $file_atom = new DivinerFileAtom();
    $file_atom->setName($file);
    $file_atom->setFile($file);
    foreach ($atoms as $atom) {
      $file_atom->addChild($atom);
    }

    $this->trees[$file]->dispose();
    unset($this->trees[$file]);

    return array($file_atom);
  }


  private function findAtomDocblock(DivinerAtom $atom, XHPASTNode $node) {
    $token = $node->getDocblockToken();
    if ($token) {
      $atom->setRawDocblock($token->getValue());
      return true;
    } else {
      return false;
    }
  }

  private function parseParams(DivinerAtom $atom, AASTNodeList $params) {
    $metadata = $atom->getDocblockMetadata();
    $docs = idx($metadata, 'param', '');
    if ($docs) {
      $docs = explode("\n", $docs);
    }

    foreach ($params as $param) {
      $name = $param->getChildByIndex(1);
      $dict = array(
        'type'    => $param->getChildByIndex(0)->getConcreteString(),
        'default' => $param->getChildByIndex(2)->getConcreteString(),
      );
      if ($docs) {
        $doc = array_shift($docs);
        if ($doc) {
          $split = preg_split('/\s+/', trim($doc), $limit = 2);
          if (!empty($split[0])) {
            $dict['doctype'] = $split[0];
          }
          if (!empty($split[1])) {
            $dict['docs'] = $split[1];
          }
        }
      }
      $atom->addParameter($name->getConcreteString(), $dict);
{'fraction_non_alphanumeric': 0.16320260256641966, 'fraction_numerical': 0.004337610699439725, 'mean_word_length': 2.918909348441926, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 161, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7373373', 'n_tokens_mistral': 3523, 'n_tokens_neox': 3432, 'n_words': 662}