<commit_before><?php

namespace CtSearchBundle\Datasource;

use Symfony\Component\Validator\Constraints as Assert;
use \CtSearchBundle\CtSearchBundle;

class OAIHarvester extends Datasource {

  private $oaiServerUrl;
  private $sets;
  private $metaDataPrefix;
  private $cookies = '';

  public function getSettings() {
    return array(
      'oaiServerUrl' => $this->getOaiServerUrl() != null ? $this->getOaiServerUrl() : '',
      'sets' => $this->getSets() != null ? $this->getSets() : '',
      'metaDataPrefix' => $this->getMetaDataPrefix() != null ? $this->getMetaDataPrefix() : '',
    );
  }

  public function initFromSettings($settings) {
    foreach ($settings as $k => $v) {
      $this->{$k} = $v;
    }
  }

  public function execute($execParams = null) {
    $sets = array_map('trim', explode(',', $this->getSets()));
    $count = 0;
    if (count($sets) > 0) {
      foreach ($sets as $set) {
        $count += $this->harvest($set);
      }
    } else {
      $this->harvest(NULL);
    }
    if ($this->getController() != null) {
      CtSearchBundle::addSessionMessage($this->getController(), 'status', 'Found ' . $count . ' documents');
    }
  }
  
  public function runCli($token){
    $sets = array_map('trim', explode(',', $this->getSets()));
    if (count($sets) > 0) {
      foreach ($sets as $set) {
        $this->harvest($set, $token, 0, true);
      }
    } else {
      $this->harvest(NULL, $token, 0, true);
    }
  }

  private function harvest($set, $resumptionToken = null, $count = 0, $cli = false) {
    $doc = new \DOMDocument();
    if ($resumptionToken == null)
      $url = $this->getOaiServerUrl() . '?verb=ListRecords&metadataPrefix=' . $this->getMetaDataPrefix() . ($set != NULL ? '&set=' . $set : '');
    else
      $url = $this->getOaiServerUrl() . '?verb=ListRecords&resumptionToken=' . urlencode($resumptionToken);
    if ($this->getOutput() != null) {
      $this->getOutput()->writeln('Harvesting url ' . $url);
    }
    $doc->loadXML($this->getContentFromUrl($url));
    $xpath = new \DOMXPath($doc);
    $result = $xpath->query("//namespace::*");

    foreach ($result as $node) {
      if ($node->nodeName == 'xmlns') {
        $xpath->registerNamespace('oai', $node->nodeValue);
      }
    }
    $items = $xpath->query('oai:ListRecords/oai:record');
    foreach ($items as $item) {
      $document = array();
      if ($xpath->query('oai:header/oai:identifier', $item)->length > 0)
        $document['identifier'] = $xpath->query('oai:header/oai:identifier', $item)->item(0)->textContent;
      if ($xpath->query('oai:header/oai:datestamp', $item)->length > 0)
        $document['datestamp'] = $xpath->query('oai:header/oai:datestamp', $item)->item(0)->textContent;
      if ($xpath->query('oai:metadata/*', $item)->length > 0)
        $document['metadata'] = '<?xml version="1.0" encoding="' . $doc->encoding . '"?>' . simplexml_import_dom($xpath->query('oai:metadata/*', $item)->item(0))->asXML();

      if ($this->getOutput() != null) {
        $this->getOutput()->writeln(($count + 1) . ' / Harvesting doc "' . $document['identifier'] . '"');
        $this->getOutput()->writeln(sprintf('Memory usage (currently) %dKB/ (max) %dKB', round(memory_get_usage(true) / 1024), memory_get_peak_usage(true) / 1024));
      }

      $this->index($document);
      unset($document);
      $count ++;
    }
    unset($items);
    if (isset($item))
      unset($item);
    if ($xpath->query('oai:ListRecords/oai:resumptionToken')->length > 0 && !empty($xpath->query('oai:ListRecords/oai:resumptionToken')->item(0)->textContent)) {
      $token = $xpath->query('oai:ListRecords/oai:resumptionToken')->item(0)->textContent;
      unset($result);
      unset($xpath);
      unset($doc);
      gc_enable();
      gc_collect_cycles();
      if(!$cli){
        $this->harvest($set, $token, $count);
      }
      else{
        $this->getOutput()->writeln($token);
        exit(9);
      }
    }
    if($cli){
      exit(0);
    }
    return $count;
  }

  private function getContentFromUrl($url) {
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_HEADER, true);
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    if(!empty($this->cookies)){
      curl_setopt($ch, CURLOPT_COOKIE, $this->cookies);
    }
    $r = curl_exec($ch);
    $response = $this->parseHttpResponse($r);
    if(isset($response['headers']['set-cookie'])){
      $this->cookies = $response['headers']['set-cookie'];
    }
    return $response['content'];
  }

  private function parseHttpResponse($string) {

    $headers = array();
    $content = '';
    $str = strtok($string, "\n");
    $h = null;
    while ($str !== false) {<commit_msg>Bug related to non XML utf8 characters in OAI harvester<commit_after><?php

namespace CtSearchBundle\Datasource;

use Symfony\Component\Validator\Constraints as Assert;
use \CtSearchBundle\CtSearchBundle;

class OAIHarvester extends Datasource {

  private $oaiServerUrl;
  private $sets;
  private $metaDataPrefix;
  private $cookies = '';

  public function getSettings() {
    return array(
      'oaiServerUrl' => $this->getOaiServerUrl() != null ? $this->getOaiServerUrl() : '',
      'sets' => $this->getSets() != null ? $this->getSets() : '',
      'metaDataPrefix' => $this->getMetaDataPrefix() != null ? $this->getMetaDataPrefix() : '',
    );
  }

  public function initFromSettings($settings) {
    foreach ($settings as $k => $v) {
      $this->{$k} = $v;
    }
  }

  public function execute($execParams = null) {
    $sets = array_map('trim', explode(',', $this->getSets()));
    $count = 0;
    if (count($sets) > 0) {
      foreach ($sets as $set) {
        $count += $this->harvest($set);
      }
    } else {
      $this->harvest(NULL);
    }
    if ($this->getController() != null) {
      CtSearchBundle::addSessionMessage($this->getController(), 'status', 'Found ' . $count . ' documents');
    }
  }
  
  public function runCli($token){
    $sets = array_map('trim', explode(',', $this->getSets()));
    if (count($sets) > 0) {
      foreach ($sets as $set) {
        $this->harvest($set, $token, 0, true);
      }
    } else {
      $this->harvest(NULL, $token, 0, true);
    }
  }

  private function harvest($set, $resumptionToken = null, $count = 0, $cli = false) {
    $doc = new \DOMDocument();
    if ($resumptionToken == null)
      $url = $this->getOaiServerUrl() . '?verb=ListRecords&metadataPrefix=' . $this->getMetaDataPrefix() . ($set != NULL ? '&set=' . $set : '');
    else
      $url = $this->getOaiServerUrl() . '?verb=ListRecords&resumptionToken=' . urlencode($resumptionToken);
    if ($this->getOutput() != null) {
      $this->getOutput()->writeln('Harvesting url ' . $url);
    }
    $doc->loadXML($this->getContentFromUrl($url));
    $xpath = new \DOMXPath($doc);
    $result = $xpath->query("//namespace::*");

    foreach ($result as $node) {
      if ($node->nodeName == 'xmlns') {
        $xpath->registerNamespace('oai', $node->nodeValue);
      }
    }
    $items = $xpath->query('oai:ListRecords/oai:record');
    foreach ($items as $item) {
      $document = array();
      if ($xpath->query('oai:header/oai:identifier', $item)->length > 0)
        $document['identifier'] = $xpath->query('oai:header/oai:identifier', $item)->item(0)->textContent;
      if ($xpath->query('oai:header/oai:datestamp', $item)->length > 0)
        $document['datestamp'] = $xpath->query('oai:header/oai:datestamp', $item)->item(0)->textContent;
      if ($xpath->query('oai:metadata/*', $item)->length > 0)
        $document['metadata'] = '<?xml version="1.0" encoding="' . $doc->encoding . '"?>' . simplexml_import_dom($xpath->query('oai:metadata/*', $item)->item(0))->asXML();

      if ($this->getOutput() != null) {
        $this->getOutput()->writeln(($count + 1) . ' / Harvesting doc "' . $document['identifier'] . '"');
        $this->getOutput()->writeln(sprintf('Memory usage (currently) %dKB/ (max) %dKB', round(memory_get_usage(true) / 1024), memory_get_peak_usage(true) / 1024));
      }

      $this->index($document);
      unset($document);
      $count ++;
    }
    unset($items);
    if (isset($item))
      unset($item);
    if ($xpath->query('oai:ListRecords/oai:resumptionToken')->length > 0 && !empty($xpath->query('oai:ListRecords/oai:resumptionToken')->item(0)->textContent)) {
      $token = $xpath->query('oai:ListRecords/oai:resumptionToken')->item(0)->textContent;
      unset($result);
      unset($xpath);
      unset($doc);
      gc_enable();
      gc_collect_cycles();
      if(!$cli){
        $this->harvest($set, $token, $count);
      }
      else{
        $this->getOutput()->writeln($token);
        exit(9);
      }
    }
    if($cli){
      exit(0);
    }
    return $count;
  }

  private function getContentFromUrl($url) {
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_HEADER, true);
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    if(!empty($this->cookies)){
      curl_setopt($ch, CURLOPT_COOKIE, $this->cookies);
    }
    $r = curl_exec($ch);
    $response = $this->parseHttpResponse($r);
    if(isset($response['headers']['set-cookie'])){
      $this->cookies = $response['headers']['set-cookie'];
    }
    return preg_replace('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $response['content']);
  }

  private function parseHttpResponse($string) {

    $headers = array();
    $content = '';
    $str = strtok($string, "\n");
    $h = null;
    while ($str !== false) {
{'fraction_non_alphanumeric': 0.20404636030906873, 'fraction_numerical': 0.008031720211468077, 'mean_word_length': 3.3681172291296626, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 2, '>': 155, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '245507', 'n_tokens_mistral': 3447, 'n_tokens_neox': 3319, 'n_words': 568}