<commit_before><?php
/**
 * checks if the current user has sudo privileges. returns TRUE or FALSE
 * @return boolean
 */
function ch_check_sudo() {
  return shell_exec('sudo -n uptime 2>&1|grep "load"|wc -l')*1 === 1;
}

/**
 * attempts to start collectd rrd stats for the current test iteration. returns
 * TRUE on success, FALSE otherwise
 * @param string $dir the base directory where collectd rrd files are stored
 * @param boolean $verbose enable verbose output
 * @return boolean
 */
function ch_collectd_rrd_start($dir, $verbose) {
  $started = FALSE;
  if (is_dir($dir) && ch_check_sudo()) {
    exec(sprintf('sudo rm -rf %s/*.bak', $dir));
    $d = dir($dir);
    while($entry = $d->read()) {
      if (!preg_match('/[a-zA-Z0-9]+/', $entry)) continue;
      $rdir = sprintf('%s/%s', $dir, $entry);
      if (is_dir($rdir) && !preg_match('/\.bak$/', $entry)) {
        $rdirBak = $rdir . '.bak';
        exec(sprintf('sudo mv %s %s', $rdir, $rdirBak));
        if (is_dir($rdirBak)) {
          $started = TRUE;
          print_msg(sprintf('Successfully renamed existing collectd rrd directory from %s to %s', $rdir, $rdirBak), $verbose, __FILE__, __LINE__);
        }
        else {
          $started = FALSE;
          print_msg(sprintf('Unable to rename existing collectd rrd directory from %s to %s', $rdir, $rdirBak), $verbose, __FILE__, __LINE__, TRUE);
        }
      }
    }
    $d->close();
  }
  return $started;
}

/**
 * attempts to stop collectd rrd stats for the current test iteration. returns
 * TRUE on success, FALSE otherwise
 * @param string $dir the base directory where collectd rrd files are stored
 * @param string $saveTo the directory where the rrd zip file should be created
 * @param array $options test options - used to construct collectd rrd results 
 * directories
 * @param boolean $verbose enable verbose output
 * @return boolean
 */
function ch_collectd_rrd_stop($dir, $saveTo, $verbose) {
  $stopped = FALSE;
  if (is_dir($dir) && ch_check_sudo()) {
    $tdir = '/tmp/' . rand();
    exec(sprintf('sudo mkdir %s', $tdir));
    print_msg(sprintf('Attempting to save collectd rrd files in %s to %s/collectd-rrd.zip using tmp directory %s', $dir, $saveTo, $tdir), $verbose, __FILE__, __LINE__);
    
    $rename = array();
    $d = dir($dir);
    while($entry = $d->read()) {
      if (!preg_match('/[a-zA-Z0-9]+/', $entry)) continue;
      $rdir = sprintf('%s/%s', $dir, $entry);
      if (substr($rdir, -1) == '/') $rdir = substr($rdir, 0, -1);
      if (!is_dir($rdir)) continue;
      if (is_dir($rdir)) {
        print_msg(sprintf('Evaluating collectd rrd directory %s', $rdir), $verbose, __FILE__, __LINE__);
        if (preg_match('/\.bak$/', $entry)) $rename[] = $rdir;
        else {
          exec($cmd = sprintf('sudo mv %s %s', $rdir, $tdir));
          print_msg(sprintf('Successfully moved test collectd rrd directory %s to %s', basename($rdir), $tdir), $verbose, __FILE__, __LINE__);
        }
      }
    }
    $d->close();
    foreach($rename as $rdir) {
      $renameTo = substr($rdir, 0, -4);
      exec($cmd = sprintf('sudo rm -rf %s;sudo mv %s %s', $renameTo, $rdir, $renameTo));
      if (is_dir($renameTo)) print_msg(sprintf('Successfully renamed backup collectd rrd directory from %s to %s', $rdir, $renameTo), $verbose, __FILE__, __LINE__);
      else print_msg(sprintf('Unable to rename existing collectd rrd directory from %s to %s', $rdir, $renameTo), $verbose, __FILE__, __LINE__, TRUE);
    }
    
    if ((shell_exec(sprintf('find %s -maxdepth 1 -type d 2>/dev/null | wc -l', $tdir))*1 < 2)) print_msg(sprintf('collectd rrd directory %s did not contain any files', $dir), $verbose, __FILE__, __LINE__, TRUE);
    else {
      $zip = sprintf('%s/collectd-rrd.zip', $saveTo);
      exec($cmd = sprintf('cd %s;sudo zip -r collectd-rrd.zip *;sudo mv collectd-rrd.zip %s', $tdir, $saveTo, $tdir));
      if ($stopped = file_exists($zip) && filesize($zip)) print_msg(sprintf('Successfully saved collectd rrd files to %s', $zip), $verbose, __FILE__, __LINE__);
      else print_msg(sprintf('Unable to save collectd rrd files: %s', $cmd), $verbose, __FILE__, __LINE__, TRUE);
    }
  }
  else print_msg(sprintf('collectd rrd directory %s does not exist or user does not have sudo access', $dir), $verbose, __FILE__, __LINE__, TRUE);
  
  // sleep for 10 seconds to allow collectd to restart
  sleep(10);
  
  return $stopped;
}

/**
 * Invokes an http request and returns the status code (or response body if 
 * $retBody is TRUE) on success, NULL on failure or FALSE if the response code 
 * is not within the $success range
 * @param string  $url the target url
 * @param string $method the http method
 * @param array $headers optional request headers to include (hash or array)
 * @param string $file optional file to pipe into the curl process as the 
 * body
 * @param string $auth optional [user]:[pswd] to use for http authentication
 * @param string $success the http response code/range that consistitutes 
 * a successful request. defaults to 200 to 299. This parameter may be a comma
 * separated list of values or ranges (e.g. "200,404" or "200-299,404")
 * @param boolean $retBody whether or not to return the response body. If 
 * FALSE (default), the status code is returned
 * @return mixed
 */
function ch_curl($url, $method='HEAD', $headers=NULL, $file=NULL, $auth=NULL, $success='200-299', $retBody=FALSE) {
  global $ch_curl_options;
  if (!isset($ch_curl_options)) $ch_curl_options = parse_args(array('v' => 'verbose'));
  
  if (!is_array($headers)) $headers = array();
  $ofile = $retBody ? '/tmp/' . rand() : '/dev/null';
  $curl = sprintf('curl -s -X %s%s -w "%s\n" -o %s', $method, $method == 'HEAD' ? ' -I' : '', '%{http_code}', $ofile);
  if ($auth) $curl .= sprintf(' -u "%s"', $auth);
  if (is_array($headers)) {
    foreach($headers as $header => $val) $curl .= sprintf(' -H "%s%s"', is_numeric($header) ? '' : $header . ':', $val); 
  }
  // input file
  if (($method == 'POST' || $method == 'PUT') && file_exists($file)) {
    $curl .= sprintf(' --data-binary @%s', $file);
    if (!isset($headers['Content-Length']) && !isset($headers['content-length'])) $curl .= sprintf(' -H "Content-Length:%d"', filesize($file));
    if (!isset($headers['Content-Type']) && !isset($headers['content-type'])) $curl .= sprintf(' -H "Content-Type:%d"', get_mime_type($file));
  }
  $curl .= sprintf(' "%s"', $url);
  $ok = array();
  foreach(explode(',', $success) as $range) {
    if (is_numeric($range)) $ok[$range*1] = TRUE;
    else if (preg_match('/^([0-9]+)\s*\-\s*([0-9]+)$/', $range, $m) && $m[1] <= $m[2]) {
      for($i=$m[1]; $i<=$m[2]; $i++) $ok[$i] = TRUE;
    }
  }
  $ok = array_keys($ok);
  sort($ok);
  
  $cmd = sprintf('%s 2>/dev/null;echo $?', $curl);
  print_msg(sprintf('Invoking curl request: %s (expecting response %s)', $curl, $success), isset($ch_curl_options['verbose']), __FILE__, __LINE__);
  
  // execute curl
  $result = shell_exec($cmd);
  $output = explode("\n", trim($result));<commit_msg>fixed content-type header for callbacks<commit_after><?php
/**
 * checks if the current user has sudo privileges. returns TRUE or FALSE
 * @return boolean
 */
function ch_check_sudo() {
  return shell_exec('sudo -n uptime 2>&1|grep "load"|wc -l')*1 === 1;
}

/**
 * attempts to start collectd rrd stats for the current test iteration. returns
 * TRUE on success, FALSE otherwise
 * @param string $dir the base directory where collectd rrd files are stored
 * @param boolean $verbose enable verbose output
 * @return boolean
 */
function ch_collectd_rrd_start($dir, $verbose) {
  $started = FALSE;
  if (is_dir($dir) && ch_check_sudo()) {
    exec(sprintf('sudo rm -rf %s/*.bak', $dir));
    $d = dir($dir);
    while($entry = $d->read()) {
      if (!preg_match('/[a-zA-Z0-9]+/', $entry)) continue;
      $rdir = sprintf('%s/%s', $dir, $entry);
      if (is_dir($rdir) && !preg_match('/\.bak$/', $entry)) {
        $rdirBak = $rdir . '.bak';
        exec(sprintf('sudo mv %s %s', $rdir, $rdirBak));
        if (is_dir($rdirBak)) {
          $started = TRUE;
          print_msg(sprintf('Successfully renamed existing collectd rrd directory from %s to %s', $rdir, $rdirBak), $verbose, __FILE__, __LINE__);
        }
        else {
          $started = FALSE;
          print_msg(sprintf('Unable to rename existing collectd rrd directory from %s to %s', $rdir, $rdirBak), $verbose, __FILE__, __LINE__, TRUE);
        }
      }
    }
    $d->close();
  }
  return $started;
}

/**
 * attempts to stop collectd rrd stats for the current test iteration. returns
 * TRUE on success, FALSE otherwise
 * @param string $dir the base directory where collectd rrd files are stored
 * @param string $saveTo the directory where the rrd zip file should be created
 * @param array $options test options - used to construct collectd rrd results 
 * directories
 * @param boolean $verbose enable verbose output
 * @return boolean
 */
function ch_collectd_rrd_stop($dir, $saveTo, $verbose) {
  $stopped = FALSE;
  if (is_dir($dir) && ch_check_sudo()) {
    $tdir = '/tmp/' . rand();
    exec(sprintf('sudo mkdir %s', $tdir));
    print_msg(sprintf('Attempting to save collectd rrd files in %s to %s/collectd-rrd.zip using tmp directory %s', $dir, $saveTo, $tdir), $verbose, __FILE__, __LINE__);
    
    $rename = array();
    $d = dir($dir);
    while($entry = $d->read()) {
      if (!preg_match('/[a-zA-Z0-9]+/', $entry)) continue;
      $rdir = sprintf('%s/%s', $dir, $entry);
      if (substr($rdir, -1) == '/') $rdir = substr($rdir, 0, -1);
      if (!is_dir($rdir)) continue;
      if (is_dir($rdir)) {
        print_msg(sprintf('Evaluating collectd rrd directory %s', $rdir), $verbose, __FILE__, __LINE__);
        if (preg_match('/\.bak$/', $entry)) $rename[] = $rdir;
        else {
          exec($cmd = sprintf('sudo mv %s %s', $rdir, $tdir));
          print_msg(sprintf('Successfully moved test collectd rrd directory %s to %s', basename($rdir), $tdir), $verbose, __FILE__, __LINE__);
        }
      }
    }
    $d->close();
    foreach($rename as $rdir) {
      $renameTo = substr($rdir, 0, -4);
      exec($cmd = sprintf('sudo rm -rf %s;sudo mv %s %s', $renameTo, $rdir, $renameTo));
      if (is_dir($renameTo)) print_msg(sprintf('Successfully renamed backup collectd rrd directory from %s to %s', $rdir, $renameTo), $verbose, __FILE__, __LINE__);
      else print_msg(sprintf('Unable to rename existing collectd rrd directory from %s to %s', $rdir, $renameTo), $verbose, __FILE__, __LINE__, TRUE);
    }
    
    if ((shell_exec(sprintf('find %s -maxdepth 1 -type d 2>/dev/null | wc -l', $tdir))*1 < 2)) print_msg(sprintf('collectd rrd directory %s did not contain any files', $dir), $verbose, __FILE__, __LINE__, TRUE);
    else {
      $zip = sprintf('%s/collectd-rrd.zip', $saveTo);
      exec($cmd = sprintf('cd %s;sudo zip -r collectd-rrd.zip *;sudo mv collectd-rrd.zip %s', $tdir, $saveTo, $tdir));
      if ($stopped = file_exists($zip) && filesize($zip)) print_msg(sprintf('Successfully saved collectd rrd files to %s', $zip), $verbose, __FILE__, __LINE__);
      else print_msg(sprintf('Unable to save collectd rrd files: %s', $cmd), $verbose, __FILE__, __LINE__, TRUE);
    }
  }
  else print_msg(sprintf('collectd rrd directory %s does not exist or user does not have sudo access', $dir), $verbose, __FILE__, __LINE__, TRUE);
  
  // sleep for 10 seconds to allow collectd to restart
  sleep(10);
  
  return $stopped;
}

/**
 * Invokes an http request and returns the status code (or response body if 
 * $retBody is TRUE) on success, NULL on failure or FALSE if the response code 
 * is not within the $success range
 * @param string  $url the target url
 * @param string $method the http method
 * @param array $headers optional request headers to include (hash or array)
 * @param string $file optional file to pipe into the curl process as the 
 * body
 * @param string $auth optional [user]:[pswd] to use for http authentication
 * @param string $success the http response code/range that consistitutes 
 * a successful request. defaults to 200 to 299. This parameter may be a comma
 * separated list of values or ranges (e.g. "200,404" or "200-299,404")
 * @param boolean $retBody whether or not to return the response body. If 
 * FALSE (default), the status code is returned
 * @return mixed
 */
function ch_curl($url, $method='HEAD', $headers=NULL, $file=NULL, $auth=NULL, $success='200-299', $retBody=FALSE) {
  global $ch_curl_options;
  if (!isset($ch_curl_options)) $ch_curl_options = parse_args(array('v' => 'verbose'));
  
  if (!is_array($headers)) $headers = array();
  $ofile = $retBody ? '/tmp/' . rand() : '/dev/null';
  $curl = sprintf('curl -s -X %s%s -w "%s\n" -o %s', $method, $method == 'HEAD' ? ' -I' : '', '%{http_code}', $ofile);
  if ($auth) $curl .= sprintf(' -u "%s"', $auth);
  if (is_array($headers)) {
    foreach($headers as $header => $val) $curl .= sprintf(' -H "%s%s"', is_numeric($header) ? '' : $header . ':', $val); 
  }
  // input file
  if (($method == 'POST' || $method == 'PUT') && file_exists($file)) {
    $curl .= sprintf(' --data-binary @%s', $file);
    if (!isset($headers['Content-Length']) && !isset($headers['content-length'])) $curl .= sprintf(' -H "Content-Length:%d"', filesize($file));
    if (!isset($headers['Content-Type']) && !isset($headers['content-type'])) $curl .= sprintf(' -H "Content-Type:%s"', get_mime_type($file));
  }
  $curl .= sprintf(' "%s"', $url);
  $ok = array();
  foreach(explode(',', $success) as $range) {
    if (is_numeric($range)) $ok[$range*1] = TRUE;
    else if (preg_match('/^([0-9]+)\s*\-\s*([0-9]+)$/', $range, $m) && $m[1] <= $m[2]) {
      for($i=$m[1]; $i<=$m[2]; $i++) $ok[$i] = TRUE;
    }
  }
  $ok = array_keys($ok);
  sort($ok);
  
  $cmd = sprintf('%s 2>/dev/null;echo $?', $curl);
  print_msg(sprintf('Invoking curl request: %s (expecting response %s)', $curl, $success), isset($ch_curl_options['verbose']), __FILE__, __LINE__);
  
  // execute curl
  $result = shell_exec($cmd);
  $output = explode("\n", trim($result));
{'fraction_non_alphanumeric': 0.17805781005268403, 'fraction_numerical': 0.008258578954862594, 'mean_word_length': 3.676098535286285, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2153885', 'n_tokens_mistral': 4858, 'n_tokens_neox': 4779, 'n_words': 1469}