<commit_before><?php
/*
Copyright 2010 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

require_once("./crawl_lib.inc");
require_once("./bootstrap.inc");

$pid_arr = array();
for ($i = 0; $i < 6; $i++) {
  $pid = pcntl_fork();
  if ($pid == -1) {
    die("cannot fork subprocesses ...");
  } else {
    if ($pid) {
      // Parent process
      $pid_arr[] = $pid;
    } elseif (0 == $i) {
      // Job submission process
      $unsubmitTests = ObtainTestsWithCode(0);
      if (!IsEmptyQuery($unsubmitTests)) {
        while ($row = mysql_fetch_assoc($unsubmitTests)) {
          //var_dump($row);
          SubmitTest($row);
        }
      }
      exit();
    } elseif (1 == $i) {
      $unsubmitted = ObtainTestsWithCode(0);
      $unfinished = ObtainTestsWithCode(1);
      if (IsEmptyQuery($unfinished) && IsEmptyQuery($unsubmitted)) {
        echo "Start loading URLS from file ...\r\n";
        LoadUrlFromFile();
      }
      exit();
    } elseif (2 == $i) {
      // Check the test status with WPT server
      CheckWPTStatus();
      exit();
    } elseif (3 == $i) {
      // Obtain XML result
      ObtainXMLResult();
      exit();
    } elseif (4 == $i) {
      // Download har file
      DownloadHar();
      exit();
    } elseif (5 == $i) {
      // Fill page table and request table
      FillTables();
      exit();
    }
  }
}

while (count($pid_arr) > 0) {
  $myId = pcntl_waitpid(-1, $status, WNOHANG);
  foreach ($pid_arr as $key => $pid) {
    if ($myId == $pid) {
      unset($pid_arr[$key]);
    }
  }
  usleep(100);
}
?>
<commit_msg>added the code to evalute $gReload value in order to determine if the tests need to be repeatly loaded.<commit_after><?php
/*
Copyright 2010 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

require_once("./crawl_lib.inc");
require_once("./bootstrap.inc");

// Create all the tables if they are not there.
createTables();
$pid_arr = array();
for ($i = 0; $i < 6; $i++) {
  $pid = pcntl_fork();
  if ($pid == -1) {
    die("cannot fork subprocesses ...");
  } else {
    if ($pid) {
      // Parent process
      $pid_arr[] = $pid;
    } elseif (0 == $i) {
      if (0 == $gReload) {
        // In this mode, the url file can only be loaded once.
        $query = "SELECT * FROM $gStatusTable";
        if (IsEmptyQuery(doQuery($query))) {
          // Initial loading
          LoadUrlFromFile();
        }
      } else {
        // In this mode, the url file can be repeatedly reloaded.
        $unsubmitted = ObtainTestsWithCode(0);
        $unfinished = ObtainTestsWithCode(1);
        if (IsEmptyQuery($unfinished) && IsEmptyQuery($unsubmitted)) {
          LoadUrlFromFile();
        }
      }
      exit();
    } elseif (1 == $i) {
      // Job submission process
      $unsubmitTests = ObtainTestsWithCode(0);
      if (!IsEmptyQuery($unsubmitTests)) {
        while ($row = mysql_fetch_assoc($unsubmitTests)) {
          //var_dump($row);
          SubmitTest($row);
        }
      }
      exit();
    } elseif (2 == $i) {
      // Check the test status with WPT server
      CheckWPTStatus();
      exit();
    } elseif (3 == $i) {
      // Obtain XML result
      ObtainXMLResult();
      exit();
    } elseif (4 == $i) {
      // Download har file
      DownloadHar();
      exit();
    } elseif (5 == $i) {
      // Fill page table and request table
      FillTables();
      exit();
    }
  }
}

while (count($pid_arr) > 0) {
  $myId = pcntl_waitpid(-1, $status, WNOHANG);
  foreach ($pid_arr as $key => $pid) {
    if ($myId == $pid) {
      unset($pid_arr[$key]);
    }
  }
  usleep(100);
}
?>

{'fraction_non_alphanumeric': 0.1388095761036679, 'fraction_numerical': 0.011201405666593455, 'mean_word_length': 2.4604863221884496, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305222', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1434, 'n_words': 453}