<commit_before><?php
\Tsugi\Core\LTIX::getConnection();

use \Tsugi\Grades\GradeUtil;

// A library for webscraping graders
require_once "vendor/autoload.php";

use \Tsugi\UI\SettingsForm;
use \Tsugi\Core\LTIX;
use \Goutte\Client;

$ngrok_fails = array(
    "ERR_NGROK_",
    "Before you can serve HTML content, you must sign up for an ngrok account and install your authtoken",
    "ngrok.com/signup",
    "Sign up for an ngrok account",
    "ngrok account and install your authtoken",
);

// Get any due date information
$dueDate = SettingsForm::getDueDate();
$penalty = $dueDate->penalty;

if ( $dueDate->message ) {
    echo('<p style="color:red;">'.$dueDate->message.'</p>'."\n");
}

function webauto_setup() {
    global $client;
    // http://symfony.com/doc/current/components/dom_crawler.html
    $client = new Client();
    $client->setMaxRedirects(5);
    $client->getClient()->setSslVerification(false);
}


/* Returns a crawler */
function webauto_load_url($url, $message=false) {
    global $client;
    global $ngrok_fails;
    global $base_url_path;
    global $webauto_http_status;
    global $webauto_http_content;
    line_out(" ");
    if ( $message ) echo("<b>".htmlentities($message)."</b><br/>\n");
    echo("<hr/><b>Loading URL:</b> ".htmlentities($url));
    $the_url = str_replace('"',"&quot;", $url);
    if ( strpos($the_url, '/') === 0 ) $the_url = $base_url_path . $the_url;
    echo(' (<a href="'.$the_url.'" target="_blank">Open URL</a>)');
    echo("<br/>\n");
    flush();
    try {
        $crawler = $client->request('GET', $url);
        $response = $client->getResponse();
        $webauto_http_status = $response->getStatus();
        $webauto_http_content = $response->getContent();

        // Check for ngrok failures
        $ngrok_fail = false;
        foreach($ngrok_fails as $fail) {
            if ( stripos($webauto_http_content, $fail) !== false ) $ngrok_fail = true;
        }
        if ( $ngrok_fail ) {
            error_out("It appears that your ngrok tunnel is not working properly: ".$webauto_http_content);
        }
    } catch(\Exception $e) {
        error_out($e->getMessage());
        return false;
    }
    return $crawler;
}

<commit_msg>Recover from "branch too early" :)<commit_after><?php
\Tsugi\Core\LTIX::getConnection();

use \Tsugi\Grades\GradeUtil;

// A library for webscraping graders
require_once "vendor/autoload.php";

use \Tsugi\UI\SettingsForm;
use \Tsugi\Core\LTIX;
use \Goutte\Client;
use Symfony\Component\HttpClient\HttpClient;

$ngrok_fails = array(
    "ERR_NGROK_",
    "Before you can serve HTML content, you must sign up for an ngrok account and install your authtoken",
    "ngrok.com/signup",
    "Sign up for an ngrok account",
    "ngrok account and install your authtoken",
);

// Get any due date information
$dueDate = SettingsForm::getDueDate();
$penalty = $dueDate->penalty;

if ( $dueDate->message ) {
    echo('<p style="color:red;">'.$dueDate->message.'</p>'."\n");
}

function webauto_setup() {
    global $client;
    // http://symfony.com/doc/current/components/dom_crawler.html
    // $client = new Client();
    // $client->getClient()->setSslVerification(false);

    // https://stackoverflow.com/questions/63539346/how-can-i-fix-the-ssl-problem-in-symfony-goutte
    $client = new Client(HttpClient::create(['verify_peer' => false, 'verify_host' => false]));
    $client->setMaxRedirects(5);
}


/* Returns a crawler */
function webauto_load_url($url, $message=false) {
    global $client;
    global $ngrok_fails;
    global $base_url_path;
    global $webauto_http_status;
    global $webauto_http_content;
    line_out(" ");
    if ( $message ) echo("<b>".htmlentities($message)."</b><br/>\n");
    echo("<hr/><b>Loading URL:</b> ".htmlentities($url));
    $the_url = str_replace('"',"&quot;", $url);
    if ( strpos($the_url, '/') === 0 ) $the_url = $base_url_path . $the_url;
    echo(' (<a href="'.$the_url.'" target="_blank">Open URL</a>)');
    echo("<br/>\n");
    flush();
    try {
        $crawler = $client->request('GET', $url);
        $response = $client->getResponse();
        $webauto_http_status = $response->getStatusCode();
        $webauto_http_content = $response->getContent();

        // Check for ngrok failures
        $ngrok_fail = false;
        foreach($ngrok_fails as $fail) {
            if ( stripos($webauto_http_content, $fail) !== false ) $ngrok_fail = true;
        }
        if ( $ngrok_fail ) {
            error_out("It appears that your ngrok tunnel is not working properly: ".$webauto_http_content);
        }
    } catch(\Exception $e) {
        error_out($e->getMessage());
        return false;
    }
    return $crawler;
}


{'fraction_non_alphanumeric': 0.17004913480025635, 'fraction_numerical': 0.002563554795983764, 'mean_word_length': 3.295412844036697, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 49, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4485328', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1619, 'n_words': 330}