<commit_before>#!/usr/bin/php
<?php
// This is not a cron job... Use it like a daemon
require_once('require/class.SBS.php');

// Check if schema is at latest version
require_once('require/class.Connection.php');
$schema = new Connection();
if ($schema::latest() === false) {
    echo "You MUST update to latest schema. Run install/index.php";
    exit();
}


$debug = true;

$SBS=new SBS();

date_default_timezone_set('UTC');
// signal handler - playing nice with sockets and dump1090
if (function_exists('pcntl_fork')) {
    pcntl_signal(SIGINT,  function($signo) {
        global $sock, $db;
        echo "\n\nctrl-c or kill signal received. Tidying up ... ";
        socket_shutdown($sock, 0);
        socket_close($sock);
        $db = null;
        die("Bye!\n");
    });
    pcntl_signal_dispatch();
}

// let's try and connect
echo "Connecting to dump1090 ... ";
// create our socket and set it to non-blocking
$sock = socket_create(AF_INET, SOCK_STREAM, SOL_TCP) or die("Unable to create socket\n");

while (!@socket_connect($sock, $globalSBS1Host, $globalSBS1Port))
{
      $err = socket_last_error($sock);
      if ($err == 115 || $err == 114)
      {
        if ((time() - $time) >= $globalSBS1TimeOut)
        {
          socket_close($sock);
          die("Connection timed out.\n");
        }
      }
      die(socket_strerror($err) . "\n");
}

// connected - lets do some work
echo "Connected!\n";
sleep(1);
echo "SCAN MODE \n\n";
while($buffer = socket_read($sock, 3000, PHP_NORMAL_READ)) {

    // lets play nice and handle signals such as ctrl-c/kill properly
    if (function_exists('pcntl_fork')) pcntl_signal_dispatch();
    $dataFound = false;

    $SBS::del();
    $buffer=trim(str_replace(array("\r\n","\r","\n","\\r","\\n","\\r\\n"),'',$buffer));
    // SBS format is CSV format
    if ($buffer != '') {
	$line = explode(',', $buffer);
        if (count($line) < 20) echo "!! Error in message : ".$buffer."\n";
	else $SBS::add($line);
    }
}
if (function_exists('pcntl_fork')) pcntl_exec($_,$argv);
?><commit_msg>Ignore AIR and ID messages for SBS<commit_after>#!/usr/bin/php
<?php
// This is not a cron job... Use it like a daemon
require_once('require/class.SBS.php');

// Check if schema is at latest version
require_once('require/class.Connection.php');
$schema = new Connection();
if ($schema::latest() === false) {
    echo "You MUST update to latest schema. Run install/index.php";
    exit();
}


$debug = true;

$SBS=new SBS();

date_default_timezone_set('UTC');
// signal handler - playing nice with sockets and dump1090
if (function_exists('pcntl_fork')) {
    pcntl_signal(SIGINT,  function($signo) {
        global $sock, $db;
        echo "\n\nctrl-c or kill signal received. Tidying up ... ";
        socket_shutdown($sock, 0);
        socket_close($sock);
        $db = null;
        die("Bye!\n");
    });
    pcntl_signal_dispatch();
}

// let's try and connect
echo "Connecting to dump1090 ... ";
// create our socket and set it to non-blocking
$sock = socket_create(AF_INET, SOCK_STREAM, SOL_TCP) or die("Unable to create socket\n");

while (!@socket_connect($sock, $globalSBS1Host, $globalSBS1Port))
{
      $err = socket_last_error($sock);
      if ($err == 115 || $err == 114)
      {
        if ((time() - $time) >= $globalSBS1TimeOut)
        {
          socket_close($sock);
          die("Connection timed out.\n");
        }
      }
      die(socket_strerror($err) . "\n");
}

// connected - lets do some work
echo "Connected!\n";
sleep(1);
echo "SCAN MODE \n\n";
while($buffer = socket_read($sock, 3000, PHP_NORMAL_READ)) {

    // lets play nice and handle signals such as ctrl-c/kill properly
    if (function_exists('pcntl_fork')) pcntl_signal_dispatch();
    $dataFound = false;

    $SBS::del();
    $buffer=trim(str_replace(array("\r\n","\r","\n","\\r","\\n","\\r\\n"),'',$buffer));
    // SBS format is CSV format
    if ($buffer != '') {
	$line = explode(',', $buffer);
        if (count($line) > 20) $SBS::add($line);
    }
}
if (function_exists('pcntl_fork')) pcntl_exec($_,$argv);
?>
{'fraction_non_alphanumeric': 0.17930353173623118, 'fraction_numerical': 0.012348728081007657, 'mean_word_length': 3.145342886386899, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7266005', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1374, 'n_words': 350}