<commit_before><?php

mb_internal_encoding('UTF-8');

// Load RSS feed
$feed_uri = 'http://habrahabr.ru/rss/hubs/new/';
$rss = simplexml_load_file($feed_uri);
if (!$rss) die('Cannot read rss');

// Initialise TwisterPost
require_once 'twisterpost.php';
$twister = new TwisterPost('habr_ru');

// Set path to twisterd directory
$twister->twisterPath = '.' . DIRECTORY_SEPARATOR . 'twister-win32-bundle' . DIRECTORY_SEPARATOR;

// Initialise RSS database
require_once 'habrrssdb.php';
$db = new HabrRSSDb('habr_db.dat');

foreach ($rss->channel->item as $item) {
    $link  = (string)$item->link;
    $title = (string)$item->title;

    // get post id from link
    $id = (int)preg_replace('#[^\d]#', '', $link);

    if ($db->isPublished($id)) {
        continue;
    }

    $msg = $twister->prettyPrint($title, $link, isset($item->category) ? $item->category : null);

    if ($twister->postMessage($msg)) {
        $db->setPublished($id);
    }
}<commit_msg>short URL [-6 chars do matter]<commit_after><?php

mb_internal_encoding('UTF-8');

// Load RSS feed
$feed_uri = 'http://habrahabr.ru/rss/hubs/new/';
$rss = simplexml_load_file($feed_uri);
if (!$rss) die('Cannot read rss');

// Initialise TwisterPost
require_once 'twisterpost.php';
$twister = new TwisterPost('habr_ru');

// Set path to twisterd directory
$twister->twisterPath = '.' . DIRECTORY_SEPARATOR . 'twister-win32-bundle' . DIRECTORY_SEPARATOR;

// Initialise RSS database
require_once 'habrrssdb.php';
$db = new HabrRSSDb('habr_db.dat');

foreach ($rss->channel->item as $item) {
    $link  = (string)$item->link;
    $title = (string)$item->title;

    // get post id from link
    $id = (int)preg_replace('#[^\d]#', '', $link);

    if ($db->isPublished($id)) {
        continue;
    }

    // short URL [-6 chars do matter]
    $link = str_replace('habrahabr.ru', 'habr.ru', $link);
    $link = rtrim($link, '/');
    $msg = $twister->prettyPrint($title, $link, isset($item->category) ? $item->category : null);

    if ($twister->postMessage($msg)) {
        $db->setPublished($id);
    }
}
{'fraction_non_alphanumeric': 0.19292292777508482, 'fraction_numerical': 0.003877847794474067, 'mean_word_length': 3.8111888111888113, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '458766', 'n_tokens_mistral': 767, 'n_tokens_neox': 745, 'n_words': 149}