<commit_before><?php
/**
 * title: Article feeds
 * description: Queries a few online resources for article links
 * version: 0.5
 * category: template
 * api: cli
 * type: cron
 * x-cron: 12 *\/4 * * * 
 *
 * Highlights version numbers in news feeds,
 * and populates templates/feed.*.htm for sidebar display.
 *
 * Some of the collected entries (*games) are displayed togerther in sidebar blocks.
 *
 */


// switch to webroot
chdir(dirname(__DIR__));


#-- RSS
$feeds = array(
    "linuxcom,7" => "http://www.linux.com/news/software?format=feed&type=rss",
    "reddit,17" => "http://www.reddit.com/r/linux/.rss",
    "linuxgames,5" => "http://www.linuxgames.com/feed",
    "gamingonlinux,4" => "http://www.gamingonlinux.com/article_rss.php",
    "freegamer,3" => "http://freegamer.blogspot.com/feeds/posts/default?alt=rss",
    "sourceforge,22" => "http://sourceforge.net/directory/release_feed/",
    "distrowatch,15" => "http://distrowatch.com/news/dwd.xml",
    "beopen,7" => "http://beopen.bplaced.net/category/projects/feed/",
);
$filter = 
    "/Please 'report' off-topic|namelessrom|machomebrew/"
;

#-- Traverse and collect entries
foreach ($feeds as $name=>$url) {

    // data
    list($name, $max) = str_getcsv($name);
    $output = "";
    $x = file_get_contents($url);
    $x = preg_replace("/[^\x20-\x7F\s]/", "", $x);
    $x = simplexml_load_string($x);
    
    // append
    $i = 0;
    foreach ($x->channel->item as $item) {
    
        # pre-filter
        list($title, $link) = array( htmlspecialchars($item->title),  htmlspecialchars($item->link) );
        if (empty($title) or empty($link) or preg_match($filter, $title) or preg_match($filter, $link)) {
            continue;
        }

        # per feed
        switch ($name) {

            // Extract project base names and version numbers
            case "sourceforge":
                if (preg_match("~^(http://sourceforge.net/projects/(\w+))/files/.+?(\d+(\.\d+)+).+?/download$~", $item->link, $m)) {
                    $output .= "<a href=\"$m[1]\">$m[2] <em>$m[3]</em></a>\n";
                    $i++;
                }
                break;

            // Extract project base names and version numbers
            case "distrowatch":
                if (preg_match("~^(\d+/\d+)\s(\D+)\s+(.+)$~", $title, $m)) {
                    $output .= "<a href=\"$link\"><small>$m[1]</small> $m[2] <em>$m[3]</em></a>\n";
                }
                break;

            // Titles as is
            default:
            case "reddit":
            case "linuxcom":
            case "linuxgames":
                if (strlen($item->link) and strlen($item->title)) {
                    $title = preg_replace("~(\d+\.[\d-.]+)~", "<em>$0</em>", $title);
                    $output .="<a href=\"$link\">$title</a>\n";
                    $i++;
                }
                break;
        }

        if ($i >= $max) { break; }
    }
    
    // save
    strlen($output) and
    file_put_contents("./template/feed.$name.htm", $output);

}


<commit_msg>Decode sourceforge URLs to skip %20 for version extraction, also try to capture -beta and -rc suffixes.<commit_after><?php
/**
 * title: Article feeds
 * description: Queries a few online resources for article links
 * version: 0.5
 * category: template
 * api: cli
 * type: cron
 * x-cron: 12 *\/4 * * * 
 *
 * Highlights version numbers in news feeds,
 * and populates templates/feed.*.htm for sidebar display.
 *
 * Some of the collected entries (*games) are displayed togerther in sidebar blocks.
 *
 */


// switch to webroot
chdir(dirname(__DIR__));


#-- RSS
$feeds = array(
    "linuxcom,7" => "http://www.linux.com/news/software?format=feed&type=rss",
    "reddit,17" => "http://www.reddit.com/r/linux/.rss",
    "linuxgames,5" => "http://www.linuxgames.com/feed",
    "gamingonlinux,4" => "http://www.gamingonlinux.com/article_rss.php",
    "freegamer,3" => "http://freegamer.blogspot.com/feeds/posts/default?alt=rss",
    "sourceforge,22" => "http://sourceforge.net/directory/release_feed/",
    "distrowatch,15" => "http://distrowatch.com/news/dwd.xml",
    "beopen,7" => "http://beopen.bplaced.net/category/projects/feed/",
);
$filter = 
    "/Please 'report' off-topic|namelessrom|machomebrew/"
;

#-- Traverse and collect entries
foreach ($feeds as $name=>$url) {

    // data
    list($name, $max) = str_getcsv($name);
    $output = "";
    $x = file_get_contents($url);
    $x = preg_replace("/[^\x20-\x7F\s]/", "", $x);
    $x = simplexml_load_string($x);
    
    // append
    $i = 0;
    foreach ($x->channel->item as $item) {
    
        # pre-filter
        list($title, $link) = array( htmlspecialchars($item->title),  htmlspecialchars($item->link) );
        if (empty($title) or empty($link) or preg_match($filter, $title) or preg_match($filter, $link)) {
            continue;
        }

        # per feed
        switch ($name) {

            // Extract project base names and version numbers
            case "sourceforge":
                if (preg_match("~^(http://sourceforge.net/projects/(\w+))/files/.+?(\d+(\.\d+)+([-_. ](rc|beta|alpha|dev)([-._]?\d[.\d]*)?)?).+?/download$~", urldecode($item->link), $m)) {
                    $output .= "<a href=\"$m[1]\">$m[2] <em>$m[3]</em></a>\n";
                    $i++;
                }
                break;

            // Extract project base names and version numbers
            case "distrowatch":
                if (preg_match("~^(\d+/\d+)\s(\D+)\s+(.+)$~", $title, $m)) {
                    $output .= "<a href=\"$link\"><small>$m[1]</small> $m[2] <em>$m[3]</em></a>\n";
                }
                break;

            // Titles as is
            default:
            case "reddit":
            case "linuxcom":
            case "linuxgames":
                if (strlen($item->link) and strlen($item->title)) {
                    $title = preg_replace("~(\d+\.[\d-.]+)~", "<em>$0</em>", $title);
                    $output .="<a href=\"$link\">$title</a>\n";
                    $i++;
                }
                break;
        }

        if ($i >= $max) { break; }
    }
    
    // save
    strlen($output) and
    file_put_contents("./template/feed.$name.htm", $output);

}



{'fraction_non_alphanumeric': 0.20073931211828994, 'fraction_numerical': 0.00900032144005143, 'mean_word_length': 2.0806930693069305, 'pattern_counts': {'":': 10, '<': 33, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100253', 'n_tokens_mistral': 2165, 'n_tokens_neox': 2048, 'n_words': 403}