<commit_before>#!/usr/bin/perl

use strict;
use warnings;
use LWP;

my $url = 'http://www.linuxsecurity.com/static-content/linuxsecurity_advisories.rss';

my $browser = LWP::UserAgent->new;

my $response = $browser->get( $url ) 
	or die("Unable to get URL '$url'");
die ("Unable to get URL '$url'.") if ($response->is_error());

my ($link, $title);
my $printOut = '';
my $newsCount = 0;
my $firstTitle = 1;
for my $rssLine ( split(/\n/, $response->content) ) {
	chomp($rssLine);
	$rssLine =~ s/^\s+//;
	$rssLine =~ s/\s+$//;
	if ( $rssLine =~ /^<title>/ ) {
		$rssLine =~ s/<(\/)?title>//g;
		if ($firstTitle == 1) {
			$firstTitle = 0;
			next;
		}
		$title = $rssLine;
	}
	if ( $title && $rssLine =~ /^<link>/) {
		$rssLine =~ s/<(\/)?link>//g;
		$rssLine =~ s/&/&#38;/g;
		$printOut.= sprintf("\t\t\t\t\t<li><a href=\"%s\">%s</a></li>\n", 
			$rssLine, $title);
		$title = undef;
		$newsCount++;
	}
} # for my $rssLine

exit 2 if ($newsCount == 0);

# TODO ~ does directory exist?

open(FILE, '>/mnt/tmp/search.slack/news/linuxsec-news.htm');
print FILE "\t\t\t<div class=\"remoteNews-right\">\n";
print FILE "\t\t\t\t<ul>\n";
print FILE $printOut;
print FILE "\t\t\t\t</ul>\n";
print FILE "\t\t\t</div>\n";
close(FILE);<commit_msg>FIX: linuxsec-get-news ~ check if dir exists<commit_after>#!/usr/bin/perl

use strict;
use warnings;
use LWP;

my $url = 'http://www.linuxsecurity.com/static-content/linuxsecurity_advisories.rss';
my $outDir = '/mnt/tmp/search.slack/news/';

my $browser = LWP::UserAgent->new;

my $response = $browser->get( $url ) 
	or die("Unable to get URL '$url'");
die ("Unable to get URL '$url'.") if ($response->is_error());

my ($link, $title);
my $printOut = '';
my $newsCount = 0;
my $firstTitle = 1;
for my $rssLine ( split(/\n/, $response->content) ) {
	chomp($rssLine);
	$rssLine =~ s/^\s+//;
	$rssLine =~ s/\s+$//;
	if ( $rssLine =~ /^<title>/ ) {
		$rssLine =~ s/<(\/)?title>//g;
		if ($firstTitle == 1) {
			$firstTitle = 0;
			next;
		}
		$title = $rssLine;
	}
	if ( $title && $rssLine =~ /^<link>/) {
		$rssLine =~ s/<(\/)?link>//g;
		$rssLine =~ s/&/&#38;/g;
		$printOut.= sprintf("\t\t\t\t\t<li><a href=\"%s\">%s</a></li>\n", 
			$rssLine, $title);
		$title = undef;
		$newsCount++;
	}
} # for my $rssLine

exit 2 if ($newsCount == 0);

unless ( -e $outDir ) {
	exit 254;
}

my $outFile = sprintf(">%s/linuxsec-news.htm", $outDir);

open(FILE, $outFile) or die("Unable write to file.");
print FILE "\t\t\t<div class=\"remoteNews-right\">\n";
print FILE "\t\t\t\t<ul>\n";
print FILE $printOut;
print FILE "\t\t\t\t</ul>\n";
print FILE "\t\t\t</div>\n";
close(FILE);
{'fraction_non_alphanumeric': 0.25595695618754805, 'fraction_numerical': 0.007302075326671791, 'mean_word_length': 3.4193548387096775, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1426197', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1109, 'n_words': 241}