<commit_before><?php

//Feed-Validator
//http://validator.w3.org/feed/

/* TODO: BITOnline Pipe (low prio)
 * TODO Twitter
 * TODO Facebook
 * TODO Rechenzentrum
 http://www.b-i-t-online.de/bitrss.xml
 mit "b.i.t.online - Ausgabe" anfangen herausfiltern
 auf Webseite verlinken
 */

//default values
$maxResult = 15;
$titleExcludeWords = array();
$feedUrl = 'http://www.ub.uni-dortmund.de/listen/inetbib/date1.html';
$feedTitle = 'Inetbib Neueste (relevante) Einträge';
$feedDescription = '';
$parseStrategy = 'mhonarc_list_date';

//overwrite these when parameters are specified
if (array_key_exists('max', $_GET) && is_numeric($_GET['max'])) {
    $maxResult = $_GET['max'];
}
if (array_key_exists('url', $_GET)) {
    $feedUrl = $_GET['url'];
}
if (array_key_exists('noanswers', $_GET)) {
    $titleExcludeWords[] = 'Re: ';
}
if (array_key_exists('nojobs', $_GET)) {
    $titleExcludeWords[] = 'Stellenanzeige';
    $titleExcludeWords[] = 'Stellenangebot';
    $titleExcludeWords[] = 'Stellenausschreibung';
}

//create message from these filters
$debugRequestParams = [];
foreach($_GET as $key => $value) {
    $debugRequestParams[] = ($value !== '') 
        ? $key . '=' . $value
        : $debugRequestParams[] = $key;
}
$debugRequestParamsString = implode(', ', $debugRequestParams);

$html= file_get_contents($feedUrl);
$dom = new DOMDocument();
@$dom->loadHTML( $html );

$xpath = new DOMXPath($dom);

function ensureAbsoluteUrl($url, $feedUrl) {
    if (substr($url, 0, 4) == 'http') {
        return $url;
    } else {
        $feedUrlInfo = parse_url($feedUrl);
        return implode('', array(
            $feedUrlInfo['scheme'], '://',
            $feedUrlInfo['host'], '/',
            dirname($feedUrlInfo['path']), '/',
            $url));
    }
}

$itemList = array();
if ($parseStrategy === 'mhonarc_list_date') {
    $elements = $xpath->query("//ul/ul/li");// war //ul//ul//li

    foreach ($elements as $e) {
        $item = array();
        $linkToMsg = $xpath->query(".//a", $e)->item(0);
        $item['title'] = $linkToMsg->nodeValue;
        // author is right of title
        $item['author'] = ltrim(explode($item['title'], $e->textContent)[1], ", ");
        $item['url'] = ensureAbsoluteUrl($linkToMsg->getAttribute('href'), $feedUrl);
        $item['date'] = date(DATE_RFC822, strtotime($e->parentNode->previousSibling->previousSibling->nodeValue));
        $item['description'] = '';
        $itemList[] = $item;
    }
}

header ("content-type: text/xml");
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
echo "<rss version=\"2.0\">\n";
echo "<channel>\n";
echo "<title>$feedTitle ($debugRequestParamsString)</title>\n";
echo "<description>$feedDescription</description>\n";
echo "<link>$feedUrl</link>\n";

$count = 0;
foreach ($itemList as $item) {
    // filter all items with any of the $titleExcludeWords in the title
    foreach ($titleExcludeWords as $titleFilter) {
        if (strpos($item['title'], $titleFilter) !== false) {
            continue 2;
        }
    }
    if ($count++ >= $maxResult) {
        break;
    }
    echo "<item>\n";
    echo "  <title>{$item['title']}</title>\n";
    echo "  <link>{$item['url']}</link>\n";
    echo "  <guid isPermaLink='false'>{$item['url']}</guid>\n";
    echo "  <author><EMAIL> ({$item['author']})</author>\n";
    echo "  <pubDate>{$item['date']}</pubDate>\n";
    echo "  <description>{$item['description']}</description>\n";
    echo "</item>\n";
}

echo "</channel>\n";
echo "</rss>\n";

/*
nested xpath

$xpath = new DOMXPath($doc);
$res = $xpath->query('//div');
$sub = $xpath->query('.//p', $res->item(1));
 */

?><commit_msg>Include Twitter Example<commit_after><?php

//Feed-Validator
//http://validator.w3.org/feed/

/*
 * TODO Facebook
 * TODO Rechenzentrum
 * TODO Rektoratsnachrichten
 * TODO: BITOnline Pipe (low prio)
 http://www.b-i-t-online.de/bitrss.xml
 mit "b.i.t.online - Ausgabe" anfangen herausfiltern
 auf Webseite verlinken
 
 */

 /*
Examples calls:

index.php?url=http://www.ub.uni-dortmund.de/listen/inetbib/date1.html&noanswers=true&nojobs

index.php?url=http://www.handle.net/mail-archive/handle-info/

index.php?url=https://twitter.com/UBMannheim&nofb&noretweet
index.php?url=https://twitter.com/hashtag/zotero
 */
 
 
//default values
$maxResult = 15;
$titleExcludeWords = array();
$exclusionTest = array();
$feedUrl = 'http://www.ub.uni-dortmund.de/listen/inetbib/date1.html';
$feedTitle = 'Inetbib Neueste (relevante) Einträge';
$feedDescription = '';
$parseStrategy = 'mhonarc_list_date';

//overwrite these when parameters are specified
if (array_key_exists('max', $_GET) && is_numeric($_GET['max'])) {
    $maxResult = $_GET['max'];
}
if (array_key_exists('url', $_GET)) {
    $feedUrl = $_GET['url'];
}
if (array_key_exists('parseStrategy', $_GET)) {
    $parseStrategy = $_GET['parseStrategy'];
} else {
	if (strpos($feedUrl, 'twitter') !== false ) {
		$parseStrategy = 'twitter';
	}
}
if (array_key_exists('noanswers', $_GET)) {
    $titleExcludeWords[] = 'Re: ';
}
if (array_key_exists('nofb', $_GET)) {// e.g. exclude automatic posting in twitter from facebook
    $titleExcludeWords[] = 'http://fb.me';
}
if (array_key_exists('noretweet', $_GET)) {
    $exclusionTest[] = 'retweet';
}
if (array_key_exists('nojobs', $_GET)) {
    $titleExcludeWords[] = 'Stellenanzeige';
    $titleExcludeWords[] = 'Stellenangebot';
    $titleExcludeWords[] = 'Stellenausschreibung';
}

//create message from these filters
$debugRequestParams = [];
foreach($_GET as $key => $value) {
    $debugRequestParams[] = ($value !== '') 
        ? $key . '=' . $value
        : $debugRequestParams[] = $key;
}
$debugRequestParamsString = implode(', ', $debugRequestParams);

$html= file_get_contents($feedUrl);
$dom = new DOMDocument();
@$dom->loadHTML( $html );

$xpath = new DOMXPath($dom);

function ensureAbsoluteUrl($url, $feedUrl) {
    if (substr($url, 0, 4) == 'http') {
        return $url;
    } else {
        $feedUrlInfo = parse_url($feedUrl);
		$result = $feedUrlInfo['scheme'] .  '://' . $feedUrlInfo['host'];
		if ( strlen(dirname($feedUrlInfo['path'])) > 1 && // i.e. not just \
			substr($url, 0, 1) !== "/" ) {// 
			if (substr($feedUrlInfo['path'],-1) == "/") {
				$result .= $feedUrlInfo['path'];
			} else {
				$result .= dirname($feedUrlInfo['path']) . '/';
			}
		}
		return $result . $url;
    }
}

$itemList = array();
if ($parseStrategy === 'mhonarc_list_date') {
	$elements = $xpath->query("//ul/ul/li");// war //ul//ul//li

    foreach ($elements as $e) {
        $item = array();
        $linkToMsg = $xpath->query(".//a", $e)->item(0);
        $item['title'] = $linkToMsg->nodeValue;
        // author is right of title
        $item['author'] = ltrim(explode($item['title'], $e->textContent)[1], ", ");
        $item['url'] = ensureAbsoluteUrl($linkToMsg->getAttribute('href'), $feedUrl);
        $item['date'] = date(DATE_RFC822, strtotime($e->parentNode->previousSibling->previousSibling->nodeValue));
        $item['description'] = '';
        $itemList[] = $item;
    }
}
if ($parseStrategy === 'twitter') {
	$elements = $xpath->query("//*[contains(@class, 'original-tweet')]");
	foreach ($elements as $e) {
        $item = array('title' => '', 'url' => '', 'author' => '', 'date' => '', 'description' => '', 'test' => '');
        $linkToMsg = $xpath->query(".//a[contains(@class, 'tweet-timestamp')]", $e)->item(0);
		$item['url'] = ensureAbsoluteUrl($linkToMsg->getAttribute('href'), $feedUrl);
        $item['title'] = $xpath->query(".//p[contains(@class, 'TweetTextSize')]", $e)->item(0)->textContent;
		$item['author'] = $xpath->query(".//strong[contains(@class, 'fullname')]", $e)->item(0)->textContent;
		$item['date'] = $linkToMsg->getAttribute('data-original-title');//TODO why is this not work as expected?
		if ( $xpath->query(".//span[contains(@class, 'js-retweet-text')]", $e)->length > 0) {
			$item['test'] = $xpath->query(".//span[contains(@class, 'js-retweet-text')]", $e)->item(0)->textContent;
		}
		$itemList[] = $item;
	}
}

header ("content-type: text/xml");
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
echo "<rss version=\"2.0\">\n";
echo "<channel>\n";
echo "<title>$feedTitle ($debugRequestParamsString)</title>\n";
echo "<description>$feedDescription</description>\n";
echo "<link>$feedUrl</link>\n";

$count = 0;
foreach ($itemList as $item) {
    // filter all items with any of the $titleExcludeWords in the title
    foreach ($titleExcludeWords as $titleFilter) {
        if (strpos($item['title'], $titleFilter) !== false) {
            continue 2;
        }
    }
	// filter all items based on the test value
	foreach ($exclusionTest as $testFilter) {
        if (strpos($item['test'], $testFilter) !== false) {
            continue 2;
        }
    }
    if ($count++ >= $maxResult) {
        break;
    }
    echo "<item>\n";
    echo "  <title>{$item['title']}</title>\n";
    echo "  <link>{$item['url']}</link>\n";
    echo "  <guid isPermaLink='false'>{$item['url']}</guid>\n";
    echo "  <author><EMAIL> ({$item['author']})</author>\n";
    echo "  <pubDate>{$item['date']}</pubDate>\n";
    echo "  <description>{$item['description']}</description>\n";
    echo "</item>\n";
}

echo "</channel>\n";
echo "</rss>\n";

/*
nested xpath

$xpath = new DOMXPath($doc);
$res = $xpath->query('//div');
$sub = $xpath->query('.//p', $res->item(1));
 */

?>
{'fraction_non_alphanumeric': 0.22344635789924056, 'fraction_numerical': 0.005241202267622206, 'mean_word_length': 3.7365754812563323, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 2, '>': 113, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1934904', 'n_tokens_mistral': 3341, 'n_tokens_neox': 3259, 'n_words': 601}