<commit_before><?php
require('./twitteroauth/twitteroauth.php');

class TweetCache {
	protected $config;
	protected $cachefile;
	protected $stat;
	protected $lastmod;
	protected $logentries;
	protected $feed;

	function __construct($config, $username, $format) {
		$this->logentries = array();
		if (!preg_match('/^\w+$/', $username))
			throw new Exception("Username not accepted.");

		if (!in_array($format, array('atom', 'xml', 'rss', 'json')))
			throw new Exception("Feed format not accepted");

		$this->config = $config;
		$this->cachefile = $config['cache']['path']."/$username/$username.$format";
		$this->stat = @stat($this->cachefile);
		if ($this->stat)
			$this->lastmod = $this->stat['mtime'];
		else
			throw new Exception("$format cache for $username not found.");
		$this->username = $username;
		$this->format = $format;
	}

	function load() {
		$this->feed = '';
		$cache = fopen($this->cachefile, 'r+b');
		if (!$cache)
			throw new Exception("Could not open cache file");
		$age = time() - $this->lastmod;
		$this->log("Cache age: $age seconds");
		$stale = $age > $this->config['cache']['max_age'];
		if ($stale || !$this->stat['size']) {
			if (flock($cache, LOCK_EX|LOCK_NB)) {
				try {
					$this->feed = $this->loadFromTwitter();
					if ($this->format!='json') {
						$xml = new DomDocument();
						$xml->loadXML($this->feed);
						#$this->filterTweets($xml);
						$this->convertToLocalTime($xml);
						$this->feed = $xml->saveXML($xml->documentElement);
					}
					$this->lastmod = time();
					$this->log("Writing to cache");
					ftruncate($cache, 0);
					fwrite($cache, $this->feed, strlen($this->feed));
				} catch(Exception $ex) {
					$this->log($ex->getMessage());
				}
				flock($cache, LOCK_UN);
			} else $this->log("Unable to get exclusive lock");
		}
		if (!$this->feed) {
			$this->log("Reading from cache");
			flock($cache, LOCK_SH);
			/*	refresh stat after getting lock,
				just in case flock was waiting for another LOCK_EX to release,
				in which case the file was just updated	*/
			$this->stat = fstat($cache);
			$bytes = $this->stat['size'];
			if ($bytes)
				$this->feed = fread($cache, $bytes);
			flock($cache, LOCK_UN);
		}
		fclose($cache);
	}

	function modified() { return $this->lastmod; }
	
	function isStale() { return $this->age() > $this->config['cache']['max_age']; }

	function mimetype() {
		$mimetype = 'application/';
		if (in_array($this->format, array('atom', 'rss')))
			$mimetype .= $this->format . '+xml';
		else
			$mimetype .= $this->format;
		return $mimetype; 
	}
	
	function loadFromTwitter() {
		$this->log("Loading tweets from twitter");
		$start = time();
		$config = $this->config['twitter']; 
		$connection = new TwitterOAuth($config['CONSUMER_KEY'],
		                               $config['CONSUMER_SECRET'],
		                               $config['OAUTH_TOKEN'],
		                               $config['OAUTH_TOKEN_SECRET']);
		$connection->connecttimeout = $config['timeout'];
		$connection->format = $this->format;
		$params = array(
			'screen_name'=>$this->username,
			'trim_user'=>1,
			'include_entities'=>0,
			'include_rts'=>0
		);
		$paramfile = $this->config['cache']['path'].'/' . $this->username . '/params.ini';
		if (file_exists($paramfile))
			$params = array_merge($params, parse_ini_file($paramfile));
		if (in_array($this->format, array('atom', 'rss'))) {
			unset($params['trim_user']);<commit_msg>unset trim_user param, and don't cache empty response from twitter.<commit_after><?php
require('./twitteroauth/twitteroauth.php');

class TweetCache {
	protected $config;
	protected $cachefile;
	protected $stat;
	protected $lastmod;
	protected $logentries;
	protected $feed;

	function __construct($config, $username, $format) {
		$this->logentries = array();
		if (!preg_match('/^\w+$/', $username))
			throw new Exception("Username not accepted.");

		if (!in_array($format, array('atom', 'xml', 'rss', 'json')))
			throw new Exception("Feed format not accepted");

		$this->config = $config;
		$this->cachefile = $config['cache']['path']."/$username/$username.$format";
		$this->stat = @stat($this->cachefile);
		if ($this->stat)
			$this->lastmod = $this->stat['mtime'];
		else
			throw new Exception("$format cache for $username not found.");
		$this->username = $username;
		$this->format = $format;
	}

	function load() {
		$this->feed = '';
		$cache = fopen($this->cachefile, 'r+b');
		if (!$cache)
			throw new Exception("Could not open cache file");
		$age = time() - $this->lastmod;
		$this->log("Cache age: $age seconds");
		$stale = $age > $this->config['cache']['max_age'];
		if ($stale || !$this->stat['size']) {
			if (flock($cache, LOCK_EX|LOCK_NB)) {
				try {
					$this->feed = $this->loadFromTwitter();
					if ($this->format!='json') {
						$xml = new DomDocument();
						$xml->loadXML($this->feed);
						#$this->filterTweets($xml);
						$this->convertToLocalTime($xml);
						$this->feed = $xml->saveXML($xml->documentElement);
					}
					$len = strlen($this->feed);
					if ($len) {
						$this->lastmod = time();
						$this->log("Writing to cache");
						ftruncate($cache, 0);
						fwrite($cache, $this->feed, $len);
					} else
						$this->log("Received empty feed");
				} catch(Exception $ex) {
					$this->log($ex->getMessage());
				}
				flock($cache, LOCK_UN);
			} else $this->log("Unable to get exclusive lock");
		}
		if (!$this->feed) {
			$this->log("Reading from cache");
			flock($cache, LOCK_SH);
			/*	refresh stat after getting lock,
				just in case flock was waiting for another LOCK_EX to release,
				in which case the file was just updated	*/
			$this->stat = fstat($cache);
			$bytes = $this->stat['size'];
			if ($bytes)
				$this->feed = fread($cache, $bytes);
			flock($cache, LOCK_UN);
		}
		fclose($cache);
	}

	function modified() { return $this->lastmod; }
	
	function isStale() { return $this->age() > $this->config['cache']['max_age']; }

	function mimetype() {
		$mimetype = 'application/';
		if (in_array($this->format, array('atom', 'rss')))
			$mimetype .= $this->format . '+xml';
		else
			$mimetype .= $this->format;
		return $mimetype; 
	}
	
	function loadFromTwitter() {
		$this->log("Loading tweets from twitter");
		$start = time();
		$config = $this->config['twitter']; 
		$connection = new TwitterOAuth($config['CONSUMER_KEY'],
		                               $config['CONSUMER_SECRET'],
		                               $config['OAUTH_TOKEN'],
		                               $config['OAUTH_TOKEN_SECRET']);
		$connection->connecttimeout = $config['timeout'];
		$connection->format = $this->format;
		$params = array(
			'screen_name'=>$this->username,
		#	'trim_user'=>1,
			'include_entities'=>0,
			'include_rts'=>0
		);
		$paramfile = $this->config['cache']['path'].'/' . $this->username . '/params.ini';
		if (file_exists($paramfile))
			$params = array_merge($params, parse_ini_file($paramfile));
		if (in_array($this->format, array('atom', 'rss'))) {
			unset($params['trim_user']);
{'fraction_non_alphanumeric': 0.2009152009152009, 'fraction_numerical': 0.001144001144001144, 'mean_word_length': 3.5533854166666665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 122, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985676', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2310, 'n_words': 492}