<commit_before><?
// The source code packaged with this file is Free Software, Copyright (C) 2005 by
// <NAME> <gallir at uib dot es>.
// It's licensed under the AFFERO GENERAL PUBLIC LICENSE unless stated otherwise.
// You can get copies of the licenses here:
//		http://www.affero.org/oagpl.html
// AFFERO GENERAL PUBLIC LICENSE is also included in the file called "COPYING".

require_once(mnminclude.'favorites.php');

class Link extends LCPBase {
	private static $clicked = 0; // Must add a click to this link->id
	public static $original_status = false;

	var $id = 0;
	var $author = -1;
	var $blog = 0;
	var $username = false;
	var $randkey = 0;
	var $karma = 0;
	var $valid = false;
	var $date = false;
	var $sent_date = 0;
	var $published_date = 0;
	var $modified = 0;
	var $url = false;
	var $url_title = '';
	var $encoding = false;
	var $status = 'discard';
	var $type = '';
	var $votes = 0;
	var $anonymous = 0;
	var $votes_avg = 0;
	var $negatives = 0;
	var $title = '';
	var $tags = '';
	var $uri = '';
	var $thumb_url = false;
	var $content = '';
	var $content_type = '';
	var $ip = '';
	var $html = false;
	var $read = false;
	var $voted = false;
	var $banned = false;
	var $thumb_status = 'unknown';
	var $clicks = 0;

	// sql fields to build an object from mysql
	const SQL = " link_id as id, link_author as author, link_blog as blog, link_status as status, sub_statuses.status as sub_status, UNIX_TIMESTAMP(sub_statuses.date) as sub_date, link_votes as votes, link_negatives as negatives, link_anonymous as anonymous, link_votes_avg as votes_avg, link_votes + link_anonymous as total_votes, link_comments as comments, link_karma as karma, sub_statuses.karma as sub_karma, link_randkey as randkey, link_url as url, link_uri as uri, link_url_title as url_title, link_title as title, link_tags as tags, link_content as content, UNIX_TIMESTAMP(link_date) as date,  UNIX_TIMESTAMP(link_sent_date) as sent_date, UNIX_TIMESTAMP(link_published_date) as published_date, UNIX_TIMESTAMP(link_modified) as modified, link_content_type as content_type, link_ip as ip, link_thumb_status as thumb_status, link_thumb_x as thumb_x, link_thumb_y as thumb_y, link_thumb as thumb, user_login as username, user_email as email, user_avatar as avatar, user_karma as user_karma, user_level as user_level, user_adcode, subs.name as sub_name, subs.id as sub_id, subs.server_name, subs.sub as is_sub, subs.owner as sub_owner, subs.base_url, subs.created_from, subs.allow_main_link, creation.status as sub_status_origen, subs.color1 as sub_color1, subs.color2 as sub_color2, favorite_link_id as favorite, clicks.counter as clicks, votes.vote_value as voted FROM links
	INNER JOIN users on (user_id = link_author)
	LEFT JOIN sub_statuses ON (@site_id > 0 and sub_statuses.id = @site_id and sub_statuses.link = links.link_id)
	LEFT JOIN (sub_statuses as creation, subs) ON (creation.link=links.link_id and creation.id=creation.origen and creation.id=subs.id)
	LEFT JOIN votes ON (link_date > @enabled_votes and vote_type='links' and vote_link_id = links.link_id and vote_user_id = @user_id and ( @user_id > 0  OR vote_ip_int = @ip_int ) )
	LEFT JOIN favorites ON (@user_id > 0 and favorite_user_id =  @user_id and favorite_type = 'link' and favorite_link_id = links.link_id)
	LEFT JOIN link_clicks as clicks on (clicks.id = links.link_id) ";

	const SQL_BASIC = " link_id as id, link_author as author, link_blog as blog, link_status as status, sub_statuses.status as sub_status, link_votes as votes, link_negatives as negatives, link_anonymous as anonymous, link_votes_avg as votes_avg, link_votes + link_anonymous as total_votes, link_comments as comments, link_karma as karma, sub_statuses.karma as sub_karma, link_randkey as randkey, link_url as url, link_uri as uri, link_url_title as url_title, link_title as title, link_tags as tags, link_content as content, UNIX_TIMESTAMP(link_date) as date,	UNIX_TIMESTAMP(link_sent_date) as sent_date, UNIX_TIMESTAMP(link_published_date) as published_date, UNIX_TIMESTAMP(link_modified) as modified, link_content_type as content_type, link_ip as ip, link_thumb_status as thumb_status, link_thumb_x as thumb_x, link_thumb_y as thumb_y, link_thumb as thumb, user_login as username, user_email as email, user_avatar as avatar, user_karma as user_karma, user_level as user_level, user_adcode, subs.name as sub_name, subs.id as sub_id, subs.server_name, subs.sub as is_sub, subs.owner as sub_owner, subs.base_url, subs.created_from, subs.allow_main_link, creation.status as sub_status_origen FROM links
	INNER JOIN users on (user_id = link_author)
	LEFT JOIN sub_statuses ON (@site_id > 0 and sub_statuses.id = @site_id and sub_statuses.link = links.link_id)
	LEFT JOIN (sub_statuses as creation, subs) ON (creation.link=links.link_id and creation.id=creation.origen and creation.id=subs.id)";


	static function from_db($id, $key = 'id', $complete = true) {
		global $db, $current_user;

		SitesMgr::my_id(); // Force to read current sub_id
		switch ($key) {
			case 'uri':
				$id = $db->escape($id);
				$selector = "link_uri = '$id'";
				break;
			default:
				$id = intval($id);
				$selector = "link_id = $id";
		}

		if ($complete) {
			$sql = "SELECT".Link::SQL."WHERE $selector";
		} else {
			$sql = "SELECT".Link::SQL_BASIC."WHERE $selector";
		}


		if(($object = $db->get_object($sql, 'Link'))) {
			$object->read = true;
			return $object;
		}
		return false;
	}

	static function count($status='', $force = false) {
		global $db, $globals;

		$my_id = SitesMgr::my_id();

		if (!$status) return Link::count('published', $force) +
							Link::count('queued', $force) +
							Link::count('discard', $force) +
							Link::count('abuse', $force) +
							Link::count('autodiscard', $force);


		$count = get_count("$my_id.$status");
		if ($count === false || $force) {
			$count = $db->get_var("select count(*) from sub_statuses where id = $my_id and status = '$status'");
			set_count("$my_id.$status", $count);
		}
		return $count;
	}

	static function duplicates($url, $site = false) {
		global $db;
		$trimmed = $db->escape(preg_replace('/\/$/', '', $url));
		$list = "'$trimmed', '$trimmed/'";
		if (preg_match('/^http.{0,1}:\/\/www\./', $trimmed)) {
			$link_alternative = preg_replace('/^(http.{0,1}):\/\/www\./', '$1://', $trimmed);
		} else {
			$link_alternative = preg_replace('/^(http.{0,1}):\/\//', '$1://www.', $trimmed);
		}
		$list .= ", '$link_alternative', '$link_alternative/'";

		/* Alternative to http and https */
		if (preg_match('/^http:/', $url)) {
			$list2 = preg_replace('/http:\/\//', 'https://', $list);
		} else {
			$list2 = preg_replace('/https:\/\//', 'http://', $list);
		}
		$list .= ", $list2";

/***** TODO: check and decid how duplicated must be dealt with
		$subs = array();
		$site_id = SitesMgr::my_id();
		$subs[] = $site_id;
		$subs = array_merge($subs, SitesMgr::get_senders(), SitesMgr::get_receivers());
		$subs = implode(',', $subs);

		// If it was abuse o autodiscarded allow other to send it again
		$found = $db->get_var("SELECT link_id FROM links, sub_statuses WHERE link_url in ($list) AND status not in ('abuse') AND link_votes > 0 AND sub_statuses.link = link_id AND sub_statuses.id in ($subs) ORDER by link_id asc limit 1");
*******/

		if ($site > 0) {
			$from_extra = ', sub_statuses';
			$where_extra = "and sub_statuses.id = $site and sub_statuses.link = link_id";
		} else {
			$from_extra = $where_extra = '';
		}
		// If it was abuse o autodiscarded allow other to send it again
		$found = $db->get_var("SELECT link_id FROM links $from_extra WHERE link_url in ($list) AND link_status not in ('abuse') AND link_votes > 0 $where_extra ORDER by link_id asc limit 1");
		return $found;
	}

	static function visited($id) {
		global $globals;

		if (! isset($_COOKIE['v']) || ! ($visited = preg_split('/x/', $_COOKIE['v'], 0, PREG_SPLIT_NO_EMPTY)) ) {
			$visited = array();
			$found = false;
		} else {
			$found = array_search($id, $visited);
			if (count($visited) > 10) {
				array_shift($visited);
			}
			if ($found !== false) {
				unset($visited[$found]);
			}
		}
		$visited[] = $id;
		setcookie('v', implode('x', $visited), 0, $globals['base_url_general'], UserAuth::domain());
		return $found !== false;
	}

	function add_click() {
		global $globals, $db;

		if (! $globals['bot']
			&& ! Link::visited($this->id)
			&& $globals['click_counter']
			&& isset($_COOKIE['k'])
			&& check_security_key($_COOKIE['k'])
			&& $this->ip != $globals['user_ip']) {
			// Delay storing
			self::$clicked = $this->id;
		}
	}

	static function store_clicks() {
		global $globals, $db;

		if (!self::$clicked) return false;
		$id = self::$clicked;
		self::$clicked = 0;

		if (! memcache_menabled()) {
			$db->query("UPDATE link_clicks SET counter=counter+1 WHERE id = $id");
			return true;
		}

		$key = 'clicks_cache';
		$cache = memcache_mget($key);
		if (!$cache) {
			$cache = array();
			$cache['time'] = $globals['start_time'];
			$cache[$id] = 1;
		} else {
			$cache[$id]++;
		}

		if ($globals['start_time'] - $cache['time'] > 4.0) {
			if( !memcache_mdelete($key)) {
				memcache_madd($key, array());
				syslog(LOG_INFO, "store_clicks: Delete failed");
			}
			ksort($cache); // To avoid transaction's deadlocks

			$show_errors = $db->show_errors;
			$db->show_errors = false; // we know there can be lock timeouts :(
			$tries = 0; // By the way, freaking locking timeouts with few updates per second with this technique
			while ($tries < 3) {
				$error = false;
				$db->transaction();
				$total= 0;
				$r = true;
				foreach ($cache as $id => $counter) {
					if ($id > 0 && $counter > 0) {
						$r = $db->query("INSERT INTO link_clicks (id, counter) VALUES ($id,$counter) ON DUPLICATE KEY UPDATE counter=counter+$counter");
						// $r = $db->query("UPDATE link_clicks SET counter=counter+$counter WHERE id = $id");
						if (!$r) {
							break;
						}
						$total += $counter;
					}
				}
				if ($r) {
					$db->commit();
					$tries = 100000; // Stop it
				} else {
					$tries++;
					syslog(LOG_INFO, "failed $tries attempts in store_clicks");
					$db->rollback();
				}
			}
			$db->show_errors = $show_errors;
		} else {
			memcache_madd($key, $cache);
		}
	}

	static function top() {
		global $globals;

		// It retrieves the annotiation generated by the Python script top-news.py
		$top = new Annotation('top-link-'.$globals['site_shortname']);
		if (!$top->read()) return false;
		$ids = explode(',',$top->text);
		if (! $ids) return false;

		$link = Link::from_db($ids[0]);
		if ($link) {
			$link->has_thumb();
		}
		return $link;
	}

	function json_votes_info($value=false) {
		$dict = array();
		$dict['id'] = $this->id;
		if ($value) {
			$dict['value'] = $value;
			if ($value < 0) {
				$dict['vote_description'] = get_negative_vote($value);
			} else {
				$dict['vote_description'] = _('¡chachi!');
			}
		}
		$dict['votes'] = $this->votes;
		$dict['anonymous'] = $this->anonymous;
		$dict['negatives'] = $this->negatives;
		if ($this->sub_status == 'published') {
			$dict['karma'] = intval($this->sub_karma);
		} else {
			$dict['karma'] = intval($this->karma);
		}
		return json_encode($dict);
	}

	function print_html() {
		echo "Valid: " . $this->valid . "<br>\n";
		echo "Url: " . $this->url . "<br>\n";
		echo "Title: " . $this->url_title . "<br>\n";
		echo "encoding: " . $this->encoding . "<br>\n";
	}

	function check_url($url, $check_ban = true, $first_level = false) {
		global $globals, $current_user;
		if(!preg_match('/^http[s]*:/', $url)) return false;
		$url_components = @parse_url($url);
		if (!$url_components) return false;
		if (!preg_match('/[a-z]+/', $url_components['host'])) return false;

		if (! $check_ban) return true;
		$quoted_domain = preg_quote(get_server_name());
		if(preg_match("/^$quoted_domain$/", $url_components['host'])) {
			$this->ban = array();
			$this->ban['comment'] = _('el servidor es local');
			syslog(LOG_NOTICE, "Meneame, server name is local name ($current_user->user_login): $url");
			return false;
		}
		require_once(mnminclude.'ban.php');
		if(($this->ban = check_ban($url, 'hostname', false, $first_level))) {
			syslog(LOG_NOTICE, "Meneame, server name is banned ($current_user->user_login): $url");
			$this->banned = true;
			return false;
		}
		return true;
	}

	function get($url, $maxlen = 150000, $check_ban = true) {
		global $globals, $current_user;
		$url=trim($url);
		$url_components = @parse_url($url);


		$this->noiframe = false;
		if (($response = get_url($url)) ) {
			$this->content_type = preg_replace('#^(\w+).+#', '$1', $response['content_type']);

			// Check if it has pingbacks
			if (preg_match('/X-Pingback: *(.+)/i', $response['header'], $match)) {
				$this->pingback = 'ping:'.clean_input_url($match[1]);
			}

			/* Were we redirected? */
			if ($response['redirect_count'] > 0) {
				/* update $url with where we were redirected to */
				$new_url = clean_input_url($response['location']);
			}
			if (!empty($new_url) && $new_url != $url) {
				syslog(LOG_NOTICE, "Meneame, redirected ($current_user->user_login): $url -> $new_url");
				/* Check again the url */
				if (!$this->check_url($new_url, $check_ban, true)) {
					$this->url = $new_url;
					return false;
				}
				// Change the url if we were directed to another host
				if (strlen($new_url) < 300	&& ($new_url_components = @parse_url($new_url))) {
					if ($url_components['host'] != $new_url_components['host']) {
						syslog(LOG_NOTICE, "Meneame, changed source URL ($current_user->user_login): $url -> $new_url");
						$url = $new_url;
						$url_components = $new_url_components;
					}
				}
			}
			$this->html = $response['content'];
			$url_ok = true;
		} else {
			syslog(LOG_NOTICE, "Meneame, error getting ($current_user->user_login): $url");
			$url_ok = false;
		}


		$this->url=$url;
		// Fill content type if empty
		// Right now only check for typical image extensions
		if (empty($this->content_type)) {
			if (preg_match('/(jpg|jpeg|gif|png)(\?|#|$)/i', $this->url)) {
				$this->content_type='image';
			}
		}
		// NO more to do
		if (!$url_ok || ! preg_match('/html/', $response['content_type'])) return true;

		// Check if it forbides including in an iframe
		if (preg_match('/X-Frame-Options: *(.+)/i', $response['header'])
			|| preg_match('/top\.location\.href *=/', $response['content'])) {
			$this->noiframe = true;
		}

		if(preg_match('/charset=([a-zA-Z0-9-_]+)/i', $this->html, $matches)) {
			$this->encoding=trim($matches[1]);
			if(strcasecmp($this->encoding, 'utf-8') != 0) {
				$this->html=iconv($this->encoding, 'UTF-8//IGNORE', $this->html);
			}
		}


		// Check if the author doesn't want to share
		if (preg_match('/<!-- *noshare *-->/', $this->html)) {
			$this->ban = array();
			$this->ban['comment'] = _('el autor no desea que se envíe el artículo, respeta sus deseos');
			syslog(LOG_NOTICE, "Meneame, noshare ($current_user->user_login): $url");
			return false;
		}

		// Now we analyse the html to find links to banned domains
		// It avoids the trick of using google or technorati
		// Ignore it if the link has a rel="nofollow" to ignore comments in blogs
		if (!preg_match('/content="[^"]*(vBulletin|phpBB)/i', $this->html)) {
			preg_match_all('/(< *meta +http-equiv|< *frame[^<]*>|window\.|document.\|parent\.|top\.|self\.)[^><]*(url|src|replace) *[=\(] *[\'"]{0,1}https*:\/\/[^\s "\'>]+[\'"\;\)]{0,1}[^>]*>/i', $this->html, $matches);
		} else {
			preg_match_all('/(<* meta +http-equiv|<* iframe|<* frame[^<]*>|window\.|document.\|parent\.|top\.|self\.)[^><]*(href|url|src|replace) *[=\(] *[\'"]{0,1}https*:\/\/[^\s "\'>]+[\'"\;\)]{0,1}[^>]*>/i', $this->html, $matches);
		}
		$check_counter = 0;
		$second_level = preg_quote(preg_replace('/^(.+\.)*([^\.]+)\.[^\.]+$/', "$2", $url_components['host']));
		foreach ($matches[0] as $match) {
			if (!preg_match('/<a.+rel=.*nofollow.*>/', $match)) {
				preg_match('/(href|url|src|replace) *[=\(] *[\'"]{0,1}(https*:\/\/[^\s "\'>]+)[\'"\;\)]{0,1}/i', $match, $url_a);
				$embeded_link  = $url_a[2];
				$new_url_components = @parse_url($embeded_link);
				if (! empty($embeded_link) && $check_counter < 5 && ! $checked_links[$new_url_components['host']]) {
					if (! preg_match("/$second_level\.[^\.]+$/", $new_url_components['host']) ) {
						$check_counter++;
					}
					$checked_links[$new_url_components['host']] = true;
					if (!$this->check_url($embeded_link, false) && $this->banned) return false;
				}
			}
		}

		// The URL has been checked
		$this->valid = true;

		if(preg_match('/<title[^<>]*>([^<>]*)<\/title>/si', $this->html, $matches)) {
			$url_title=clean_text($matches[1]);
			if (mb_strlen($url_title) > 3) {
				$this->url_title=$url_title;
			}
		}

		if(preg_match('/< *meta +name=[\'"]description[\'"] +content=[\'"]([^<>]+)[\'"] *\/*>/si', $this->html, $matches)) {
			$this->url_description=clean_text_with_tags($matches[1], 0, false, 400);
		}

		return true;
	}


	function trackback() {
		// Now detect trackbacks
		if (preg_match('/trackback:ping="([^"]+)"/i', $this->html, $matches) ||
			preg_match('/trackback:ping +rdf:resource="([^>]+)"/i', $this->html, $matches) ||
			preg_match('/<trackback:ping>([^<>]+)/i', $this->html, $matches)) {
			$trackback=trim($matches[1]);
		} elseif (preg_match('/<a[^>]+rel="trackback"[^>]*>/i', $this->html, $matches)) {
			if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
				$trackback=trim($matches2[1]);
			}
		} elseif (preg_match('/<a[^>]+href=[^>#]+>[^>]*trackback[^>]*<\/a>/i', $this->html, $matches)) {
			if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
				$trackback=trim($matches2[1]);
			}
		}  elseif (preg_match('/(http:\/\/[^\s#]+\/trackback[\/\w]*)/i', $this->html, $matches)) {
			$trackback=trim($matches[0]);
		}

		if (!empty($trackback)) {
			$this->trackback = clean_input_url($trackback);
			return true;
		}
		return false;
	}

	function pingback() {
		$url_components = @parse_url($this->url);
		// Now we use previous pingback or detect it
		if ((!empty($url_components['query']) || preg_match('|^/.*[\.-/]+|', $url_components['path']))) {
			if (!empty($this->pingback)) {
				$trackback = $this->pingback;
			} elseif (preg_match('/<link[^>]+rel="pingback"[^>]*>/i', $this->html, $matches)) {
				if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
					$trackback='ping:'.trim($matches2[1]);
				}
			}
		}
		if (!empty($trackback)) {
			$this->trackback = clean_input_url($trackback);
			return true;
		}
		return false;
	}

	function enqueue() {
		global $db, $globals, $current_user;
		// Check this one was not already queued
		if($this->votes == 0 && $this->author == $current_user->user_id && $this->status != 'queued') {
			$this->status='queued';
			$this->sent_date = $this->date=time();
			$this->get_uri();
			$db->transaction();
			if( ! $this->store()) {
				$db->rollback();
				return false;
			}
			$this->insert_vote($current_user->user_karma);

			// Add the new link log/event
			Log::conditional_insert('link_new', $this->id, $this->author);

			$db->query("delete from links where link_author = $this->author and link_date > date_sub(now(), interval 30 minute) and link_status='discard' and link_votes=0");
			if(!empty($_POST['trackback'])) {
				$trackres = new Trackback;
				$trackres->url=clean_input_url($_POST['trackback']);
				$trackres->link_id=$this->id;
				$trackres->link=$this->url;
				$trackres->author=$this->author;
				$trackres->status = 'pendent';
				$trackres->store();
			}
			$db->commit();
			fork("backend/send_pingbacks.php?id=$this->id");
		}
	}


	function has_rss() {
		return preg_match('/<link[^>]+(text\/xml|application\/rss\+xml|application\/atom\+xml)[^>]+>/i', $this->html);
	}

	function create_blog_entry() {
		$blog = new Blog();
		$blog->analyze_html($this->url, $this->html);
		if(!$blog->read('key')
			|| ($blog->type != 'noiframe' && $this->noiframe)) {

			if ($blog->type != 'noiframe' && $this->noiframe) {
				$blog->type = 'noiframe';
				syslog(LOG_INFO, "Meneame, changed to noiframe ($blog->id, $blog->url)");
			}
			$blog->store();
		}
		$this->blog=$blog->id;
		$this->type=$blog->type;
	}

	function type() {
		if (empty($this->type)) {
			if ($this->blog > 0) {
				$blog = new Blog();
				$blog->id = $this->blog;
				if($blog->read()) {
					$this->type=$blog->type;
					return $this->type;
				}
			}
			return 'normal';
		}
		return $this->type;
	}

	function store() {
		global $db, $current_user, $globals;

		$link_url = $db->escape($this->url);
		$link_uri = $db->escape($this->uri);
		$link_url_title = $db->escape($this->url_title);
		$link_title = $db->escape($this->title);
		$link_tags = $db->escape($this->tags);
		$link_content = $db->escape($this->content);
		$link_thumb = $db->escape($this->thumb);
		$link_thumb_x = intval($this->thumb_x);
		$link_thumb_y = intval($this->thumb_y);
		$link_thumb_status = $db->escape($this->thumb_status);
		$db->transaction();
		if (! $this->store_basic()) {
			$db->rollback();
			return false;
		}

		$r = $db->query("UPDATE links set link_url='$link_url', link_uri='$link_uri', link_url_title='$link_url_title', link_title='$link_title', link_content='$link_content', link_tags='$link_tags', link_thumb='$link_thumb', link_thumb_x=$link_thumb_x, link_thumb_y=$link_thumb_y, link_thumb_status='$link_thumb_status' WHERE link_id=$this->id");
		$db->commit();
		return $r;

	}

	function store_basic($really_basic = false) {
		global $db, $current_user, $globals;

		if(!$this->date) $this->date=$globals['now'];
		$link_author = $this->author;
		$link_blog = $this->blog;
		$link_status = $db->escape($this->status);
		$link_anonymous = $this->anonymous;
		$link_karma = $this->karma;
		$link_votes_avg = $this->votes_avg;
		$link_randkey = $this->randkey;
		$link_date = $this->date;
		$link_sent_date = $this->sent_date;
		$link_published_date = $this->published_date;
		$link_content_type = $db->escape($this->content_type);
		$db->transaction();
		if($this->id===0) {
			$this->ip = $globals['user_ip'];
			$link_ip = $db->escape($this->ip);
			$this->ip_int = $globals['user_ip_int'];

			$r = $db->query("INSERT INTO links (link_author, link_blog, link_status, link_randkey, link_date, link_sent_date, link_published_date, link_karma, link_anonymous, link_votes_avg, link_content_type, link_ip_int, link_ip) VALUES ($link_author, $link_blog, '$link_status', $link_randkey FROM_UNIXTIME($link_date), FROM_UNIXTIME($link_sent_date), FROM_UNIXTIME($link_published_date), $link_karma, $link_anonymous, $link_votes_avg, '$link_content_type', $this->ip_int, '$link_ip')");
			$this->id = $db->insert_id;
		} else {
		// update
			$r = $db->query("UPDATE links set link_author=$link_author, link_blog=$link_blog, link_status='$link_status', link_randkey=$link_randkey, link_date=FROM_UNIXTIME($link_date), link_sent_date=FROM_UNIXTIME($link_sent_date), link_published_date=FROM_UNIXTIME($link_published_date), link_karma=$link_karma, link_votes_avg=$link_votes_avg, link_content_type='$link_content_type' WHERE link_id=$this->id");<commit_msg>Fixed syntax error in SQL<commit_after><?
// The source code packaged with this file is Free Software, Copyright (C) 2005 by
// <NAME> <gallir at uib dot es>.
// It's licensed under the AFFERO GENERAL PUBLIC LICENSE unless stated otherwise.
// You can get copies of the licenses here:
//		http://www.affero.org/oagpl.html
// AFFERO GENERAL PUBLIC LICENSE is also included in the file called "COPYING".

require_once(mnminclude.'favorites.php');

class Link extends LCPBase {
	private static $clicked = 0; // Must add a click to this link->id
	public static $original_status = false;

	var $id = 0;
	var $author = -1;
	var $blog = 0;
	var $username = false;
	var $randkey = 0;
	var $karma = 0;
	var $valid = false;
	var $date = false;
	var $sent_date = 0;
	var $published_date = 0;
	var $modified = 0;
	var $url = false;
	var $url_title = '';
	var $encoding = false;
	var $status = 'discard';
	var $type = '';
	var $votes = 0;
	var $anonymous = 0;
	var $votes_avg = 0;
	var $negatives = 0;
	var $title = '';
	var $tags = '';
	var $uri = '';
	var $thumb_url = false;
	var $content = '';
	var $content_type = '';
	var $ip = '';
	var $html = false;
	var $read = false;
	var $voted = false;
	var $banned = false;
	var $thumb_status = 'unknown';
	var $clicks = 0;

	// sql fields to build an object from mysql
	const SQL = " link_id as id, link_author as author, link_blog as blog, link_status as status, sub_statuses.status as sub_status, UNIX_TIMESTAMP(sub_statuses.date) as sub_date, link_votes as votes, link_negatives as negatives, link_anonymous as anonymous, link_votes_avg as votes_avg, link_votes + link_anonymous as total_votes, link_comments as comments, link_karma as karma, sub_statuses.karma as sub_karma, link_randkey as randkey, link_url as url, link_uri as uri, link_url_title as url_title, link_title as title, link_tags as tags, link_content as content, UNIX_TIMESTAMP(link_date) as date,  UNIX_TIMESTAMP(link_sent_date) as sent_date, UNIX_TIMESTAMP(link_published_date) as published_date, UNIX_TIMESTAMP(link_modified) as modified, link_content_type as content_type, link_ip as ip, link_thumb_status as thumb_status, link_thumb_x as thumb_x, link_thumb_y as thumb_y, link_thumb as thumb, user_login as username, user_email as email, user_avatar as avatar, user_karma as user_karma, user_level as user_level, user_adcode, subs.name as sub_name, subs.id as sub_id, subs.server_name, subs.sub as is_sub, subs.owner as sub_owner, subs.base_url, subs.created_from, subs.allow_main_link, creation.status as sub_status_origen, subs.color1 as sub_color1, subs.color2 as sub_color2, favorite_link_id as favorite, clicks.counter as clicks, votes.vote_value as voted FROM links
	INNER JOIN users on (user_id = link_author)
	LEFT JOIN sub_statuses ON (@site_id > 0 and sub_statuses.id = @site_id and sub_statuses.link = links.link_id)
	LEFT JOIN (sub_statuses as creation, subs) ON (creation.link=links.link_id and creation.id=creation.origen and creation.id=subs.id)
	LEFT JOIN votes ON (link_date > @enabled_votes and vote_type='links' and vote_link_id = links.link_id and vote_user_id = @user_id and ( @user_id > 0  OR vote_ip_int = @ip_int ) )
	LEFT JOIN favorites ON (@user_id > 0 and favorite_user_id =  @user_id and favorite_type = 'link' and favorite_link_id = links.link_id)
	LEFT JOIN link_clicks as clicks on (clicks.id = links.link_id) ";

	const SQL_BASIC = " link_id as id, link_author as author, link_blog as blog, link_status as status, sub_statuses.status as sub_status, link_votes as votes, link_negatives as negatives, link_anonymous as anonymous, link_votes_avg as votes_avg, link_votes + link_anonymous as total_votes, link_comments as comments, link_karma as karma, sub_statuses.karma as sub_karma, link_randkey as randkey, link_url as url, link_uri as uri, link_url_title as url_title, link_title as title, link_tags as tags, link_content as content, UNIX_TIMESTAMP(link_date) as date,	UNIX_TIMESTAMP(link_sent_date) as sent_date, UNIX_TIMESTAMP(link_published_date) as published_date, UNIX_TIMESTAMP(link_modified) as modified, link_content_type as content_type, link_ip as ip, link_thumb_status as thumb_status, link_thumb_x as thumb_x, link_thumb_y as thumb_y, link_thumb as thumb, user_login as username, user_email as email, user_avatar as avatar, user_karma as user_karma, user_level as user_level, user_adcode, subs.name as sub_name, subs.id as sub_id, subs.server_name, subs.sub as is_sub, subs.owner as sub_owner, subs.base_url, subs.created_from, subs.allow_main_link, creation.status as sub_status_origen FROM links
	INNER JOIN users on (user_id = link_author)
	LEFT JOIN sub_statuses ON (@site_id > 0 and sub_statuses.id = @site_id and sub_statuses.link = links.link_id)
	LEFT JOIN (sub_statuses as creation, subs) ON (creation.link=links.link_id and creation.id=creation.origen and creation.id=subs.id)";


	static function from_db($id, $key = 'id', $complete = true) {
		global $db, $current_user;

		SitesMgr::my_id(); // Force to read current sub_id
		switch ($key) {
			case 'uri':
				$id = $db->escape($id);
				$selector = "link_uri = '$id'";
				break;
			default:
				$id = intval($id);
				$selector = "link_id = $id";
		}

		if ($complete) {
			$sql = "SELECT".Link::SQL."WHERE $selector";
		} else {
			$sql = "SELECT".Link::SQL_BASIC."WHERE $selector";
		}


		if(($object = $db->get_object($sql, 'Link'))) {
			$object->read = true;
			return $object;
		}
		return false;
	}

	static function count($status='', $force = false) {
		global $db, $globals;

		$my_id = SitesMgr::my_id();

		if (!$status) return Link::count('published', $force) +
							Link::count('queued', $force) +
							Link::count('discard', $force) +
							Link::count('abuse', $force) +
							Link::count('autodiscard', $force);


		$count = get_count("$my_id.$status");
		if ($count === false || $force) {
			$count = $db->get_var("select count(*) from sub_statuses where id = $my_id and status = '$status'");
			set_count("$my_id.$status", $count);
		}
		return $count;
	}

	static function duplicates($url, $site = false) {
		global $db;
		$trimmed = $db->escape(preg_replace('/\/$/', '', $url));
		$list = "'$trimmed', '$trimmed/'";
		if (preg_match('/^http.{0,1}:\/\/www\./', $trimmed)) {
			$link_alternative = preg_replace('/^(http.{0,1}):\/\/www\./', '$1://', $trimmed);
		} else {
			$link_alternative = preg_replace('/^(http.{0,1}):\/\//', '$1://www.', $trimmed);
		}
		$list .= ", '$link_alternative', '$link_alternative/'";

		/* Alternative to http and https */
		if (preg_match('/^http:/', $url)) {
			$list2 = preg_replace('/http:\/\//', 'https://', $list);
		} else {
			$list2 = preg_replace('/https:\/\//', 'http://', $list);
		}
		$list .= ", $list2";

/***** TODO: check and decid how duplicated must be dealt with
		$subs = array();
		$site_id = SitesMgr::my_id();
		$subs[] = $site_id;
		$subs = array_merge($subs, SitesMgr::get_senders(), SitesMgr::get_receivers());
		$subs = implode(',', $subs);

		// If it was abuse o autodiscarded allow other to send it again
		$found = $db->get_var("SELECT link_id FROM links, sub_statuses WHERE link_url in ($list) AND status not in ('abuse') AND link_votes > 0 AND sub_statuses.link = link_id AND sub_statuses.id in ($subs) ORDER by link_id asc limit 1");
*******/

		if ($site > 0) {
			$from_extra = ', sub_statuses';
			$where_extra = "and sub_statuses.id = $site and sub_statuses.link = link_id";
		} else {
			$from_extra = $where_extra = '';
		}
		// If it was abuse o autodiscarded allow other to send it again
		$found = $db->get_var("SELECT link_id FROM links $from_extra WHERE link_url in ($list) AND link_status not in ('abuse') AND link_votes > 0 $where_extra ORDER by link_id asc limit 1");
		return $found;
	}

	static function visited($id) {
		global $globals;

		if (! isset($_COOKIE['v']) || ! ($visited = preg_split('/x/', $_COOKIE['v'], 0, PREG_SPLIT_NO_EMPTY)) ) {
			$visited = array();
			$found = false;
		} else {
			$found = array_search($id, $visited);
			if (count($visited) > 10) {
				array_shift($visited);
			}
			if ($found !== false) {
				unset($visited[$found]);
			}
		}
		$visited[] = $id;
		setcookie('v', implode('x', $visited), 0, $globals['base_url_general'], UserAuth::domain());
		return $found !== false;
	}

	function add_click() {
		global $globals, $db;

		if (! $globals['bot']
			&& ! Link::visited($this->id)
			&& $globals['click_counter']
			&& isset($_COOKIE['k'])
			&& check_security_key($_COOKIE['k'])
			&& $this->ip != $globals['user_ip']) {
			// Delay storing
			self::$clicked = $this->id;
		}
	}

	static function store_clicks() {
		global $globals, $db;

		if (!self::$clicked) return false;
		$id = self::$clicked;
		self::$clicked = 0;

		if (! memcache_menabled()) {
			$db->query("UPDATE link_clicks SET counter=counter+1 WHERE id = $id");
			return true;
		}

		$key = 'clicks_cache';
		$cache = memcache_mget($key);
		if (!$cache) {
			$cache = array();
			$cache['time'] = $globals['start_time'];
			$cache[$id] = 1;
		} else {
			$cache[$id]++;
		}

		if ($globals['start_time'] - $cache['time'] > 4.0) {
			if( !memcache_mdelete($key)) {
				memcache_madd($key, array());
				syslog(LOG_INFO, "store_clicks: Delete failed");
			}
			ksort($cache); // To avoid transaction's deadlocks

			$show_errors = $db->show_errors;
			$db->show_errors = false; // we know there can be lock timeouts :(
			$tries = 0; // By the way, freaking locking timeouts with few updates per second with this technique
			while ($tries < 3) {
				$error = false;
				$db->transaction();
				$total= 0;
				$r = true;
				foreach ($cache as $id => $counter) {
					if ($id > 0 && $counter > 0) {
						$r = $db->query("INSERT INTO link_clicks (id, counter) VALUES ($id,$counter) ON DUPLICATE KEY UPDATE counter=counter+$counter");
						// $r = $db->query("UPDATE link_clicks SET counter=counter+$counter WHERE id = $id");
						if (!$r) {
							break;
						}
						$total += $counter;
					}
				}
				if ($r) {
					$db->commit();
					$tries = 100000; // Stop it
				} else {
					$tries++;
					syslog(LOG_INFO, "failed $tries attempts in store_clicks");
					$db->rollback();
				}
			}
			$db->show_errors = $show_errors;
		} else {
			memcache_madd($key, $cache);
		}
	}

	static function top() {
		global $globals;

		// It retrieves the annotiation generated by the Python script top-news.py
		$top = new Annotation('top-link-'.$globals['site_shortname']);
		if (!$top->read()) return false;
		$ids = explode(',',$top->text);
		if (! $ids) return false;

		$link = Link::from_db($ids[0]);
		if ($link) {
			$link->has_thumb();
		}
		return $link;
	}

	function json_votes_info($value=false) {
		$dict = array();
		$dict['id'] = $this->id;
		if ($value) {
			$dict['value'] = $value;
			if ($value < 0) {
				$dict['vote_description'] = get_negative_vote($value);
			} else {
				$dict['vote_description'] = _('¡chachi!');
			}
		}
		$dict['votes'] = $this->votes;
		$dict['anonymous'] = $this->anonymous;
		$dict['negatives'] = $this->negatives;
		if ($this->sub_status == 'published') {
			$dict['karma'] = intval($this->sub_karma);
		} else {
			$dict['karma'] = intval($this->karma);
		}
		return json_encode($dict);
	}

	function print_html() {
		echo "Valid: " . $this->valid . "<br>\n";
		echo "Url: " . $this->url . "<br>\n";
		echo "Title: " . $this->url_title . "<br>\n";
		echo "encoding: " . $this->encoding . "<br>\n";
	}

	function check_url($url, $check_ban = true, $first_level = false) {
		global $globals, $current_user;
		if(!preg_match('/^http[s]*:/', $url)) return false;
		$url_components = @parse_url($url);
		if (!$url_components) return false;
		if (!preg_match('/[a-z]+/', $url_components['host'])) return false;

		if (! $check_ban) return true;
		$quoted_domain = preg_quote(get_server_name());
		if(preg_match("/^$quoted_domain$/", $url_components['host'])) {
			$this->ban = array();
			$this->ban['comment'] = _('el servidor es local');
			syslog(LOG_NOTICE, "Meneame, server name is local name ($current_user->user_login): $url");
			return false;
		}
		require_once(mnminclude.'ban.php');
		if(($this->ban = check_ban($url, 'hostname', false, $first_level))) {
			syslog(LOG_NOTICE, "Meneame, server name is banned ($current_user->user_login): $url");
			$this->banned = true;
			return false;
		}
		return true;
	}

	function get($url, $maxlen = 150000, $check_ban = true) {
		global $globals, $current_user;
		$url=trim($url);
		$url_components = @parse_url($url);


		$this->noiframe = false;
		if (($response = get_url($url)) ) {
			$this->content_type = preg_replace('#^(\w+).+#', '$1', $response['content_type']);

			// Check if it has pingbacks
			if (preg_match('/X-Pingback: *(.+)/i', $response['header'], $match)) {
				$this->pingback = 'ping:'.clean_input_url($match[1]);
			}

			/* Were we redirected? */
			if ($response['redirect_count'] > 0) {
				/* update $url with where we were redirected to */
				$new_url = clean_input_url($response['location']);
			}
			if (!empty($new_url) && $new_url != $url) {
				syslog(LOG_NOTICE, "Meneame, redirected ($current_user->user_login): $url -> $new_url");
				/* Check again the url */
				if (!$this->check_url($new_url, $check_ban, true)) {
					$this->url = $new_url;
					return false;
				}
				// Change the url if we were directed to another host
				if (strlen($new_url) < 300	&& ($new_url_components = @parse_url($new_url))) {
					if ($url_components['host'] != $new_url_components['host']) {
						syslog(LOG_NOTICE, "Meneame, changed source URL ($current_user->user_login): $url -> $new_url");
						$url = $new_url;
						$url_components = $new_url_components;
					}
				}
			}
			$this->html = $response['content'];
			$url_ok = true;
		} else {
			syslog(LOG_NOTICE, "Meneame, error getting ($current_user->user_login): $url");
			$url_ok = false;
		}


		$this->url=$url;
		// Fill content type if empty
		// Right now only check for typical image extensions
		if (empty($this->content_type)) {
			if (preg_match('/(jpg|jpeg|gif|png)(\?|#|$)/i', $this->url)) {
				$this->content_type='image';
			}
		}
		// NO more to do
		if (!$url_ok || ! preg_match('/html/', $response['content_type'])) return true;

		// Check if it forbides including in an iframe
		if (preg_match('/X-Frame-Options: *(.+)/i', $response['header'])
			|| preg_match('/top\.location\.href *=/', $response['content'])) {
			$this->noiframe = true;
		}

		if(preg_match('/charset=([a-zA-Z0-9-_]+)/i', $this->html, $matches)) {
			$this->encoding=trim($matches[1]);
			if(strcasecmp($this->encoding, 'utf-8') != 0) {
				$this->html=iconv($this->encoding, 'UTF-8//IGNORE', $this->html);
			}
		}


		// Check if the author doesn't want to share
		if (preg_match('/<!-- *noshare *-->/', $this->html)) {
			$this->ban = array();
			$this->ban['comment'] = _('el autor no desea que se envíe el artículo, respeta sus deseos');
			syslog(LOG_NOTICE, "Meneame, noshare ($current_user->user_login): $url");
			return false;
		}

		// Now we analyse the html to find links to banned domains
		// It avoids the trick of using google or technorati
		// Ignore it if the link has a rel="nofollow" to ignore comments in blogs
		if (!preg_match('/content="[^"]*(vBulletin|phpBB)/i', $this->html)) {
			preg_match_all('/(< *meta +http-equiv|< *frame[^<]*>|window\.|document.\|parent\.|top\.|self\.)[^><]*(url|src|replace) *[=\(] *[\'"]{0,1}https*:\/\/[^\s "\'>]+[\'"\;\)]{0,1}[^>]*>/i', $this->html, $matches);
		} else {
			preg_match_all('/(<* meta +http-equiv|<* iframe|<* frame[^<]*>|window\.|document.\|parent\.|top\.|self\.)[^><]*(href|url|src|replace) *[=\(] *[\'"]{0,1}https*:\/\/[^\s "\'>]+[\'"\;\)]{0,1}[^>]*>/i', $this->html, $matches);
		}
		$check_counter = 0;
		$second_level = preg_quote(preg_replace('/^(.+\.)*([^\.]+)\.[^\.]+$/', "$2", $url_components['host']));
		foreach ($matches[0] as $match) {
			if (!preg_match('/<a.+rel=.*nofollow.*>/', $match)) {
				preg_match('/(href|url|src|replace) *[=\(] *[\'"]{0,1}(https*:\/\/[^\s "\'>]+)[\'"\;\)]{0,1}/i', $match, $url_a);
				$embeded_link  = $url_a[2];
				$new_url_components = @parse_url($embeded_link);
				if (! empty($embeded_link) && $check_counter < 5 && ! $checked_links[$new_url_components['host']]) {
					if (! preg_match("/$second_level\.[^\.]+$/", $new_url_components['host']) ) {
						$check_counter++;
					}
					$checked_links[$new_url_components['host']] = true;
					if (!$this->check_url($embeded_link, false) && $this->banned) return false;
				}
			}
		}

		// The URL has been checked
		$this->valid = true;

		if(preg_match('/<title[^<>]*>([^<>]*)<\/title>/si', $this->html, $matches)) {
			$url_title=clean_text($matches[1]);
			if (mb_strlen($url_title) > 3) {
				$this->url_title=$url_title;
			}
		}

		if(preg_match('/< *meta +name=[\'"]description[\'"] +content=[\'"]([^<>]+)[\'"] *\/*>/si', $this->html, $matches)) {
			$this->url_description=clean_text_with_tags($matches[1], 0, false, 400);
		}

		return true;
	}


	function trackback() {
		// Now detect trackbacks
		if (preg_match('/trackback:ping="([^"]+)"/i', $this->html, $matches) ||
			preg_match('/trackback:ping +rdf:resource="([^>]+)"/i', $this->html, $matches) ||
			preg_match('/<trackback:ping>([^<>]+)/i', $this->html, $matches)) {
			$trackback=trim($matches[1]);
		} elseif (preg_match('/<a[^>]+rel="trackback"[^>]*>/i', $this->html, $matches)) {
			if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
				$trackback=trim($matches2[1]);
			}
		} elseif (preg_match('/<a[^>]+href=[^>#]+>[^>]*trackback[^>]*<\/a>/i', $this->html, $matches)) {
			if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
				$trackback=trim($matches2[1]);
			}
		}  elseif (preg_match('/(http:\/\/[^\s#]+\/trackback[\/\w]*)/i', $this->html, $matches)) {
			$trackback=trim($matches[0]);
		}

		if (!empty($trackback)) {
			$this->trackback = clean_input_url($trackback);
			return true;
		}
		return false;
	}

	function pingback() {
		$url_components = @parse_url($this->url);
		// Now we use previous pingback or detect it
		if ((!empty($url_components['query']) || preg_match('|^/.*[\.-/]+|', $url_components['path']))) {
			if (!empty($this->pingback)) {
				$trackback = $this->pingback;
			} elseif (preg_match('/<link[^>]+rel="pingback"[^>]*>/i', $this->html, $matches)) {
				if (preg_match('/href="([^"]+)"/i', $matches[0], $matches2)) {
					$trackback='ping:'.trim($matches2[1]);
				}
			}
		}
		if (!empty($trackback)) {
			$this->trackback = clean_input_url($trackback);
			return true;
		}
		return false;
	}

	function enqueue() {
		global $db, $globals, $current_user;
		// Check this one was not already queued
		if($this->votes == 0 && $this->author == $current_user->user_id && $this->status != 'queued') {
			$this->status='queued';
			$this->sent_date = $this->date=time();
			$this->get_uri();
			$db->transaction();
			if( ! $this->store()) {
				$db->rollback();
				return false;
			}
			$this->insert_vote($current_user->user_karma);

			// Add the new link log/event
			Log::conditional_insert('link_new', $this->id, $this->author);

			$db->query("delete from links where link_author = $this->author and link_date > date_sub(now(), interval 30 minute) and link_status='discard' and link_votes=0");
			if(!empty($_POST['trackback'])) {
				$trackres = new Trackback;
				$trackres->url=clean_input_url($_POST['trackback']);
				$trackres->link_id=$this->id;
				$trackres->link=$this->url;
				$trackres->author=$this->author;
				$trackres->status = 'pendent';
				$trackres->store();
			}
			$db->commit();
			fork("backend/send_pingbacks.php?id=$this->id");
		}
	}


	function has_rss() {
		return preg_match('/<link[^>]+(text\/xml|application\/rss\+xml|application\/atom\+xml)[^>]+>/i', $this->html);
	}

	function create_blog_entry() {
		$blog = new Blog();
		$blog->analyze_html($this->url, $this->html);
		if(!$blog->read('key')
			|| ($blog->type != 'noiframe' && $this->noiframe)) {

			if ($blog->type != 'noiframe' && $this->noiframe) {
				$blog->type = 'noiframe';
				syslog(LOG_INFO, "Meneame, changed to noiframe ($blog->id, $blog->url)");
			}
			$blog->store();
		}
		$this->blog=$blog->id;
		$this->type=$blog->type;
	}

	function type() {
		if (empty($this->type)) {
			if ($this->blog > 0) {
				$blog = new Blog();
				$blog->id = $this->blog;
				if($blog->read()) {
					$this->type=$blog->type;
					return $this->type;
				}
			}
			return 'normal';
		}
		return $this->type;
	}

	function store() {
		global $db, $current_user, $globals;

		$link_url = $db->escape($this->url);
		$link_uri = $db->escape($this->uri);
		$link_url_title = $db->escape($this->url_title);
		$link_title = $db->escape($this->title);
		$link_tags = $db->escape($this->tags);
		$link_content = $db->escape($this->content);
		$link_thumb = $db->escape($this->thumb);
		$link_thumb_x = intval($this->thumb_x);
		$link_thumb_y = intval($this->thumb_y);
		$link_thumb_status = $db->escape($this->thumb_status);
		$db->transaction();
		if (! $this->store_basic()) {
			$db->rollback();
			return false;
		}

		$r = $db->query("UPDATE links set link_url='$link_url', link_uri='$link_uri', link_url_title='$link_url_title', link_title='$link_title', link_content='$link_content', link_tags='$link_tags', link_thumb='$link_thumb', link_thumb_x=$link_thumb_x, link_thumb_y=$link_thumb_y, link_thumb_status='$link_thumb_status' WHERE link_id=$this->id");
		$db->commit();
		return $r;

	}

	function store_basic($really_basic = false) {
		global $db, $current_user, $globals;

		if(!$this->date) $this->date=$globals['now'];
		$link_author = $this->author;
		$link_blog = $this->blog;
		$link_status = $db->escape($this->status);
		$link_anonymous = $this->anonymous;
		$link_karma = $this->karma;
		$link_votes_avg = $this->votes_avg;
		$link_randkey = $this->randkey;
		$link_date = $this->date;
		$link_sent_date = $this->sent_date;
		$link_published_date = $this->published_date;
		$link_content_type = $db->escape($this->content_type);
		$db->transaction();
		if($this->id===0) {
			$this->ip = $globals['user_ip'];
			$link_ip = $db->escape($this->ip);
			$this->ip_int = $globals['user_ip_int'];

			$r = $db->query("INSERT INTO links (link_author, link_blog, link_status, link_randkey, link_date, link_sent_date, link_published_date, link_karma, link_anonymous, link_votes_avg, link_content_type, link_ip_int, link_ip) VALUES ($link_author, $link_blog, '$link_status', $link_randkey, FROM_UNIXTIME($link_date), FROM_UNIXTIME($link_sent_date), FROM_UNIXTIME($link_published_date), $link_karma, $link_anonymous, $link_votes_avg, '$link_content_type', $this->ip_int, '$link_ip')");
			$this->id = $db->insert_id;
		} else {
		// update
			$r = $db->query("UPDATE links set link_author=$link_author, link_blog=$link_blog, link_status='$link_status', link_randkey=$link_randkey, link_date=FROM_UNIXTIME($link_date), link_sent_date=FROM_UNIXTIME($link_sent_date), link_published_date=FROM_UNIXTIME($link_published_date), link_karma=$link_karma, link_votes_avg=$link_votes_avg, link_content_type='$link_content_type' WHERE link_id=$this->id");
{'fraction_non_alphanumeric': 0.19498274777023067, 'fraction_numerical': 0.005468631323104967, 'mean_word_length': 4.208771334915791, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 509, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '911095', 'n_tokens_mistral': 18962, 'n_tokens_neox': 16891, 'n_words': 3697}