<commit_before><?php
	
function Websites_scrape_post($params)
{
	// don't let just anyone call this, but only pages loaded from valid sessions
	Q_Valid::nonce(true);

    $params = array_merge($_REQUEST, $params);

	$fields = Q::take($params, array('url', 'skipStream'));

	$url = $fields['url'];
	$skipStream = (bool)$fields['skipStream'];

	if (parse_url($url, PHP_URL_SCHEME) === null) {
		$url = 'http://'.$url;
	}

	// if stream for this URL already exist, return it
	$streamExist = Websites_scrape_fetchStream($url);
	if ($streamExist) {
		Q_Response::setSlot('result', $streamExist);
		return;
	}

	$result = Websites_Webpage::scrape($url);

	if (!$skipStream) {
        // if stream for this URL already exist, return it
        $stream = Websites_scrape_fetchStream($result['url']);
        if (!$stream) {
            $stream = Q::event('Websites/webpage/post', $result);
        }

        Q_Response::setSlot('stream', $stream);
    }

	Q_Response::setSlot('result', $result);
}

function Websites_scrape_fetchStream($url) {
	// if stream for this URL already exist, return it
	$stream = Websites_Webpage::fetchStream($url);
	if (!$stream) {
		return null;
	}

	$interest = $stream->getAttribute('interest');
	$interestPublisherId = Q::ifset($interest, 'publisherId', null);
	$interestStreamName = Q::ifset($interest, 'streamName', null);
	$interestData = array(
		'title' => null,
		'icon' => null
	);
	if ($interestPublisherId && $interestStreamName) {
		$interest = Streams::fetchOne($interestPublisherId, $interestPublisherId, $interestStreamName);

		if ($interest) {
			$interestData['title'] = $interest->title;
			$interestData['icon'] = $interest->iconUrl($interest->getAttribute('iconSize'));
		}
	}

    Q_Response::setSlot('stream', $stream);
	Q_Response::setSlot('publisherId', $stream->publisherId);
	Q_Response::setSlot('streamName', $stream->name);

	return array(
		'title' => $stream->title,
		'description' => $stream->content,
		'smallIcon' => $interestData['icon'],
		'host' => $stream->getAttribute('urlParsed')['host'],
		'bigIcon' => $stream->getAttribute('icon'),
		'url' => $stream->getAttribute('url'),
		'alreadyExist' => true
	);
}<commit_msg>Modified Websites/scrape/post handler. Added option "withStream" which force to create stream (if not created yet).<commit_after><?php
	
function Websites_scrape_post($params)
{
	// don't let just anyone call this, but only pages loaded from valid sessions
	Q_Valid::nonce(true);

    $params = array_merge($_REQUEST, $params);

	$fields = Q::take($params, array('url', 'skipStream'));

	$url = $fields['url'];
	$withStream = (bool)$fields['withStream'];

	if (parse_url($url, PHP_URL_SCHEME) === null) {
		$url = 'http://'.$url;
	}

	$result = Websites_Webpage::scrape($url);

	if ($withStream) {
		// if stream for this URL already exist, return it
		$stream = Websites_scrape_fetchStream($result['url']);
		if (!$stream) {
			$stream = Q::event('Websites/webpage/post', $result);
		}

		$result['publisherId'] = $stream->publisherId;
		$result['streamName'] = $stream->name;
		Q_Response::setSlot('stream', $stream);
    } else {
		$result['publisherId'] = null;
		$result['streamName'] = null;
	}

	Q_Response::setSlot('result', $result);
}

/**
 * Along with getting stream, this method get small icon from interest
 * @method Websites_scrape_fetchStream
 * @static
 * @param string $url
 * @return array
 */
function Websites_scrape_fetchStream($url) {
	// if stream for this URL already exist, return it
	$stream = Websites_Webpage::fetchStream($url);
	if (!$stream) {
		return null;
	}

	$interest = $stream->getAttribute('interest');
	$interestPublisherId = Q::ifset($interest, 'publisherId', null);
	$interestStreamName = Q::ifset($interest, 'streamName', null);
	$interestData = array(
		'title' => null,
		'icon' => null
	);
	if ($interestPublisherId && $interestStreamName) {
		$interest = Streams::fetchOne($interestPublisherId, $interestPublisherId, $interestStreamName);

		if ($interest) {
			$interestData['title'] = $interest->title;
			$interestData['icon'] = $interest->iconUrl($interest->getAttribute('iconSize'));
		}
	}

    Q_Response::setSlot('stream', $stream);
	Q_Response::setSlot('publisherId', $stream->publisherId);
	Q_Response::setSlot('streamName', $stream->name);

	return array(
		'title' => $stream->title,
		'description' => $stream->content,
		'smallIcon' => $interestData['icon'],
		'host' => $stream->getAttribute('urlParsed')['host'],
		'iconBig' => $stream->getAttribute('iconBig'),
		'url' => $stream->getAttribute('url'),
		'alreadyExist' => true
	);
}
{'fraction_non_alphanumeric': 0.18183802663173979, 'fraction_numerical': 0.0, 'mean_word_length': 5.2170963364993215, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4890568', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1512, 'n_words': 310}