<commit_before><?php
$DB = "howiknowit";
$db = "";

/*
	Connect to the database
*/
function connectDB() {
	global $DB;
	global $db;
	
	//Connects to MongoDB instance running at localhost:27017
	$connection = new MongoClient();

	//Select a db
	$db = $connection->$DB;
}

/*
	Inserts the given document into the specified collection/table
*/
function insertIntoCollection($table, $document) {
	global $db;

	//Select the collection/table
	$collection = $db->$table;
	
	//Perform the insert
	//$options = array("w" => 1);
	//resut =
	$collection->insert($document);//, $options);

	//TODO: check for error

	//Return the generated ObjectId
	//return $result['upserted'];
	return $document['_id'];
}

/*
	Queries the specified collection/table for documents using $q as the query
	Returns a cursor to the query results
*/
function queryCollection($collection, $q) {
	global $db;

	$cursor = $db->$collection->find($q);

	return $cursor;

	//TODO: error check
}

/*
	Returns an array containing the title of each media piece
*/
function getMedia() {
	global $db;

	//Array to store media items returned
	$media = array();

	//Get a cursor to the documents in the media collection
	$collection = "media";
	$cursor = $db->$collection->find();

	//Iterate through all the items the cursor has access to
	foreach($cursor as $doc)
	{
		//Put that item into the media array
    	array_push($media, $doc['title']);
	}

	return $media;
}

/*
	Parses a large block of text
	Breaks the block in paragraphs, storing each in the database
	Further, looks at each word in each paragraph and keep track of important words
*/
function parseText($media_id, $text) {
	//Generate the paragraphs
	$paragraphs = explode("\r\n\r\n", $text);

	//Add each paragraph to the paragraph collection
	for($i = 0; $i < count($paragraphs); ++$i) {
		//Create the document
		$document = array(
							"media_id" => $media_id,
							"location" => $i,
							"text"	   => $paragraphs[$i]
						 );

		//Insert it into the paragraphs collection
		$paragraph_id = insertIntoCollection("paragraphs", $document);

		//Generate words from the paragraph
		parseWords($media_id, $paragraph_id, $paragraphs[$i]);
	}
}

/*
	Given a paragraph, extracts important words and saves them to the database
*/
function parseWords($media_id, $paragraph_id, $paragraph) {
	//Remove punctuation, then fix for extra white spaces
	$paragraph = preg_replace("/[^\w]+/", "", $paragraph);
	$paragraph = preg_replace("/\s{2,}/g", " ", $paragraph);

	//Generate an array of all the words
	$words = explode(" ", $paragraph);

	//Go through all the words
	for($i = 0; $i < count($words); ++$i) {

		//If the cursor returned no results
		if(!inBlacklist($words[$i])) {
			//Create the document for the word
			$document = array(
								"media_id" 		=> $media_id,
								"paragraph_id"	=> $paragraph_id,
								"word"			=> $words[$i]
							 );

			//Insert it into the words collection
			insertIntoCollection("words", $document);
		} //Else do nothing - we don't care about the word
	}
}

/*
	Checks to see if a given word is in the blacklist (the list of words deemed unimportant)
*/
function inBlacklist($word) {
	if($word = "") {
		return false;
	}

	//Gets a cursor to the results of a query for the word
	$cursor = queryCollection("blacklist-words", array('word' => $word));

	return $cursor->count() > 0;
}

/*
	Given  media_id, returns the text to be displayed with mark-up (special words have a link)
*/
function howiknowit($media_id) {
	//Get a list of words for this document
	$words = array();
	$cursor = queryCollection();
	foreach($cursor as $doc) {
		array_push($words, $doc);<commit_msg>fixed an incorrect return boolean<commit_after><?php
$DB = "howiknowit";
$db = "";

/*
	Connect to the database
*/
function connectDB() {
	global $DB;
	global $db;
	
	//Connects to MongoDB instance running at localhost:27017
	$connection = new MongoClient();

	//Select a db
	$db = $connection->$DB;
}

/*
	Inserts the given document into the specified collection/table
*/
function insertIntoCollection($table, $document) {
	global $db;

	//Select the collection/table
	$collection = $db->$table;
	
	//Perform the insert
	//$options = array("w" => 1);
	//resut =
	$collection->insert($document);//, $options);

	//TODO: check for error

	//Return the generated ObjectId
	//return $result['upserted'];
	return $document['_id'];
}

/*
	Queries the specified collection/table for documents using $q as the query
	Returns a cursor to the query results
*/
function queryCollection($collection, $q) {
	global $db;

	$cursor = $db->$collection->find($q);

	return $cursor;

	//TODO: error check
}

/*
	Returns an array containing the title of each media piece
*/
function getMedia() {
	global $db;

	//Array to store media items returned
	$media = array();

	//Get a cursor to the documents in the media collection
	$collection = "media";
	$cursor = $db->$collection->find();

	//Iterate through all the items the cursor has access to
	foreach($cursor as $doc)
	{
		//Put that item into the media array
    	array_push($media, $doc['title']);
	}

	return $media;
}

/*
	Parses a large block of text
	Breaks the block in paragraphs, storing each in the database
	Further, looks at each word in each paragraph and keep track of important words
*/
function parseText($media_id, $text) {
	//Generate the paragraphs
	$paragraphs = explode("\r\n\r\n", $text);

	//Add each paragraph to the paragraph collection
	for($i = 0; $i < count($paragraphs); ++$i) {
		//Create the document
		$document = array(
							"media_id" => $media_id,
							"location" => $i,
							"text"	   => $paragraphs[$i]
						 );

		//Insert it into the paragraphs collection
		$paragraph_id = insertIntoCollection("paragraphs", $document);

		//Generate words from the paragraph
		parseWords($media_id, $paragraph_id, $paragraphs[$i]);
	}
}

/*
	Given a paragraph, extracts important words and saves them to the database
*/
function parseWords($media_id, $paragraph_id, $paragraph) {
	//Remove punctuation, then fix for extra white spaces
	$paragraph = preg_replace("/[^\w]+/", "", $paragraph);
	$paragraph = preg_replace("/\s{2,}/g", " ", $paragraph);

	//Generate an array of all the words
	$words = explode(" ", $paragraph);

	//Go through all the words
	foreach($words as $word) {
		//If the cursor into the blacklist returned no results
		if(!inBlacklist($word)) {
			//Create the document for the word
			$document = array(
								"media_id" 		=> $media_id,
								"paragraph_id"	=> $paragraph_id,
								"word"			=> $word
							 );

			//Insert it into the words collection
			insertIntoCollection("words", $document);
		} //Else do nothing - we don't care about the word
	}
}

/*
	Checks to see if a given word is in the blacklist (the list of words deemed unimportant)
*/
function inBlacklist($word) {
	//TODO: empty strings shouldn't be being picked up to begin with
	if($word = "") {
		return true;
	}

	//Gets a cursor to the results of a query for the word
	$cursor = queryCollection("blacklist-words", array('word' => $word));

	return $cursor->count() > 0;
}

/*
	Given  media_id, returns the text to be displayed with mark-up (special words have a link)
*/
function howiknowit($media_id) {
	//Get a list of words for this document
	$words = array();
	$cursor = queryCollection();
	foreach($cursor as $doc) {
		array_push($words, $doc);
{'fraction_non_alphanumeric': 0.13920609026644915, 'fraction_numerical': 0.002582925502990756, 'mean_word_length': 3.612539184952978, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2284110', 'n_tokens_mistral': 2510, 'n_tokens_neox': 2270, 'n_words': 815}