<commit_before><?php
	set_time_limit(0);
	//connectDB library
	chdir("/var/www/html/beautyUniversity/model/libs");
	require 'connectDB.php';
	//curl helper function library
	require 'LIB_http.php';
	
	class storeData {
		//get Fan's page
		public function getPage($url, $target) {
			$webPage = http_get($url, $target);
			$webPage = $webPage["FILE"];
			return json_decode($webPage, true);
		}
	}
	
	$storeData = new storeData();
	$token = "<KEY>";
	$counter = 0;
	//get fan's page
	$page_arr = array(
		"477809915685496",
		"476194535860239"
	);

	$connection = new connectDB();
	$conn = $connection -> initialDB();
	while($counter < count($page_arr)) {
		$url = "https://graph.facebook.com/" . $page_arr[$counter] . "/feed?access_token=" . $token;
		$data = array();
		$page = $storeData -> getPage($url, "");
	
		$count = count($page["data"]);
		while($count !== 0) {
			for($count_data=0;$count_data<$count;$count_data++) {
				if(isset($page["data"][$count_data]["message"]))
					$message = $page["data"][$count_data]["message"];
				else
					$message = "";
				if(isset($page["data"][$count_data]["object_id"]))
					$object_id = $page["data"][$count_data]["object_id"];
				else
					$object_id = "";
				if(isset($page["data"][$count_data]["created_time"]))
					$created_time = $page["data"][$count_data]["created_time"];
				else
					$created_time = "";
			
				$data["message"] = str_replace("☺ ", "", trim($message));
				$data["message"] = str_replace("\n", "", $data["message"]);
				$data["object_id"] = $object_id;
				$data["created_time"] = $created_time;
			
				if($page_arr[$counter] === "477809915685496")
					$sql = "INSERT INTO beauty_info(message,obj_id,created_time) VALUES(:message,:object_id,:created_time)";
				else
					$sql = "INSERT INTO beauty_FJU(message,obj_id,created_time) VALUES(:message,:object_id,:created_time)";

				$result = $connection -> processData($conn, $sql, $data, "insert-record");
			
				switch($result) {
					case "insert-fail":
						file_put_contents("log/error.log", "insert-fail,".$message.",".$object_id."\r\n", FILE_APPEND);
						break;
					case "duplicate-entry":
						file_put_contents("log/error.log", "duplicate-entry,".$message.",".$object_id."\r\n", FILE_APPEND);
						break;
				}
			}
		
			$page = $storeData -> getPage($page["paging"]["next"], "");
			$count = count($page["data"]);
		}
		
		$counter += 1;
	}
	
	$connection -> connectClose();
?><commit_msg>modify $conn db to null<commit_after><?php
	set_time_limit(0);
	//connectDB library
	chdir("/var/www/html/beautyUniversity/model/libs");
	require 'connectDB.php';
	//curl helper function library
	require 'LIB_http.php';
	
	class storeData {
		//get Fan's page
		public function getPage($url, $target) {
			$webPage = http_get($url, $target);
			$webPage = $webPage["FILE"];
			return json_decode($webPage, true);
		}
	}
	
	$storeData = new storeData();
	$token = "<KEY>";
	$counter = 0;
	//get fan's page
	$page_arr = array(
		"477809915685496",
		"476194535860239"
	);

	$connection = new connectDB();
	$conn = $connection -> initialDB();
	while($counter < count($page_arr)) {
		$url = "https://graph.facebook.com/" . $page_arr[$counter] . "/feed?access_token=" . $token;
		$data = array();
		$page = $storeData -> getPage($url, "");
	
		$count = count($page["data"]);
		while($count !== 0) {
			for($count_data=0;$count_data<$count;$count_data++) {
				if(isset($page["data"][$count_data]["message"]))
					$message = $page["data"][$count_data]["message"];
				else
					$message = "";
				if(isset($page["data"][$count_data]["object_id"]))
					$object_id = $page["data"][$count_data]["object_id"];
				else
					$object_id = "";
				if(isset($page["data"][$count_data]["created_time"]))
					$created_time = $page["data"][$count_data]["created_time"];
				else
					$created_time = "";
			
				$data["message"] = str_replace("☺ ", "", trim($message));
				$data["message"] = str_replace("\n", "", $data["message"]);
				$data["object_id"] = $object_id;
				$data["created_time"] = $created_time;
			
				if($page_arr[$counter] === "477809915685496")
					$sql = "INSERT INTO beauty_info(message,obj_id,created_time) VALUES(:message,:object_id,:created_time)";
				else
					$sql = "INSERT INTO beauty_FJU(message,obj_id,created_time) VALUES(:message,:object_id,:created_time)";

				$result = $conn -> processData($conn, $sql, $data, "insert-record");
			
				switch($result) {
					case "insert-fail":
						file_put_contents("log/error.log", "insert-fail,".$message.",".$object_id."\r\n", FILE_APPEND);
						break;
					case "duplicate-entry":
						file_put_contents("log/error.log", "duplicate-entry,".$message.",".$object_id."\r\n", FILE_APPEND);
						break;
				}
			}
		
			$page = $storeData -> getPage($page["paging"]["next"], "");
			$count = count($page["data"]);
		}
		
		$counter += 1;
	}
	
	$conn = null;
?>
{'fraction_non_alphanumeric': 0.21327306348880215, 'fraction_numerical': 0.02054653790836244, 'mean_word_length': 4.145877378435518, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '474148', 'n_tokens_mistral': 2116, 'n_tokens_neox': 1745, 'n_words': 270}