<commit_before><?php

namespace OCA\OwnNote\Lib;

\OCP\User::checkLoggedIn();
\OCP\App::checkAppEnabled('ownnote');

use DateTime;
use DOMDocument;

class Backend {

	public function startsWith($haystack, $needle) {
		return $needle === "" || strripos($haystack, $needle, -strlen($haystack)) !== FALSE;
	}

	public function endsWith($string, $test) {
		$strlen = strlen($string);
		$testlen = strlen($test);
		if ($testlen > $strlen) return false;
		return substr_compare($string, $test, $strlen - $testlen, $testlen, true) === 0;
	}

	public function getAnnouncement() {
		$ret = "";
		$url = 'https://raw.githubusercontent.com/Fmstrat/announcements/master/ownnote/announcement.html';
		$ch = curl_init();
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curl1, CURLOPT_FRESH_CONNECT, TRUE);
		//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
		//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
		curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.0; S60/3.0 NokiaN73-1/2.0(2.0617.0.0.7) Profile/MIDP-2.0 Configuration/CLDC-1.1)");
		$result = curl_exec($ch);
		if ($result === FALSE) {
			die('Curl failed: ' . curl_error($ch));
		} else {
			$ret = $result;
		}
		curl_close($ch);
		return $ret;
	}

	public function checkEvernote($folder, $file) {
		$html = "";
		if ($html = \OC\Files\Filesystem::file_get_contents($folder."/".$file)) {
			$DOM = new DOMDocument;
			$DOM->loadHTML($html);
			$items = $DOM->getElementsByTagName('meta');
			$isEvernote = false;
			for ($i = 0; $i < $items->length; $i++) {
				$item = $items->item($i);
				if ($item->hasAttributes()) {
					$attrs = $item->attributes;
					foreach ($attrs as $a => $attr) {
						if ($attr->name == "name") {
							if ($attr->value == "exporter-version" || $attr->value == "Generator") {
								$isEvernote = true;
								continue;
							}
						}
					}
				}
			}
			if ($isEvernote) {
				$items = $DOM->getElementsByTagName('img');
				$isEvernote = false;
				for ($i = 0; $i < $items->length; $i++) {
					$item = $items->item($i);
					if ($item->hasAttributes()) {
						$attrs = $item->attributes;
						foreach ($attrs as $a => $attr) {
							if ($attr->name == "src") {
								$url = $attr->value;
								if (!$this->startsWith($url, "http") && !$this->startsWith($url, "/") && !$this->startsWith($url,"data")) {
									if ($data = \OC\Files\Filesystem::file_get_contents($folder."/".$url)) {
										$type = pathinfo($url, PATHINFO_EXTENSION);
										$base64 = "data:image/".$type.";base64,".base64_encode($data);
										$html = str_replace($url, $base64, $html);
									}
								}
							}
						}
					}
				}
				\OC\Files\Filesystem::file_put_contents($folder."/".$file, $html);
			}
		}
	}

	public function getTimeString($filetime, $now, $l) {
		$difftime = $filetime->diff($now);
		$years = $difftime->y;
		$months = $difftime->m;
		$days = $difftime->d;
		$hours = $difftime->h;
		$minutes = $difftime->i;
		$seconds = $difftime->s;
		$timestring = "";
		if ($timestring == "" && $years == 1) $timestring = str_replace('#', $years, $l->t("# year ago"));
		if ($timestring == "" && $years > 0) $timestring = str_replace('#', $years, $l->t("# years ago"));
		if ($timestring == "" && $months == 1) $timestring = str_replace('#', $months, $l->t("# month ago"));
		if ($timestring == "" && $months > 0) $timestring = str_replace('#', $months, $l->t("# months ago"));
		if ($timestring == "" && $days == 1) $timestring = str_replace('#', $days, $l->t("# day ago"));
		if ($timestring == "" && $days > 0) $timestring = str_replace('#', $days, $l->t("# days ago"));
		if ($timestring == "" && $hours == 1) $timestring = str_replace('#', $hours, $l->t("# hour ago"));
		if ($timestring == "" && $hours > 0) $timestring = str_replace('#', $hours, $l->t("# hours ago"));
		if ($timestring == "" && $minutes == 1) $timestring = str_replace('#', $minutes, $l->t("# minute ago"));
		if ($timestring == "" && $minutes > 0) $timestring = str_replace('#', $minutes, $l->t("# minutes ago"));
		if ($timestring == "" && $seconds == 1) $timestring = str_replace('#', $seconds, $l->t("# second ago"));
		if ($timestring == "" && $seconds > 0) $timestring = str_replace('#', $seconds, $l->t("# seconds ago"));
		return $timestring;
	}

	public function splitContent($str) {
		$maxlength = 2621440; // 5 Megs (2 bytes per character)
		$count = 0;
		$strarray = array();
		while (true) { 
			if (strlen($str) <= $maxlength) { 
				$strarray[$count++] = $str;
				return $strarray;
			} else {
				$strarray[$count++] = substr($str, 0, $maxlength);
				$str = substr($str, $maxlength);
			}
		}
	}

	public function getListing($FOLDER, $showdel) {
		// Get the listing from the database
		$requery = false;
		$uid = \OCP\User::getUser();
		$query = \OCP\DB::prepare("SELECT id, name, grouping, mtime, deleted FROM *PREFIX*ownnote WHERE uid=? ORDER BY name");
		$results = $query->execute(Array($uid))->fetchAll();
		$results2 = $results;
		if ($results)
			foreach($results as $result)
				foreach($results2 as $result2)
					if ($result['id'] != $result2['id'] && $result['name'] == $result2['name'] && $result['grouping'] == $result2['grouping'] && $result['mtime'] == $result2['mtime']) {
						// We have a duplicate that should not exist. Need to remove the offending record first
						$delid = $result['id'];
						if ($result['id'] > $result2['id'])
							$delid = $result2['id'];
						$delquery = \OCP\DB::prepare("DELETE FROM *PREFIX*ownnote WHERE id=?");
						$delquery->execute(Array($delid));
						$requery = true;
					}
		if ($requery) {
			$query = \OCP\DB::prepare("SELECT id, name, grouping, mtime, deleted FROM *PREFIX*ownnote WHERE uid=? ORDER BY name");
			$results = $query->execute(Array($uid))->fetchAll();
			$requery = false;
		}
		// Tests to add a bunch of notes
		//$now = new DateTime();
		//for ($x = 0; $x < 199; $x++) {
			//saveNote('', "Test ".$x, '', '', $now->getTimestamp());
		//}
		$farray = array();
		if ($FOLDER != '') {
			// Create the folder if it doesn't exist
			if (!\OC\Files\Filesystem::is_dir($FOLDER)) {
				if (!\OC\Files\Filesystem::mkdir($FOLDER)) {
					echo "ERROR: Could not create ownNote directory.";
					exit;
				}
			}
			// Synchronize files to the database
			$filearr = array();<commit_msg>Fix duplicates issue<commit_after><?php

namespace OCA\OwnNote\Lib;

\OCP\User::checkLoggedIn();
\OCP\App::checkAppEnabled('ownnote');

use DateTime;
use DOMDocument;

class Backend {

	public function startsWith($haystack, $needle) {
		return $needle === "" || strripos($haystack, $needle, -strlen($haystack)) !== FALSE;
	}

	public function endsWith($string, $test) {
		$strlen = strlen($string);
		$testlen = strlen($test);
		if ($testlen > $strlen) return false;
		return substr_compare($string, $test, $strlen - $testlen, $testlen, true) === 0;
	}

	public function getAnnouncement() {
		$ret = "";
		$url = 'https://raw.githubusercontent.com/Fmstrat/announcements/master/ownnote/announcement.html';
		$ch = curl_init();
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curl1, CURLOPT_FRESH_CONNECT, TRUE);
		//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
		//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
		curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.0; S60/3.0 NokiaN73-1/2.0(2.0617.0.0.7) Profile/MIDP-2.0 Configuration/CLDC-1.1)");
		$result = curl_exec($ch);
		if ($result === FALSE) {
			die('Curl failed: ' . curl_error($ch));
		} else {
			$ret = $result;
		}
		curl_close($ch);
		return $ret;
	}

	public function checkEvernote($folder, $file) {
		$html = "";
		if ($html = \OC\Files\Filesystem::file_get_contents($folder."/".$file)) {
			$DOM = new DOMDocument;
			$DOM->loadHTML($html);
			$items = $DOM->getElementsByTagName('meta');
			$isEvernote = false;
			for ($i = 0; $i < $items->length; $i++) {
				$item = $items->item($i);
				if ($item->hasAttributes()) {
					$attrs = $item->attributes;
					foreach ($attrs as $a => $attr) {
						if ($attr->name == "name") {
							if ($attr->value == "exporter-version" || $attr->value == "Generator") {
								$isEvernote = true;
								continue;
							}
						}
					}
				}
			}
			if ($isEvernote) {
				$items = $DOM->getElementsByTagName('img');
				$isEvernote = false;
				for ($i = 0; $i < $items->length; $i++) {
					$item = $items->item($i);
					if ($item->hasAttributes()) {
						$attrs = $item->attributes;
						foreach ($attrs as $a => $attr) {
							if ($attr->name == "src") {
								$url = $attr->value;
								if (!$this->startsWith($url, "http") && !$this->startsWith($url, "/") && !$this->startsWith($url,"data")) {
									if ($data = \OC\Files\Filesystem::file_get_contents($folder."/".$url)) {
										$type = pathinfo($url, PATHINFO_EXTENSION);
										$base64 = "data:image/".$type.";base64,".base64_encode($data);
										$html = str_replace($url, $base64, $html);
									}
								}
							}
						}
					}
				}
				\OC\Files\Filesystem::file_put_contents($folder."/".$file, $html);
			}
		}
	}

	public function getTimeString($filetime, $now, $l) {
		$difftime = $filetime->diff($now);
		$years = $difftime->y;
		$months = $difftime->m;
		$days = $difftime->d;
		$hours = $difftime->h;
		$minutes = $difftime->i;
		$seconds = $difftime->s;
		$timestring = "";
		if ($timestring == "" && $years == 1) $timestring = str_replace('#', $years, $l->t("# year ago"));
		if ($timestring == "" && $years > 0) $timestring = str_replace('#', $years, $l->t("# years ago"));
		if ($timestring == "" && $months == 1) $timestring = str_replace('#', $months, $l->t("# month ago"));
		if ($timestring == "" && $months > 0) $timestring = str_replace('#', $months, $l->t("# months ago"));
		if ($timestring == "" && $days == 1) $timestring = str_replace('#', $days, $l->t("# day ago"));
		if ($timestring == "" && $days > 0) $timestring = str_replace('#', $days, $l->t("# days ago"));
		if ($timestring == "" && $hours == 1) $timestring = str_replace('#', $hours, $l->t("# hour ago"));
		if ($timestring == "" && $hours > 0) $timestring = str_replace('#', $hours, $l->t("# hours ago"));
		if ($timestring == "" && $minutes == 1) $timestring = str_replace('#', $minutes, $l->t("# minute ago"));
		if ($timestring == "" && $minutes > 0) $timestring = str_replace('#', $minutes, $l->t("# minutes ago"));
		if ($timestring == "" && $seconds == 1) $timestring = str_replace('#', $seconds, $l->t("# second ago"));
		if ($timestring == "" && $seconds > 0) $timestring = str_replace('#', $seconds, $l->t("# seconds ago"));
		return $timestring;
	}

	public function splitContent($str) {
		$maxlength = 2621440; // 5 Megs (2 bytes per character)
		$count = 0;
		$strarray = array();
		while (true) { 
			if (strlen($str) <= $maxlength) { 
				$strarray[$count++] = $str;
				return $strarray;
			} else {
				$strarray[$count++] = substr($str, 0, $maxlength);
				$str = substr($str, $maxlength);
			}
		}
	}

	public function getListing($FOLDER, $showdel) {
		// Get the listing from the database
		$requery = false;
		$uid = \OCP\User::getUser();
		$query = \OCP\DB::prepare("SELECT id, name, grouping, mtime, deleted FROM *PREFIX*ownnote WHERE uid=? ORDER BY name");
		$results = $query->execute(Array($uid))->fetchAll();
		$results2 = $results;
		if ($results)
			foreach($results as $result)
				foreach($results2 as $result2)
					if ($result['id'] != $result2['id'] && $result['name'] == $result2['name'] && $result['grouping'] == $result2['grouping']) {
						// We have a duplicate that should not exist. Need to remove the offending record first
						$delid = -1;
						if ($result['mtime'] == $result2['mtime']) {
							// If the mtime's match, delete the oldest ID.
							$delid = $result['id'];
							if ($result['id'] > $result2['id'])
								$delid = $result2['id'];
						} elseif ($result['mtime'] > $result2['mtime']) {
							// Again, delete the oldest
							$delid = $result2['id'];
						} elseif ($result['mtime'] < $result2['mtime']) {
							// The only thing left is if result is older
							$delid = $result['id'];
						}
						if ($delid != -1) {
							$delquery = \OCP\DB::prepare("DELETE FROM *PREFIX*ownnote WHERE id=?");
							$delquery->execute(Array($delid));
							$requery = true;
						}
					}
		if ($requery) {
			$query = \OCP\DB::prepare("SELECT id, name, grouping, mtime, deleted FROM *PREFIX*ownnote WHERE uid=? ORDER BY name");
			$results = $query->execute(Array($uid))->fetchAll();
			$requery = false;
		}
		// Tests to add a bunch of notes
		//$now = new DateTime();
		//for ($x = 0; $x < 199; $x++) {
			//saveNote('', "Test ".$x, '', '', $now->getTimestamp());
		//}
		$farray = array();
		if ($FOLDER != '') {
			// Create the folder if it doesn't exist
			if (!\OC\Files\Filesystem::is_dir($FOLDER)) {
				if (!\OC\Files\Filesystem::mkdir($FOLDER)) {
					echo "ERROR: Could not create ownNote directory.";
					exit;
				}
			}
			// Synchronize files to the database
			$filearr = array();
{'fraction_non_alphanumeric': 0.21903740036787248, 'fraction_numerical': 0.011725935009196812, 'mean_word_length': 3.447511929107021, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 112, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747558', 'n_tokens_mistral': 5654, 'n_tokens_neox': 4669, 'n_words': 1046}