<commit_before><?php

class dbfts_pgsql extends dbfts_abs {
	/*
	 * Constructs a query part to match textfields. Abstracted so we can use
	 * a database specific FTS engine if one is provided by the DBMS
	 */
	function createTextQuery($searchFields) {
		SpotTiming::start(__FUNCTION__);

		/*
		 * Initialize some basic values which are used as return values to
		 * make sure always return a valid set
		 */
		$filterValueSql = array();
		$additionalFields = array();
		$sortFields = array();

		foreach($searchFields as $searchItem) {
			$searchValue = trim($searchItem['value']);
			$field = $searchItem['fieldname'];

			/*
			 * if we get multiple textsearches, we sort them per order
			 * in the system
			 */
			$tmpSortCounter = count($additionalFields);
			
			# Prepare the to_tsvector and to_tsquery strings
			$ts_vector = "to_tsvector('Dutch', " . $field . ")";
			$ts_query = "plainto_tsquery('" . $this->_db->safe(strtolower($searchValue)) . "')";
			
			$filterValueSql[] = " " . $ts_vector . " @@ " . $ts_query;
			$additionalFields[] = " ts_rank(" . $ts_vector . ", " . $ts_query . ") AS searchrelevancy" . $tmpSortCounter;
			$sortFields[] = array('field' => 'searchrelevancy' . $tmpSortCounter,
								  'direction' => 'DESC',
								  'autoadded' => true,
								  'friendlyname' => null);
		} # foreach

		SpotTiming::stop(__FUNCTION__, array($filterValueSql,$additionalFields,$sortFields));
		<commit_msg>Make sure we stem for the query using the same language as the index created<commit_after><?php

class dbfts_pgsql extends dbfts_abs {
	/*
	 * Constructs a query part to match textfields. Abstracted so we can use
	 * a database specific FTS engine if one is provided by the DBMS
	 */
	function createTextQuery($searchFields) {
		SpotTiming::start(__FUNCTION__);

		/*
		 * Initialize some basic values which are used as return values to
		 * make sure always return a valid set
		 */
		$filterValueSql = array();
		$additionalFields = array();
		$sortFields = array();

		foreach($searchFields as $searchItem) {
			$searchValue = trim($searchItem['value']);
			$field = $searchItem['fieldname'];

			/*
			 * if we get multiple textsearches, we sort them per order
			 * in the system
			 */
			$tmpSortCounter = count($additionalFields);
			
			# Prepare the to_tsvector and to_tsquery strings
			$ts_vector = "to_tsvector('Dutch', " . $field . ")";
			$ts_query = "plainto_tsquery('Dutch', '" . $this->_db->safe(strtolower($searchValue)) . "')";
			
			$filterValueSql[] = " " . $ts_vector . " @@ " . $ts_query;
			$additionalFields[] = " ts_rank(" . $ts_vector . ", " . $ts_query . ") AS searchrelevancy" . $tmpSortCounter;
			$sortFields[] = array('field' => 'searchrelevancy' . $tmpSortCounter,
								  'direction' => 'DESC',
								  'autoadded' => true,
								  'friendlyname' => null);
		} # foreach

		SpotTiming::stop(__FUNCTION__, array($filterValueSql,$additionalFields,$sortFields));
		
{'fraction_non_alphanumeric': 0.1393051771117166, 'fraction_numerical': 0.0, 'mean_word_length': 3.574766355140187, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1649965', 'n_tokens_mistral': 1058, 'n_tokens_neox': 879, 'n_words': 240}