<commit_before><?php
/**
 * Adds methods for limited kinds of faceting using the silverstripe ORM.
 * This is used by the default ShopSearchSimple adapter but also can
 * be added to other contexts (such as ProductCategory).
 *
 * TODO: Facet class + subclasses
 *
 * @author <NAME> <<EMAIL>>
 * @date 10.21.2013
 * @package shop_search
 * @subpackage helpers
 */
class FacetHelper extends Object
{
	/**
	 * @return FacetHelper
	 */
	public static function inst() {
		return Injector::inst()->get('FacetHelper');
	}


	/**
	 * @param SS_List $list
	 * @param array    $filters
	 * @param DataObject|string $sing - just a singleton object we can get information off of
	 * @return SS_List
	 */
	public function addFiltersToDataList(SS_List $list, array $filters, $sing=null) {
		if (!$sing) $sing = singleton($list->dataClass());
		if (is_string($sing)) $sing = singleton($sing);

		if (!empty($filters)) {
			foreach ($filters as $filterField => $filterVal) {
				$list = $list->filter($this->processFilterField($sing, $filterField, $filterVal));
			}
		}

		return $list;
	}


	/**
	 * @param DataObject $rec           This would normally just be a singleton but we don't want to have to create it over and over
	 * @param string     $filterField
	 * @param mixed      $filterVal
	 * @return array - returns the new filter added
	 */
	public function processFilterField($rec, $filterField, $filterVal) {
		// First check for VFI fields
		if ($rec->hasExtension('VirtualFieldIndex') && ($spec = $rec->getVFISpec($filterField))) {
			if ($spec['Type'] == VirtualFieldIndex::TYPE_LIST) {
				// Lists have to be handled a little differently
				$f = $rec->getVFIFieldName($filterField) . ':PartialMatch';
				if (is_array($filterVal)) {
					foreach ($filterVal as &$val) $val = '|' . $val . '|';
					return array($f => $filterVal);
				} else {
					return array($f => '|' . $filterVal . '|');
				}
			} else {
				// Simples are simple
				$filterField = $rec->getVFIFieldName($filterField);
			}
		}

		// Next check for regular db fields
		if ($rec->dbObject($filterField)) {
			// Is it a range value?
			if (preg_match('/^RANGE\~(.+)\~(.+)$/', $filterVal, $m)) {
				$filterField .= ':Between';
				$filterVal = array_slice($m, 1, 2);
			}

			return array($filterField => $filterVal);
		}

		return array();
	}


	/**
	 * Processes the facet spec and removes any shorthand (field => label).
	 * @param array $facetSpec
	 * @return array
	 */
	public function expandFacetSpec(array $facetSpec) {
		if (is_null($facetSpec)) return array();
		$facets = array();

		foreach ($facetSpec as $field => $label) {
			if (is_array($label)) {
				$facets[$field] = $label;
			} else {
				$facets[$field] = array('Label' => $label);
			}

			if (empty($facets[$field]['Source'])) $facets[$field]['Source'] = $field;
			if (empty($facets[$field]['Type']))  $facets[$field]['Type']  = ShopSearch::FACET_TYPE_LINK;

			if (empty($facets[$field]['Values'])) {
				$facets[$field]['Values'] = array();
			} else {
				$vals = $facets[$field]['Values'];
				if (is_string($vals)) $vals = eval('return ' . $vals . ';');
				$facets[$field]['Values'] = array();
				foreach ($vals as $val => $lbl) {
					$facets[$field]['Values'][$val] = new ArrayData(array(
						'Label'     => $lbl,
						'Value'     => $val,
						'Count'     => 0,
					));
				}
			}
		}

		return $facets;
	}


	/**
	 * This is super-slow. I'm assuming if you're using facets you
	 * probably also ought to be using Solr or something else. Or
	 * maybe you have unlimited time and can refactor this feature
	 * and submit a pull request...
	 *
	 * TODO: If this is going to be used for categories we're going
	 * to have to really clean it up and speed it up.
	 * Suggestion:
	 *  - option to turn off counts
	 *  - switch order of nested array so we don't go through results unless needed
	 *  - if not doing counts, min/max and link facets can be handled w/ queries
	 *  - separate that bit out into a new function
	 *
	 * Output - list of ArrayData in the format:
	 *   Label - name of the facet
	 *   Source - field name of the facet
	 *   Type - one of the ShopSearch::FACET_TYPE_XXXX constants
	 *   Values - SS_List of possible values for this facet
	 *
	 * @param SS_List $matches
	 * @param array $facetSpec
	 * @return ArrayList
	 */
	public function buildFacets(SS_List $matches, array $facetSpec) {
		$facets = $this->expandFacetSpec($facetSpec);
		if (empty($facets) || !$matches || !$matches->count()) return new ArrayList();

		// fill them in
		foreach ($matches as $rec) {
			foreach ($facets as $field => &$facet) {
				// If it's a range facet, set up the min/max
				if ($facet['Type'] == ShopSearch::FACET_TYPE_RANGE) {
					if (isset($facet['RangeMin'])) $facet['MinValue'] = $facet['RangeMin'];
					if (isset($facet['RangeMax'])) $facet['MaxValue'] = $facet['RangeMax'];
				}

				// If the field is accessible via normal methods, including
				// a user-defined getter, prefer that
				$fieldValue = $rec->relObject($field);<commit_msg>Added faster faceting method for building some types of manual facets<commit_after><?php
/**
 * Adds methods for limited kinds of faceting using the silverstripe ORM.
 * This is used by the default ShopSearchSimple adapter but also can
 * be added to other contexts (such as ProductCategory).
 *
 * TODO: Facet class + subclasses
 *
 * @author <NAME> <<EMAIL>>
 * @date 10.21.2013
 * @package shop_search
 * @subpackage helpers
 */
class FacetHelper extends Object
{
	/** @var bool - if this is turned on it will use an algorith that doesn't require traversing the dataset if possible */
	private static $faster_faceting = false;

	/**
	 * @return FacetHelper
	 */
	public static function inst() {
		return Injector::inst()->get('FacetHelper');
	}


	/**
	 * @param SS_List $list
	 * @param array    $filters
	 * @param DataObject|string $sing - just a singleton object we can get information off of
	 * @return SS_List
	 */
	public function addFiltersToDataList(SS_List $list, array $filters, $sing=null) {
		if (!$sing) $sing = singleton($list->dataClass());
		if (is_string($sing)) $sing = singleton($sing);

		if (!empty($filters)) {
			foreach ($filters as $filterField => $filterVal) {
				$list = $list->filter($this->processFilterField($sing, $filterField, $filterVal));
			}
		}

		return $list;
	}


	/**
	 * @param DataObject $rec           This would normally just be a singleton but we don't want to have to create it over and over
	 * @param string     $filterField
	 * @param mixed      $filterVal
	 * @return array - returns the new filter added
	 */
	public function processFilterField($rec, $filterField, $filterVal) {
		// First check for VFI fields
		if ($rec->hasExtension('VirtualFieldIndex') && ($spec = $rec->getVFISpec($filterField))) {
			if ($spec['Type'] == VirtualFieldIndex::TYPE_LIST) {
				// Lists have to be handled a little differently
				$f = $rec->getVFIFieldName($filterField) . ':PartialMatch';
				if (is_array($filterVal)) {
					foreach ($filterVal as &$val) $val = '|' . $val . '|';
					return array($f => $filterVal);
				} else {
					return array($f => '|' . $filterVal . '|');
				}
			} else {
				// Simples are simple
				$filterField = $rec->getVFIFieldName($filterField);
			}
		}

		// Next check for regular db fields
		if ($rec->dbObject($filterField)) {
			// Is it a range value?
			if (preg_match('/^RANGE\~(.+)\~(.+)$/', $filterVal, $m)) {
				$filterField .= ':Between';
				$filterVal = array_slice($m, 1, 2);
			}

			return array($filterField => $filterVal);
		}

		return array();
	}


	/**
	 * Processes the facet spec and removes any shorthand (field => label).
	 * @param array $facetSpec
	 * @return array
	 */
	public function expandFacetSpec(array $facetSpec) {
		if (is_null($facetSpec)) return array();
		$facets = array();

		foreach ($facetSpec as $field => $label) {
			if (is_array($label)) {
				$facets[$field] = $label;
			} else {
				$facets[$field] = array('Label' => $label);
			}

			if (empty($facets[$field]['Source'])) $facets[$field]['Source'] = $field;
			if (empty($facets[$field]['Type']))  $facets[$field]['Type']  = ShopSearch::FACET_TYPE_LINK;

			if (empty($facets[$field]['Values'])) {
				$facets[$field]['Values'] = array();
			} else {
				$vals = $facets[$field]['Values'];
				if (is_string($vals)) $vals = eval('return ' . $vals . ';');
				$facets[$field]['Values'] = array();
				foreach ($vals as $val => $lbl) {
					$facets[$field]['Values'][$val] = new ArrayData(array(
						'Label'     => $lbl,
						'Value'     => $val,
						'Count'     => 0,
					));
				}
			}
		}

		return $facets;
	}


	/**
	 * This is super-slow. I'm assuming if you're using facets you
	 * probably also ought to be using Solr or something else. Or
	 * maybe you have unlimited time and can refactor this feature
	 * and submit a pull request...
	 *
	 * TODO: If this is going to be used for categories we're going	 *
	 * to have to really clean it up and speed it up.
	 * Suggestion:
	 *  - option to turn off counts
	 *  - switch order of nested array so we don't go through results unless needed
	 *  - if not doing counts, min/max and link facets can be handled w/ queries
	 *  - separate that bit out into a new function
	 * NOTE: This is partially done with the "faster_faceting" config
	 * option but more could be done, particularly by covering link facets as well.
	 *
	 * Output - list of ArrayData in the format:
	 *   Label - name of the facet
	 *   Source - field name of the facet
	 *   Type - one of the ShopSearch::FACET_TYPE_XXXX constants
	 *   Values - SS_List of possible values for this facet
	 *
	 * @param SS_List $matches
	 * @param array $facetSpec
	 * @return ArrayList
	 */
	public function buildFacets(SS_List $matches, array $facetSpec) {
		$facets = $this->expandFacetSpec($facetSpec);
		if (empty($facets) || !$matches || !$matches->count()) return new ArrayList();
		$fasterMethod = (bool)$this->config()->faster_faceting;

		// fill them in
		foreach ($facets as $field => &$facet) {
			// NOTE: using this method range and checkbox facets don't get counts
			if ($fasterMethod && $facet['Type'] != ShopSearch::FACET_TYPE_LINK) {
				if ($facet['Type'] == ShopSearch::FACET_TYPE_RANGE) {
					if (isset($facet['RangeMin'])) $facet['MinValue'] = $facet['RangeMin'];
					if (isset($facet['RangeMax'])) $facet['MaxValue'] = $facet['RangeMax'];
				}

				continue;
			}

			foreach ($matches as $rec) {
				// If it's a range facet, set up the min/max
				if ($facet['Type'] == ShopSearch::FACET_TYPE_RANGE) {
					if (isset($facet['RangeMin'])) $facet['MinValue'] = $facet['RangeMin'];
					if (isset($facet['RangeMax'])) $facet['MaxValue'] = $facet['RangeMax'];
				}

				// If the field is accessible via normal methods, including
				// a user-defined getter, prefer that
				$fieldValue = $rec->relObject($field);
{'fraction_non_alphanumeric': 0.15344256259204714, 'fraction_numerical': 0.002025036818851252, 'mean_word_length': 3.3582029683112715, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3838627', 'n_tokens_mistral': 3935, 'n_tokens_neox': 3364, 'n_words': 1074}