<commit_before><?php

/**
 * Clope Clustering

 * @link https://drive.google.com/file/d/1---LtHj1jLHCOcCUZChMRZQVirmt9nR6F8Zmw6bgNBBNyiBB6zmWsNuGIOfF Clope Clustering Algorithm

 * @author maxleonov <<EMAIL>>
 */

/**
 * 
 */
class Clope extends AppModel {

	/**
	 * {@inheritdoc}
	 *
	 * @var bool
	 */
	public $useTable = false;

	/**
	 * @var bool
	 */
    private $clustering_changed = false;

	/**
	 * Constructor
	 */
	public function __construct() {
		$this->ClopeTransaction = ClassRegistry::init('ClopeClustering.ClopeTransaction');
		$this->ClopeCluster = ClassRegistry::init('ClopeClustering.ClopeCluster');
		$this->ClopeAttribute = ClassRegistry::init('ClopeClustering.ClopeAttribute');
	}

	/**
	 * Clusterize Transactions
	 *
	 * @param array $transactions array((int)ID => array((int|string)attr1, attr2, ...), ...)
	 * @param array $params array('repulsion' => (float)$number)
	 *
	 * @return array
	 */
	public function clusterize($transactions, $params) {
		$this->repulsion = $params['repulsion'];

		// Add transactions with attributes
		foreach($transactions as $id=>&$transaction) {
			$data = array(
				'ClopeTransaction' => array(
					'custom_id' => $id
				), 
				'ClopeAttribute' => Hash::map($transaction, '{n}', create_function('$attr', 'return array(\'attribute\'=>$attr);'))
			);
			$this->ClopeTransaction->saveAssociated($data, array('deep' => true));
		}

		// Clustering
		$this->_clusterize();

		// Result
		$groupedByCluster = array();
		foreach ($transactions as $id=>&$transaction) {
			// ->clusterID()
			// res[ clusterID ] [] = trans
			$groupedByCluster[$this->ClopeTransaction->clusterID($id)][] = $transaction;
			unset($transactions[$id]);
			// del( trans )
		}

		return $groupedByCluster;
	}

	/**
	 * 
	 */
	private function _clusterize() {
		$this->setClusteringIncomplete();
		while (!$this->ifClusteringComplete()) {
			$this->ClopeTransaction->reset_pointer();
			$this->setClusteringComplete();
			while ($transaction = $this->ClopeTransaction->getNext()) {
				$bestClusterID = $this->bestClusterID($transaction, $this->repulsion);<commit_msg>Clear data storage before clustering starts<commit_after><?php

/**
 * Clope Clustering

 * @link https://drive.google.com/file/d/1---LtHj1jLHCOcCUZChMRZQVirmt9nR6F8Zmw6bgNBBNyiBB6zmWsNuGIOfF Clope Clustering Algorithm

 * @author maxleonov <<EMAIL>>
 */

/**
 * 
 */
class Clope extends AppModel {

	/**
	 * {@inheritdoc}
	 *
	 * @var bool
	 */
	public $useTable = false;

	/**
	 * @var bool
	 */
    private $clustering_changed = false;

	/**
	 * Constructor
	 */
	public function __construct() {
		$this->ClopeTransaction = ClassRegistry::init('ClopeClustering.ClopeTransaction');
		$this->ClopeCluster = ClassRegistry::init('ClopeClustering.ClopeCluster');
		$this->ClopeAttribute = ClassRegistry::init('ClopeClustering.ClopeAttribute');
	}

	/**
	 * Clusterize Transactions
	 *
	 * @param array $transactions array((int)ID => array((int|string)attr1, attr2, ...), ...)
	 * @param array $params array('repulsion' => (float)$number)
	 *
	 * @return array
	 */
	public function clusterize($transactions, $params) {
		$this->repulsion = $params['repulsion'];

		// Empty data storage
		$this->ClopeCluster->deleteAll('1=1');
		$this->ClopeTransaction->deleteAll('1=1');
		$this->ClopeAttribute->deleteAll('1=1');

		// Add transactions with attributes
		foreach($transactions as $id=>&$transaction) {
			$data = array(
				'ClopeTransaction' => array(
					'custom_id' => $id
				), 
				'ClopeAttribute' => Hash::map($transaction, '{n}', create_function('$attr', 'return array(\'attribute\'=>$attr);'))
			);
			$this->ClopeTransaction->saveAssociated($data, array('deep' => true));
		}

		// Clustering
		$this->_clusterize();

		// Result
		$groupedByCluster = array();
		foreach ($transactions as $id=>&$transaction) {
			$groupedByCluster[$this->ClopeTransaction->clusterID($id)][] = $transaction;
			unset($transactions[$id]);
		}

		return $groupedByCluster;
	}

	/**
	 * 
	 */
	private function _clusterize() {
		$this->setClusteringIncomplete();
		while (!$this->ifClusteringComplete()) {
			$this->ClopeTransaction->reset_pointer();
			$this->setClusteringComplete();
			while ($transaction = $this->ClopeTransaction->getNext()) {
				$bestClusterID = $this->bestClusterID($transaction, $this->repulsion);
{'fraction_non_alphanumeric': 0.17912520249942143, 'fraction_numerical': 0.0055542698449433, 'mean_word_length': 4.290085679314566, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 68, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1624917', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1392, 'n_words': 230}