<commit_before><?php

function discojuice_entity_cmp($a, $b) {
    if ($a['entityID'] == $b['entityID']) {
        return 0;
    }
    return ($a['entityID'] < $b['entityID']) ? -1 : 1;
}




class DiscoFeed {

	protected $feedId;
	protected $feedconfig;
	protected $metadata;

	protected $idplist;
	protected $store;
	
	function __construct($feedId, $feedconfig, $metadata) {

		$this->metadata = $metadata;
		$this->feedconfig = $feedconfig;
		$this->feedId = $feedId;
		
		$this->store = DiscoStore::getStore();

	}


	public function process() {

		$start = microtime(true);

		$this->idplist = array();
		$processedEntities = array();
		$i = 0;
		foreach($this->metadata AS $entityId => $metadataEntry) {

			// if (rand(0, 2) === 2) {
			// 	DiscoUtils::log('Skipping random entity: ' . $entityId); continue;
			// }

			DiscoUtils::log('Processing ' . $entityId);


			if (isset($metadataEntry["EntityAttributes"]) && isset($metadataEntry["EntityAttributes"]["http://macedir.org/entity-category"])) {
				if (in_array("http://refeds.org/category/hide-from-discovery", $metadataEntry["EntityAttributes"]["http://macedir.org/entity-category"])) {
					DiscoUtils::log('Skipping entry because refeds.org../hide-from-discovery ' . $entityId . "");
					print_r($metadataEntry); exit;
					continue;
				}
				
			}

			$entry = new FeedItem($entityId, $this->feedId, $metadataEntry, $this->feedconfig);
			$entry->process();
			$data = $entry->getJSON();
			
			// echo json_encode($data['disco'], JSON_PRETTY_PRINT) . "\n\n";

			$this->idplist[] = $entry;
			$processedEntities[] = $entityId;

			// if ($i++ > 5) break;
		}


		foreach($this->idplist AS $item) {<commit_msg>Remove debugging logging. Fixes andreassolberg/DiscoJuice#51<commit_after><?php

function discojuice_entity_cmp($a, $b) {
    if ($a['entityID'] == $b['entityID']) {
        return 0;
    }
    return ($a['entityID'] < $b['entityID']) ? -1 : 1;
}




class DiscoFeed {

	protected $feedId;
	protected $feedconfig;
	protected $metadata;

	protected $idplist;
	protected $store;
	
	function __construct($feedId, $feedconfig, $metadata) {

		$this->metadata = $metadata;
		$this->feedconfig = $feedconfig;
		$this->feedId = $feedId;
		
		$this->store = DiscoStore::getStore();

	}


	public function process() {

		$start = microtime(true);

		$this->idplist = array();
		$processedEntities = array();
		$i = 0;
		foreach($this->metadata AS $entityId => $metadataEntry) {

			// if (rand(0, 2) === 2) {
			// 	DiscoUtils::log('Skipping random entity: ' . $entityId); continue;
			// }

			DiscoUtils::log('Processing ' . $entityId);


			if (isset($metadataEntry["EntityAttributes"]) && isset($metadataEntry["EntityAttributes"]["http://macedir.org/entity-category"])) {
				if (in_array("http://refeds.org/category/hide-from-discovery", $metadataEntry["EntityAttributes"]["http://macedir.org/entity-category"])) {
					DiscoUtils::log('Skipping entry because refeds.org../hide-from-discovery ' . $entityId . "");
					continue;
				}
				
			}

			$entry = new FeedItem($entityId, $this->feedId, $metadataEntry, $this->feedconfig);
			$entry->process();
			$data = $entry->getJSON();
			
			// echo json_encode($data['disco'], JSON_PRETTY_PRINT) . "\n\n";

			$this->idplist[] = $entry;
			$processedEntities[] = $entityId;

			// if ($i++ > 5) break;
		}


		foreach($this->idplist AS $item) {
{'fraction_non_alphanumeric': 0.19222782557104717, 'fraction_numerical': 0.005339661821417977, 'mean_word_length': 3.76271186440678, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1500223', 'n_tokens_mistral': 1320, 'n_tokens_neox': 1144, 'n_words': 206}