<commit_before><?php
namespace Slimpd\Modules\importer;
/* Copyright (C) 2016 othmar52 <<EMAIL>>
 *
 * This file is part of sliMpd - a php based mpd web client
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
class Migrator extends \Slimpd\Modules\importer\AbstractImporter {
	private $dbTstampsTrack = array();
	private $dbTstampsAlbum = array();

	private $skipAlbum = TRUE;

	private $batchSize = 5000;

	public $migratedAlbums = 0;

	/* start with empty database? - then we can maximise speed by using the "Batcher" */
	public $useBatcher = FALSE;

	// only for development purposes
	public function tempResetMigrationPhase() {
		cliLog('truncating all tables with migrated data', 1, 'red', TRUE);
		foreach(\Slimpd\Modules\importer\DatabaseStuff::getInitialDatabaseQueries() as $query) {
			\Slim\Slim::getInstance()->db->query($query);
		}
		$this->useBatcher = TRUE;
	}

	/** 
	 * @return array 'directoryHash' => 'most-recent-timestamp' 
	 */
	public static function getMigratedAlbumTimstamps() {
		return self::getMigratedTimstamps('album');
	}

	public static function getMigratedTrackTimstamps() {
		return self::getMigratedTimstamps('track');
	}

	public static function getMigratedTimstamps($tablename) {
		$timestampsMysql = array();

		$query = "SELECT relPathHash,filemtime FROM " . az09($tablename);
		$result = \Slim\Slim::getInstance()->db->query($query);
		while($record = $result->fetch_assoc()) {
			$timestampsMysql[ $record['relPathHash'] ] = $record['filemtime'];
		}
		return $timestampsMysql;
	}

	/**
	 * TODO: how to deal with album-orphans?
	 * TODO: reset album:lastScan for each migrated album 
	 */
	public function run($resetMigrationPhase = FALSE) {

		# only for development
		# TODO: make this step optional controllable via gui
		if($resetMigrationPhase === TRUE) {
			$this->tempResetMigrationPhase();
		}

		$this->jobPhase = 4;
		$this->beginJob(array(
			'msg' => "migrateRawtagdataTable"
		), __FUNCTION__);
		$app = \Slim\Slim::getInstance();
		$this->migratorConfig = \Slimpd\Modules\albummigrator\AlbumMigrator::parseConfig();
		$this->dbTstampsAlbum = self::getMigratedAlbumTimstamps();
		$this->dbTstampsTrack = self::getMigratedTrackTimstamps();
		$this->skipAlbum = TRUE;

		$this->migratedAlbums = 0;

		$query = "SELECT count(uid) AS itemsTotal FROM rawtagdata";
		$this->itemsTotal = (int) $app->db->query($query)->fetch_assoc()['itemsTotal'];
		$this->checkBatch();
		$app->batcher->finishAll();

		$this->finishJob(array(
			'msg' => 'migrated ' . $this->itemsChecked . ' files',
			'migratedAlbums' => $this->migratedAlbums
		), __FUNCTION__);
	}

	/**
	 * limiting SELECT query to $this->batchSize to avoid retrieving millions of records within a single query
	 * so this method calls itself recursively
	 *
	 */
	private function checkBatch() {
		$query = "SELECT * FROM rawtagdata ORDER BY relDirPathHash LIMIT " . $this->itemsChecked . "," . $this->batchSize;
		$result = \Slim\Slim::getInstance()->db->query($query);
		$counter = 0;
		while($record = $result->fetch_assoc()) {
			$this->itemsChecked++;
			$counter++;

			if($counter % $this->batchSize === 0) {
				cliLog(" shortening itemsChecked by " . $this->prevAlb->getTrackCount(), 10, "cyan");
				$this->itemsChecked -= ($this->prevAlb->getTrackCount()+1);
				cliLog(" batch complete...",9 , "cyan");
				// recursion
				return $this->checkBatch();
			}

			if($counter === 1) {
				$this->newPrevAlb($record);
				$this->skipAlbum = TRUE;
			}

			// directory had changed within the loop
			if($record['relDirPathHash'] !== $this->prevAlb->getRelDirPathHash() ) {
				$this->mayMigrate();
				cliLog('resetting previousAlbum', 10);
				$this->newPrevAlb($record);
				cliLog('  adding hash of dir '. $record['relDirPath'] .' to previousAlbum', 10);
				$this->skipAlbum = TRUE;
			}

			$this->updateJob(array(
				'currentItem' => $record['relPath'],
				'migratedAlbums' => $this->migratedAlbums
			));
			$this->checkTrackSkip($record);
			$this->prevAlb->addTrack($record);
			cliLog('adding track to previousAlbum: ' . $record['relPath'], 10);

			cliLog("#" . $this->itemsChecked . " " . $record['relPath'],2);

			// dont forget to check the last directory
			if($this->itemsChecked === $this->itemsTotal && $this->itemsTotal > 1) {
				$this->mayMigrate();
			}
		}
	}

	private function checkAlbumSkip() {	
		// decide if we have to process album or if we can skip it
		if(isset($this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ]) === FALSE) {
			cliLog('album does NOT exist in migrated data. migrating: ' . $this->prevAlb->getRelDirPath(), 5);
			$this->skipAlbum = FALSE;
			return;
		}
		if($this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ] < $this->prevAlb->getDirectoryMtime()) {
			cliLog('dir-timestamp raw is more recent than migrated. migrating: ' . $this->prevAlb->getRelDirPath(), 5);
			cliLog('dir-timestamps mig: '. $this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ] .' raw: ' . $this->prevAlb->getDirectoryMtime(), 10, 'yellow');
			$this->skipAlbum = FALSE;
			return;
		}
	}

	private function checkTrackSkip($record) {
				// decide if we have to process album based on single-track-change or if we can skip it
		if(isset($this->dbTstampsTrack[ $record['relPathHash'] ]) === FALSE) {
			cliLog('track does NOT exist in migrated data. migrating: ' . $record['relPath'], 5);
			$this->skipAlbum = FALSE;
			return;
		} 
		if($this->dbTstampsTrack[ $record['relPathHash'] ] < $record['filemtime']) {
			cliLog('track-imestamp raw is more recent than migrated. migrating: ' . $record['relPath'], 5);
			$this->skipAlbum = FALSE;
		}
	}

	private function mayMigrate() {
		$this->checkAlbumSkip();
		if($this->skipAlbum === TRUE) {
			cliLog('skipping migration for: ' . $this->prevAlb->getRelDirPath(), 5);
			return;
		}
		$this->prevAlb->run();
		$this->itemsProcessed += $this->prevAlb->getTrackCount();
		$this->migratedAlbums++;
	}

	private function newPrevAlb($record) {
		$this->prevAlb = new \Slimpd\Modules\albummigrator\AlbumMigrator();
		$this->prevAlb->conf = $this->migratorConfig;
		$this->prevAlb->useBatcher = $this->useBatcher;
		$this->prevAlb->setRelDirPathHash($record['relDirPathHash']);
		$this->prevAlb->setRelDirPath($record['relDirPath']);
		$this->prevAlb->setDirectoryMtime($record['directoryMtime']);
	}
}
<commit_msg>increased batch size in importer migrator phase<commit_after><?php
namespace Slimpd\Modules\importer;
/* Copyright (C) 2016 othmar52 <<EMAIL>>
 *
 * This file is part of sliMpd - a php based mpd web client
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
class Migrator extends \Slimpd\Modules\importer\AbstractImporter {
	private $dbTstampsTrack = array();
	private $dbTstampsAlbum = array();

	private $skipAlbum = TRUE;

	private $batchSize = 1500000; // process only 1.5 million tracks per batch

	public $migratedAlbums = 0;

	/* start with empty database? - then we can maximise speed by using the "Batcher" */
	public $useBatcher = FALSE;

	// only for development purposes
	public function tempResetMigrationPhase() {
		cliLog('truncating all tables with migrated data', 1, 'red', TRUE);
		foreach(\Slimpd\Modules\importer\DatabaseStuff::getInitialDatabaseQueries() as $query) {
			\Slim\Slim::getInstance()->db->query($query);
		}
		$this->useBatcher = TRUE;
	}

	/** 
	 * @return array 'directoryHash' => 'most-recent-timestamp' 
	 */
	public static function getMigratedAlbumTimstamps() {
		return self::getMigratedTimstamps('album');
	}

	public static function getMigratedTrackTimstamps() {
		return self::getMigratedTimstamps('track');
	}

	public static function getMigratedTimstamps($tablename) {
		$timestampsMysql = array();

		$query = "SELECT relPathHash,filemtime FROM " . az09($tablename);
		$result = \Slim\Slim::getInstance()->db->query($query);
		while($record = $result->fetch_assoc()) {
			$timestampsMysql[ $record['relPathHash'] ] = $record['filemtime'];
		}
		return $timestampsMysql;
	}

	/**
	 * TODO: how to deal with album-orphans?
	 * TODO: reset album:lastScan for each migrated album 
	 */
	public function run($resetMigrationPhase = FALSE) {

		# only for development
		# TODO: make this step optional controllable via gui
		if($resetMigrationPhase === TRUE) {
			$this->tempResetMigrationPhase();
		}

		$this->jobPhase = 4;
		$this->beginJob(array(
			'msg' => "migrateRawtagdataTable"
		), __FUNCTION__);
		$app = \Slim\Slim::getInstance();
		$this->migratorConfig = \Slimpd\Modules\albummigrator\AlbumMigrator::parseConfig();
		$this->dbTstampsAlbum = self::getMigratedAlbumTimstamps();
		$this->dbTstampsTrack = self::getMigratedTrackTimstamps();
		$this->skipAlbum = TRUE;

		$this->migratedAlbums = 0;

		$query = "SELECT count(uid) AS itemsTotal FROM rawtagdata";
		$this->itemsTotal = (int) $app->db->query($query)->fetch_assoc()['itemsTotal'];
		$this->checkBatch();
		$app->batcher->finishAll();

		$this->finishJob(array(
			'msg' => 'migrated ' . $this->itemsChecked . ' files',
			'migratedAlbums' => $this->migratedAlbums
		), __FUNCTION__);
	}

	/**
	 * limiting SELECT query to $this->batchSize to avoid retrieving millions of records within a single query
	 * so this method calls itself recursively
	 *
	 */
	private function checkBatch() {
		$query = "SELECT * FROM rawtagdata ORDER BY relDirPathHash LIMIT " . $this->itemsChecked . "," . $this->batchSize;
		$result = \Slim\Slim::getInstance()->db->query($query);
		$counter = 0;
		while($record = $result->fetch_assoc()) {
			$this->itemsChecked++;
			$counter++;

			if($counter % $this->batchSize === 0) {
				cliLog(" shortening itemsChecked by " . $this->prevAlb->getTrackCount(), 10, "cyan");
				$this->itemsChecked -= ($this->prevAlb->getTrackCount()+1);
				cliLog(" batch complete...",9 , "cyan");
				// recursion
				return $this->checkBatch();
			}

			if($counter === 1) {
				$this->newPrevAlb($record);
				$this->skipAlbum = TRUE;
			}

			// directory had changed within the loop
			if($record['relDirPathHash'] !== $this->prevAlb->getRelDirPathHash() ) {
				$this->mayMigrate();
				cliLog('resetting previousAlbum', 10);
				$this->newPrevAlb($record);
				cliLog('  adding hash of dir '. $record['relDirPath'] .' to previousAlbum', 10);
				$this->skipAlbum = TRUE;
			}

			$this->updateJob(array(
				'currentItem' => $record['relPath'],
				'migratedAlbums' => $this->migratedAlbums
			));
			$this->checkTrackSkip($record);
			$this->prevAlb->addTrack($record);
			cliLog('adding track to previousAlbum: ' . $record['relPath'], 10);

			cliLog("#" . $this->itemsChecked . " " . $record['relPath'],2);

			// dont forget to check the last directory
			if($this->itemsChecked === $this->itemsTotal && $this->itemsTotal > 1) {
				$this->mayMigrate();
			}
		}
	}

	private function checkAlbumSkip() {	
		// decide if we have to process album or if we can skip it
		if(isset($this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ]) === FALSE) {
			cliLog('album does NOT exist in migrated data. migrating: ' . $this->prevAlb->getRelDirPath(), 5);
			$this->skipAlbum = FALSE;
			return;
		}
		if($this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ] < $this->prevAlb->getDirectoryMtime()) {
			cliLog('dir-timestamp raw is more recent than migrated. migrating: ' . $this->prevAlb->getRelDirPath(), 5);
			cliLog('dir-timestamps mig: '. $this->dbTstampsAlbum[ $this->prevAlb->getRelDirPathHash() ] .' raw: ' . $this->prevAlb->getDirectoryMtime(), 10, 'yellow');
			$this->skipAlbum = FALSE;
			return;
		}
	}

	private function checkTrackSkip($record) {
				// decide if we have to process album based on single-track-change or if we can skip it
		if(isset($this->dbTstampsTrack[ $record['relPathHash'] ]) === FALSE) {
			cliLog('track does NOT exist in migrated data. migrating: ' . $record['relPath'], 5);
			$this->skipAlbum = FALSE;
			return;
		} 
		if($this->dbTstampsTrack[ $record['relPathHash'] ] < $record['filemtime']) {
			cliLog('track-imestamp raw is more recent than migrated. migrating: ' . $record['relPath'], 5);
			$this->skipAlbum = FALSE;
		}
	}

	private function mayMigrate() {
		$this->checkAlbumSkip();
		if($this->skipAlbum === TRUE) {
			cliLog('skipping migration for: ' . $this->prevAlb->getRelDirPath(), 5);
			return;
		}
		$this->prevAlb->run();
		$this->itemsProcessed += $this->prevAlb->getTrackCount();
		$this->migratedAlbums++;
	}

	private function newPrevAlb($record) {
		$this->prevAlb = new \Slimpd\Modules\albummigrator\AlbumMigrator();
		$this->prevAlb->conf = $this->migratorConfig;
		$this->prevAlb->useBatcher = $this->useBatcher;
		$this->prevAlb->setRelDirPathHash($record['relDirPathHash']);
		$this->prevAlb->setRelDirPath($record['relDirPath']);
		$this->prevAlb->setDirectoryMtime($record['directoryMtime']);
	}
}

{'fraction_non_alphanumeric': 0.15725114416475972, 'fraction_numerical': 0.005935354691075515, 'mean_word_length': 4.257518796992481, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 225, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867280', 'n_tokens_mistral': 5083, 'n_tokens_neox': 4514, 'n_words': 1087}