<commit_before><?php
namespace wcf\system\package\plugin;
use wcf\data\IEditableCachedObject;
use wcf\data\language\category\LanguageCategory;
use wcf\data\language\category\LanguageCategoryAction;
use wcf\data\language\item\LanguageItemEditor;
use wcf\data\language\item\LanguageItemList;
use wcf\data\language\Language;
use wcf\data\language\LanguageEditor;
use wcf\system\database\util\PreparedStatementConditionBuilder;
use wcf\system\devtools\pip\DevtoolsPipEntryList;
use wcf\system\devtools\pip\IDevtoolsPipEntryList;
use wcf\system\devtools\pip\IGuiPackageInstallationPlugin;
use wcf\system\devtools\pip\TMultiXmlGuiPackageInstallationPlugin;
use wcf\system\exception\SystemException;
use wcf\system\form\builder\container\FormContainer;
use wcf\system\form\builder\field\dependency\ValueFormFieldDependency;
use wcf\system\form\builder\field\MultilineTextFormField;
use wcf\system\form\builder\field\RadioButtonFormField;
use wcf\system\form\builder\field\SingleSelectionFormField;
use wcf\system\form\builder\field\TextFormField;
use wcf\system\form\builder\field\validation\FormFieldValidationError;
use wcf\system\form\builder\field\validation\FormFieldValidator;
use wcf\system\form\builder\field\validation\FormFieldValidatorUtil;
use wcf\system\form\builder\IFormDocument;
use wcf\system\language\LanguageFactory;
use wcf\system\package\PackageArchive;
use wcf\system\WCF;
use wcf\util\DOMUtil;
use wcf\util\StringUtil;
use wcf\util\XML;

/**
 * Installs, updates and deletes languages, their categories and items.
 * 
 * @author	<NAME>
 * @copyright	2001-2018 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Package\Plugin
 */
class LanguagePackageInstallationPlugin extends AbstractXMLPackageInstallationPlugin implements IGuiPackageInstallationPlugin {
	use TMultiXmlGuiPackageInstallationPlugin;
	
	/**
	 * @inheritDoc
	 */
	public $className = LanguageItemEditor::class;
	
	/**
	 * newly created language categories when saving language item via GUI
	 * @var	LanguageCategory[]
	 */
	public $newLanguageCategories = [];
	
	/**
	 * @inheritDoc
	 */
	public $tableName = 'language_item';
	
	/**
	 * @inheritDoc
	 */
	public $tagName = 'item';
	
	/**
	 * @inheritDoc
	 */
	public function install() {
		AbstractPackageInstallationPlugin::install();
		
		// get language files
		$languageFiles = [];
		$multipleFiles = false;
		$filename = $this->instruction['value'];
		if (strpos($filename, '*') !== false) {
			// wildcard syntax; import multiple files
			$multipleFiles = true;
			$files = $this->installation->getArchive()->getTar()->getContentList();
			$pattern = str_replace("\*", ".*", preg_quote($filename));
			
			foreach ($files as $file) {
				if (preg_match('!'.$pattern.'!i', $file['filename'])) {
					if (preg_match('~([a-z-]+)\.xml$~i', $file['filename'], $match)) {
						$languageFiles[$match[1]] = $file['filename'];
					}
					else {
						throw new SystemException("Cannot determine language code of language file '".$file['filename']."'");
					}
				}
			}
		}
		else {
			if (!empty($this->instruction['attributes']['languagecode'])) {
				$languageCode = $this->instruction['attributes']['languagecode'];
			}
			else if (!empty($this->instruction['attributes']['language'])) {
				$languageCode = $this->instruction['attributes']['language'];
			}
			else if (preg_match('~([a-z-]+)\.xml$~i', $filename, $match)) {
				$languageCode = $match[1];
			}
			else {
				throw new SystemException("Cannot determine language code of language file '".$filename."'");
			}
			
			$languageFiles[$languageCode] = $filename;
		}
		
		// get installed languages
		$sql = "SELECT		*
			FROM		wcf".WCF_N."_language
			ORDER BY	isDefault DESC";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute();
		$installedLanguages = $statement->fetchAll(\PDO::FETCH_ASSOC);
		
		// install language
		foreach ($installedLanguages as $installedLanguage) {
			$languageFile = null;
			$updateExistingItems = true;
			if (isset($languageFiles[$installedLanguage['languageCode']])) {
				$languageFile = $languageFiles[$installedLanguage['languageCode']];
			}
			else if ($multipleFiles) {
				// do not update existing items, only add new ones
				$updateExistingItems = false;
				
				// use default language
				if (isset($languageFiles[$installedLanguages[0]['languageCode']])) {
					$languageFile = $languageFiles[$installedLanguages[0]['languageCode']];
				}
				
				// use english (if installed)
				else if (isset($languageFiles['en'])) {
					foreach ($installedLanguages as $installedLanguage2) {
						if ($installedLanguage2['languageCode'] == 'en') {
							$languageFile = $languageFiles['en'];
							break;
						}
					}
				}
				
				// use any installed language
				if ($languageFile === null) {
					foreach ($installedLanguages as $installedLanguage2) {
						if (isset($languageFiles[$installedLanguage2['languageCode']])) {
							$languageFile = $languageFiles[$installedLanguage2['languageCode']];
							break;
						}
					}
				}
				
				// use first delivered language
				if ($languageFile === null) {
					foreach ($languageFiles as $languageFile) break;
				}
			}
			
			// save language
			if ($languageFile !== null) {
				if ($xml = $this->readLanguage($languageFile)) {
					// get language object
					$languageEditor = new LanguageEditor(new Language(null, $installedLanguage));
					
					// import xml
					// don't update language files if package is an application
					$languageEditor->updateFromXML($xml, $this->installation->getPackageID(), !$this->installation->getPackage()->isApplication, $updateExistingItems);
				}
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	public function uninstall() {
		parent::uninstall();
		
		// delete language items
		// Get all items and their categories
		// which where installed from this package.
		$sql = "SELECT	languageItemID, languageCategoryID, languageID
			FROM	wcf".WCF_N."_language_item
			WHERE	packageID = ?";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute([$this->installation->getPackageID()]);
		$itemIDs = [];
		$categoryIDs = [];
		while ($row = $statement->fetchArray()) {
			$itemIDs[] = $row['languageItemID'];
			
			// Store categories
			$categoryIDs[$row['languageCategoryID']] = true;
		}
		
		if (!empty($itemIDs)) {
			$sql = "DELETE FROM	wcf".WCF_N."_".$this->tableName."
				WHERE		languageItemID = ?
						AND packageID = ?";
			$statement = WCF::getDB()->prepareStatement($sql);
			
			foreach ($itemIDs as $itemID) {
				$statement->execute([
					$itemID,
					$this->installation->getPackageID()
				]);
			}
			
			$this->deleteEmptyCategories(array_keys($categoryIDs), $this->installation->getPackageID());
		}
	}
	
	/**
	 * Extracts the language file and parses it. If the specified language file
	 * was not found, an exception message is thrown.
	 * 
	 * @param	string		$filename
	 * @return	XML
	 * @throws	SystemException
	 */
	protected function readLanguage($filename) {
		// search language files in package archive
		// throw error message if not found
		if (($fileIndex = $this->installation->getArchive()->getTar()->getIndexByFilename($filename)) === false) {
			throw new SystemException("language file '".$filename."' not found.");
		}
		
		// extract language file and parse with DOMDocument
		$xml = new XML();
		$xml->loadXML($filename, $this->installation->getArchive()->getTar()->extractToString($fileIndex));
		return $xml;
	}
	
	/**
	 * Deletes categories which where changed by an update or uninstallation
	 * in case they are now empty.
	 * 
	 * @param	array		$categoryIDs
	 * @param	integer		$packageID
	 */
	protected function deleteEmptyCategories(array $categoryIDs, $packageID) {
		// Get empty categories which where changed by this package.
		$conditions = new PreparedStatementConditionBuilder();
		$conditions->add("language_category.languageCategoryID IN (?)", [$categoryIDs]);
		
		$sql = "SELECT		COUNT(item.languageItemID) AS count,
					language_category.languageCategoryID,
					language_category.languageCategory
			FROM		wcf".WCF_N."_language_category language_category
			LEFT JOIN	wcf".WCF_N."_language_item item
			ON		(item.languageCategoryID = language_category.languageCategoryID)
			".$conditions."
			GROUP BY	language_category.languageCategoryID ASC,
					language_category.languageCategory ASC";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute($conditions->getParameters());
		$categoriesToDelete = [];
		while ($row = $statement->fetchArray()) {
			if ($row['count'] == 0) {
				$categoriesToDelete[$row['languageCategoryID']] = $row['languageCategory'];
			}
		}
		
		// Delete categories from DB.
		if (!empty($categoriesToDelete)) {
			$sql = "DELETE FROM	wcf".WCF_N."_language_category
				WHERE		languageCategory = ?";
			$statement = WCF::getDB()->prepareStatement($sql);
			
			foreach ($categoriesToDelete as $category) {
				$statement->execute([$category]);
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	protected function handleDelete(array $items) {
		// does nothing
	}
	
	/**
	 * @inheritDoc
	 */
	protected function prepareImport(array $data) {
		// does nothing
	}
	
	/**
	 * @inheritDoc
	 */
	protected function findExistingItem(array $data) {
		// does nothing
	}
	
	/**
	 * @see	\wcf\system\package\plugin\IPackageInstallationPlugin::getDefaultFilename()
	 * @since	3.0
	 */
	public static function getDefaultFilename() {
		return 'language/*.xml';
	}
	/**
	 * @inheritDoc
	 */
	public static function isValid(PackageArchive $archive, $instruction) {
		return true;
	}
	
	/**
	 * @inheritDoc
	 * @since	3.1
	 */
	public static function getSyncDependencies() {
		return [];
	}
	
	/**
	 * @inheritDoc
	 * @since	3.2
	 */
	protected function addFormFields(IFormDocument $form) {
		/** @var FormContainer $dataContainer */
		$dataContainer = $form->getNodeById('data');
		<commit_msg>Reset language cache after installation<commit_after><?php
namespace wcf\system\package\plugin;
use wcf\data\IEditableCachedObject;
use wcf\data\language\category\LanguageCategory;
use wcf\data\language\category\LanguageCategoryAction;
use wcf\data\language\item\LanguageItemEditor;
use wcf\data\language\item\LanguageItemList;
use wcf\data\language\Language;
use wcf\data\language\LanguageEditor;
use wcf\system\database\util\PreparedStatementConditionBuilder;
use wcf\system\devtools\pip\DevtoolsPipEntryList;
use wcf\system\devtools\pip\IDevtoolsPipEntryList;
use wcf\system\devtools\pip\IGuiPackageInstallationPlugin;
use wcf\system\devtools\pip\TMultiXmlGuiPackageInstallationPlugin;
use wcf\system\exception\SystemException;
use wcf\system\form\builder\container\FormContainer;
use wcf\system\form\builder\field\dependency\ValueFormFieldDependency;
use wcf\system\form\builder\field\MultilineTextFormField;
use wcf\system\form\builder\field\RadioButtonFormField;
use wcf\system\form\builder\field\SingleSelectionFormField;
use wcf\system\form\builder\field\TextFormField;
use wcf\system\form\builder\field\validation\FormFieldValidationError;
use wcf\system\form\builder\field\validation\FormFieldValidator;
use wcf\system\form\builder\field\validation\FormFieldValidatorUtil;
use wcf\system\form\builder\IFormDocument;
use wcf\system\language\LanguageFactory;
use wcf\system\package\PackageArchive;
use wcf\system\WCF;
use wcf\util\DOMUtil;
use wcf\util\StringUtil;
use wcf\util\XML;

/**
 * Installs, updates and deletes languages, their categories and items.
 * 
 * @author	<NAME>
 * @copyright	2001-2018 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Package\Plugin
 */
class LanguagePackageInstallationPlugin extends AbstractXMLPackageInstallationPlugin implements IGuiPackageInstallationPlugin {
	use TMultiXmlGuiPackageInstallationPlugin;
	
	/**
	 * @inheritDoc
	 */
	public $className = LanguageItemEditor::class;
	
	/**
	 * newly created language categories when saving language item via GUI
	 * @var	LanguageCategory[]
	 */
	public $newLanguageCategories = [];
	
	/**
	 * @inheritDoc
	 */
	public $tableName = 'language_item';
	
	/**
	 * @inheritDoc
	 */
	public $tagName = 'item';
	
	/**
	 * @inheritDoc
	 */
	public function install() {
		AbstractPackageInstallationPlugin::install();
		
		// get language files
		$languageFiles = [];
		$multipleFiles = false;
		$filename = $this->instruction['value'];
		if (strpos($filename, '*') !== false) {
			// wildcard syntax; import multiple files
			$multipleFiles = true;
			$files = $this->installation->getArchive()->getTar()->getContentList();
			$pattern = str_replace("\*", ".*", preg_quote($filename));
			
			foreach ($files as $file) {
				if (preg_match('!'.$pattern.'!i', $file['filename'])) {
					if (preg_match('~([a-z-]+)\.xml$~i', $file['filename'], $match)) {
						$languageFiles[$match[1]] = $file['filename'];
					}
					else {
						throw new SystemException("Cannot determine language code of language file '".$file['filename']."'");
					}
				}
			}
		}
		else {
			if (!empty($this->instruction['attributes']['languagecode'])) {
				$languageCode = $this->instruction['attributes']['languagecode'];
			}
			else if (!empty($this->instruction['attributes']['language'])) {
				$languageCode = $this->instruction['attributes']['language'];
			}
			else if (preg_match('~([a-z-]+)\.xml$~i', $filename, $match)) {
				$languageCode = $match[1];
			}
			else {
				throw new SystemException("Cannot determine language code of language file '".$filename."'");
			}
			
			$languageFiles[$languageCode] = $filename;
		}
		
		// get installed languages
		$sql = "SELECT		*
			FROM		wcf".WCF_N."_language
			ORDER BY	isDefault DESC";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute();
		$installedLanguages = $statement->fetchAll(\PDO::FETCH_ASSOC);
		
		// install language
		foreach ($installedLanguages as $installedLanguage) {
			$languageFile = null;
			$updateExistingItems = true;
			if (isset($languageFiles[$installedLanguage['languageCode']])) {
				$languageFile = $languageFiles[$installedLanguage['languageCode']];
			}
			else if ($multipleFiles) {
				// do not update existing items, only add new ones
				$updateExistingItems = false;
				
				// use default language
				if (isset($languageFiles[$installedLanguages[0]['languageCode']])) {
					$languageFile = $languageFiles[$installedLanguages[0]['languageCode']];
				}
				
				// use english (if installed)
				else if (isset($languageFiles['en'])) {
					foreach ($installedLanguages as $installedLanguage2) {
						if ($installedLanguage2['languageCode'] == 'en') {
							$languageFile = $languageFiles['en'];
							break;
						}
					}
				}
				
				// use any installed language
				if ($languageFile === null) {
					foreach ($installedLanguages as $installedLanguage2) {
						if (isset($languageFiles[$installedLanguage2['languageCode']])) {
							$languageFile = $languageFiles[$installedLanguage2['languageCode']];
							break;
						}
					}
				}
				
				// use first delivered language
				if ($languageFile === null) {
					foreach ($languageFiles as $languageFile) break;
				}
			}
			
			// save language
			if ($languageFile !== null) {
				if ($xml = $this->readLanguage($languageFile)) {
					// get language object
					$languageEditor = new LanguageEditor(new Language(null, $installedLanguage));
					
					// import xml
					// don't update language files if package is an application
					$languageEditor->updateFromXML($xml, $this->installation->getPackageID(), !$this->installation->getPackage()->isApplication, $updateExistingItems);
				}
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	public function uninstall() {
		parent::uninstall();
		
		// delete language items
		// Get all items and their categories
		// which where installed from this package.
		$sql = "SELECT	languageItemID, languageCategoryID, languageID
			FROM	wcf".WCF_N."_language_item
			WHERE	packageID = ?";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute([$this->installation->getPackageID()]);
		$itemIDs = [];
		$categoryIDs = [];
		while ($row = $statement->fetchArray()) {
			$itemIDs[] = $row['languageItemID'];
			
			// Store categories
			$categoryIDs[$row['languageCategoryID']] = true;
		}
		
		if (!empty($itemIDs)) {
			$sql = "DELETE FROM	wcf".WCF_N."_".$this->tableName."
				WHERE		languageItemID = ?
						AND packageID = ?";
			$statement = WCF::getDB()->prepareStatement($sql);
			
			foreach ($itemIDs as $itemID) {
				$statement->execute([
					$itemID,
					$this->installation->getPackageID()
				]);
			}
			
			$this->deleteEmptyCategories(array_keys($categoryIDs), $this->installation->getPackageID());
		}
	}
	
	/**
	 * Extracts the language file and parses it. If the specified language file
	 * was not found, an exception message is thrown.
	 * 
	 * @param	string		$filename
	 * @return	XML
	 * @throws	SystemException
	 */
	protected function readLanguage($filename) {
		// search language files in package archive
		// throw error message if not found
		if (($fileIndex = $this->installation->getArchive()->getTar()->getIndexByFilename($filename)) === false) {
			throw new SystemException("language file '".$filename."' not found.");
		}
		
		// extract language file and parse with DOMDocument
		$xml = new XML();
		$xml->loadXML($filename, $this->installation->getArchive()->getTar()->extractToString($fileIndex));
		return $xml;
	}
	
	/**
	 * Deletes categories which where changed by an update or uninstallation
	 * in case they are now empty.
	 * 
	 * @param	array		$categoryIDs
	 * @param	integer		$packageID
	 */
	protected function deleteEmptyCategories(array $categoryIDs, $packageID) {
		// Get empty categories which where changed by this package.
		$conditions = new PreparedStatementConditionBuilder();
		$conditions->add("language_category.languageCategoryID IN (?)", [$categoryIDs]);
		
		$sql = "SELECT		COUNT(item.languageItemID) AS count,
					language_category.languageCategoryID,
					language_category.languageCategory
			FROM		wcf".WCF_N."_language_category language_category
			LEFT JOIN	wcf".WCF_N."_language_item item
			ON		(item.languageCategoryID = language_category.languageCategoryID)
			".$conditions."
			GROUP BY	language_category.languageCategoryID ASC,
					language_category.languageCategory ASC";
		$statement = WCF::getDB()->prepareStatement($sql);
		$statement->execute($conditions->getParameters());
		$categoriesToDelete = [];
		while ($row = $statement->fetchArray()) {
			if ($row['count'] == 0) {
				$categoriesToDelete[$row['languageCategoryID']] = $row['languageCategory'];
			}
		}
		
		// Delete categories from DB.
		if (!empty($categoriesToDelete)) {
			$sql = "DELETE FROM	wcf".WCF_N."_language_category
				WHERE		languageCategory = ?";
			$statement = WCF::getDB()->prepareStatement($sql);
			
			foreach ($categoriesToDelete as $category) {
				$statement->execute([$category]);
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	protected function handleDelete(array $items) {
		// does nothing
	}
	
	/**
	 * @inheritDoc
	 */
	protected function prepareImport(array $data) {
		// does nothing
	}
	
	/**
	 * @inheritDoc
	 */
	protected function findExistingItem(array $data) {
		// does nothing
	}
	
	/**
	 * @inheritDoc
	 */
	protected function postImport() {
		LanguageFactory::getInstance()->deleteLanguageCache();
	}
	
	/**
	 * @see	\wcf\system\package\plugin\IPackageInstallationPlugin::getDefaultFilename()
	 * @since	3.0
	 */
	public static function getDefaultFilename() {
		return 'language/*.xml';
	}
	/**
	 * @inheritDoc
	 */
	public static function isValid(PackageArchive $archive, $instruction) {
		return true;
	}
	
	/**
	 * @inheritDoc
	 * @since	3.1
	 */
	public static function getSyncDependencies() {
		return [];
	}
	
	/**
	 * @inheritDoc
	 * @since	3.2
	 */
	protected function addFormFields(IFormDocument $form) {
		/** @var FormContainer $dataContainer */
		$dataContainer = $form->getNodeById('data');
		
{'fraction_non_alphanumeric': 0.14408848587514325, 'fraction_numerical': 0.0023915101390065267, 'mean_word_length': 4.001744330924495, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1570776', 'n_tokens_mistral': 7046, 'n_tokens_neox': 5877, 'n_words': 1241}