<commit_before><?php

/*
 * The MIT License
 *
 * Copyright 2014 <NAME> (namide.com).
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

namespace Flea;

/**
 * Creates and saves all the pages in database
 *
 * @author Namide
 */
class PageListCreate
{
	private static $_INSTANCE = null;
	
	/**
	 * Add all pages of a directory
	 * 
	 * @param type $dir				Root directory
	 */
	public function addPagesByDir( $dir )
	{
		$listOfPages = $this->addPageByDirRecurs( $dir, '' );
		$this->db_insertPages($listOfPages);
	}
	
	/**
	 * Add 301 redirections and other commands
	 * 
	 * @param string $file				File with commands
	 */
	public function commands( $file )
	{
		$listOfPages = array();
		
		include $file;
		if ( isset($redirect301) )
		{
			foreach ( $redirect301 as $oldURL => $newURL )
			{
				$page = new Page();
				$page->setType( Page::$TYPE_REDIRECT301 );
				$page->setPageUrl( $oldURL );
				$page->setHtmlBody( $newURL );
				$page->setVisible( false );
				$page->setCachable( false );
				
				array_push( $listOfPages, $page );
			}
		}
		
		$this->db_insertPages($listOfPages);
	}
	
	private function db_createPagesDB()
	{
		$tableName = DataBase::objectToTableName( Page::getEmptyPage() );
		
		$db = DataBase::getInstance(_DB_DSN_PAGE);
		
		$request = SqlQuery::getTemp( SqlQuery::$TYPE_CREATE );
		$request->initCreate($tableName, Page::getEmptyPage()->getObjectVars() );
		$db->execute( $request );
		
		$request->clean( SqlQuery::$TYPE_CREATE );
		$request->initCreate($tableName.'_array', array( 'page_id'=>'TEXT', 'page_prop'=>'TEXT', 'key'=>'TEXT', 'value'=>'TEXT') );
		$db->execute( $request );
	}

	private function db_insertPages( array $list )
	{
		$tableName = DataBase::objectToTableName( Page::getEmptyPage() );
		$db = DataBase::getInstance(_DB_DSN_PAGE);
		
		if ( !$db->exist($tableName) )
		{
			$this->db_createPagesDB();
		}
		
		// create CSV
		$createCSV = false;
		
		$strTemp = '';
		foreach ($list as $page) 
		{
			if ($createCSV)
			{
				$strTemp .= $page->getName() . ',';
				$strTemp .= $page->getLang() . ',';
				$strTemp .= $page->getPageUrl() . ',';
				$strTemp .= $page->getTemplate() . ',';
				$strTemp .= $page->getDate() . ',';
				$strTemp .= '"' . implode( ' ; ', $page->getTags()->getArray() ) . '",';
				$strTemp .= '"' . $page->getHtmlTitle() . '",';
				$strTemp .= '"' . $page->getHtmlDescription() . '",';

				$strTemp .= '"';
				foreach ($page->getContents()->getArray() as $key => $value) {
					$strTemp .= $key . ' : ' . $value . ' ; ';
				}
				if ($page->getHtmlHeader() != '')
					$strTemp .= 'header : ' . $page->getHtmlHeader() . ' ; ';
				if (count($page->getContents()->getArray()) > 0 || $page->getHtmlHeader() != '')
					$strTemp = substr($strTemp, 0, -2);
				$strTemp .= '",';

				$strTemp .= ($page->getVisible() ? '1' : '0') . ',';
				$strTemp .= ($page->getCachable() ? '1' : '0') . ',';
				$strTemp .= '"' . $page->getType() . '",';
				$strTemp .= ($page->getGetEnabled() ? '1' : '0') . ',';
				$strTemp .= ($page->getGetExplicit() ? '1' : '0') . ',';
				$strTemp .= '"' . implode( ' ; ', $page->getAdditionalUrls()->getArray() ) . '",';
				$strTemp .= '"' . $page->getPhpHeader() . '"';
				$strTemp .= "\n";
			}
				
			
			$pageVars = $page->getObjectVars();
			
			$request = SqlQuery::getTemp( SqlQuery::$TYPE_INSERT );
			$request->initInsertValues( $tableName, $pageVars);
			$db->execute($request);
			
			foreach ($pageVars as $key => $value)
			{
				if(	gettype($value) == 'array' )
				{
					foreach ($value as $key2 => $val2)
					{
						$obj = array();
						$obj['page_id'] = $pageVars['_id'];
						$obj['page_prop'] = $key;
						$obj['key'] = $key2;
						$obj['value'] = $val2;
						
						$request->clean( SqlQuery::$TYPE_INSERT );
						$request->initInsertValues( $tableName.'_array', $obj );
						$db->execute($request);
					}
				}
			}
		}
		
		if ($createCSV)
			echo utf8_decode($strTemp);
	}	
			
	
	/**
	 * Add all pages of a directory recursivly
	 * 
	 * @param type $dir				Root directory
	 * @param type $fileDirRel		Relative directory (for the recursivity)	
	 * @return array				List of the pages added
	 */
	private function addPageByDirRecurs( $dir, $fileDirRel = '' )
	{
		$list = array();
		
		if ( !file_exists($dir) ) { return $list; }

		$dirOpen = opendir($dir);
		while($file = @readdir($dirOpen))
		{
			if( $file != "." &&
				$file != ".." &&
				is_dir($dir.'/'.$file) )
			{
				$list1 = $this->addPageByDirRecurs( $dir.'/'.$file.'/', (($fileDirRel != '')?$fileDirRel.'/':'').$file );<commit_msg>Revert "Hack to generate CSV from init pages files"<commit_after><?php

/*
 * The MIT License
 *
 * Copyright 2014 <NAME> (namide.com).
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

namespace Flea;

/**
 * Creates and saves all the pages in database
 *
 * @author Namide
 */
class PageListCreate
{
	private static $_INSTANCE = null;
	
	/**
	 * Add all pages of a directory
	 * 
	 * @param type $dir				Root directory
	 */
	public function addPagesByDir( $dir )
	{
		$listOfPages = $this->addPageByDirRecurs( $dir, '' );
		$this->db_insertPages($listOfPages);
	}
	
	/**
	 * Add 301 redirections and other commands
	 * 
	 * @param string $file				File with commands
	 */
	public function commands( $file )
	{
		$listOfPages = array();
		
		include $file;
		if ( isset($redirect301) )
		{
			foreach ( $redirect301 as $oldURL => $newURL )
			{
				$page = new Page();
				$page->setType( Page::$TYPE_REDIRECT301 );
				$page->setPageUrl( $oldURL );
				$page->setHtmlBody( $newURL );
				$page->setVisible( false );
				$page->setCachable( false );
				
				array_push( $listOfPages, $page );
			}
		}
		
		$this->db_insertPages($listOfPages);
	}
	
	private function db_createPagesDB()
	{
		$tableName = DataBase::objectToTableName( Page::getEmptyPage() );
		
		$db = DataBase::getInstance(_DB_DSN_PAGE);
		
		$request = SqlQuery::getTemp( SqlQuery::$TYPE_CREATE );
		$request->initCreate($tableName, Page::getEmptyPage()->getObjectVars() );
		$db->execute( $request );
		
		$request->clean( SqlQuery::$TYPE_CREATE );
		$request->initCreate($tableName.'_array', array( 'page_id'=>'TEXT', 'page_prop'=>'TEXT', 'key'=>'TEXT', 'value'=>'TEXT') );
		$db->execute( $request );
	}


	private function db_insertPages( array $list )
	{
		$tableName = DataBase::objectToTableName( Page::getEmptyPage() );
		$db = DataBase::getInstance(_DB_DSN_PAGE);
		
		if ( !$db->exist($tableName) )
		{
			$this->db_createPagesDB();
		}
		
		foreach ($list as $page) 
		{
			$pageVars = $page->getObjectVars();
			
			$request = SqlQuery::getTemp( SqlQuery::$TYPE_INSERT );
			$request->initInsertValues( $tableName, $pageVars);
			$db->execute($request);
			
			foreach ($pageVars as $key => $value)
			{
				if(	gettype($value) == 'array' )
				{
					foreach ($value as $key2 => $val2)
					{
						$obj = array();
						$obj['page_id'] = $pageVars['_id'];
						$obj['page_prop'] = $key;
						$obj['key'] = $key2;
						$obj['value'] = $val2;
						
						$request->clean( SqlQuery::$TYPE_INSERT );
						$request->initInsertValues( $tableName.'_array', $obj );
						$db->execute($request);
					}
				}
			}
		}
	}
	
	/**
	 * Add all pages of a directory recursivly
	 * 
	 * @param type $dir				Root directory
	 * @param type $fileDirRel		Relative directory (for the recursivity)	
	 * @return array				List of the pages added
	 */
	private function addPageByDirRecurs( $dir, $fileDirRel = '' )
	{
		$list = array();
		
		if ( !file_exists($dir) ) { return $list; }

		$dirOpen = opendir($dir);
		while($file = @readdir($dirOpen))
		{
			if( $file != "." &&
				$file != ".." &&
				is_dir($dir.'/'.$file) )
			{
				$list1 = $this->addPageByDirRecurs( $dir.'/'.$file.'/', (($fileDirRel != '')?$fileDirRel.'/':'').$file );
{'fraction_non_alphanumeric': 0.1690772051334284, 'fraction_numerical': 0.005500101853738032, 'mean_word_length': 3.2933974639265413, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3850948', 'n_tokens_mistral': 3757, 'n_tokens_neox': 3185, 'n_words': 848}