<commit_before>#!/usr/bin/php
<?php
/*
	Copyright 2002-2007 <NAME> <<EMAIL>>

	This file is part of LL.

	LL is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	LL is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with LL.  If not, see <http://www.gnu.org/licenses/>.
*/

if (php_sapi_name() != 'cli')
	{
	die ('run from cli only!');
	}

define('IN_LL', null);

require ('modules/Modul.php');
require ('modules/Settings.php');
require ('modules/Exceptions.php');
require ('modules/Functions.php');
require ('modules/Input.php');
require ('modules/Output.php');
require ('modules/L10n.php');

Modul::set('Settings', new Settings());
$Input = Modul::set('Input', new Input());
Modul::set('L10n', new L10n());
$Output = Modul::set('Output', new Output());

function __autoload($class)
	{
	Modul::loadModul($class);
	}

class Update extends Modul {


public function run()
	{
	set_time_limit(0);

	$this->DB->connect(
		$this->Settings->getValue('sql_host'),
		$this->Settings->getValue('sql_user'),
		$this->Settings->getValue('sql_password'),
		$this->Settings->getValue('sql_database'));
		
	$_SERVER['HTTP_HOST'] = $this->Board->getHost();
	$tables = $this->DB->getColumnSet('SHOW TABLES');
	$this->DB->execute('LOCK TABLES `'.implode('` WRITE, `', $tables).'` WRITE');

	$this->updateDB();
	$this->updateMarkup();

	$this->DB->execute('UNLOCK TABLES');
	}

private function updateDB()
	{
	$this->DB->execute('DROP TABLE `cache`');
	$this->DB->execute
		('CREATE TABLE `search` (
			`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
			`query` varchar(255) NOT NULL,
			`expires` int(10) unsigned NOT NULL,
		PRIMARY KEY (`id`),
		UNIQUE KEY `query` (`query`)
		) ENGINE=MyISAM  DEFAULT CHARSET=utf8');
	$this->DB->execute
		('CREATE TABLE `search_threads` (
			`searchid` int(10) unsigned NOT NULL,
			`threadid` int(10) unsigned NOT NULL,
			`score` double unsigned NOT NULL,
		KEY `searchid` (`searchid`,`threadid`)
		) ENGINE=MyISAM DEFAULT CHARSET=utf8');
	}

private function updateMarkup()
	{
	$totalPosts = $this->DB->getColumn
		('SELECT
			COUNT(*)
		FROM
			posts
		');
	$posts = $this->DB->getRowSet
		('
		SELECT
			id,
			text
		FROM
			posts
		');
	$update = $this->DB->prepare
		('
		UPDATE
			posts
		SET
			text = ?
		WHERE
			id = ?
		');

	$brokenPosts = array();

	$i = 0;
	foreach ($posts as $post)
		{
		$i++;
		if ($i % 1000 == 0)
			{
			echo "Processing $i of $totalPosts...\t";
			}
		$text = $this->OldUnMarkup->fromHtml($post['text']);
		
		$text = str_replace('<angel>', '0:-)', $text);
		$text = str_replace('<blank>', ':-|', $text);
		$text = str_replace('', '€', $text);
		$text = str_replace('', '€', $text);

		try
			{
			$text = $this->Markup->toHtml($text);
			}
		catch (RuntimeException $e)
			{
			$brokenPosts[] = $post['id'];
			AdminFunctions::delPost($post['id']);
			continue;
			}

		$update->bindString($text);<commit_msg>convert broken characters to utf8 during update<commit_after>#!/usr/bin/php
<?php
/*
	Copyright 2002-2007 <NAME> <<EMAIL>>

	This file is part of LL.

	LL is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	LL is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with LL.  If not, see <http://www.gnu.org/licenses/>.
*/

if (php_sapi_name() != 'cli')
	{
	die ('run from cli only!');
	}

define('IN_LL', null);

require ('modules/Modul.php');
require ('modules/Settings.php');
require ('modules/Exceptions.php');
require ('modules/Functions.php');
require ('modules/Input.php');
require ('modules/Output.php');
require ('modules/L10n.php');

Modul::set('Settings', new Settings());
$Input = Modul::set('Input', new Input());
Modul::set('L10n', new L10n());
$Output = Modul::set('Output', new Output());

function __autoload($class)
	{
	Modul::loadModul($class);
	}

class Update extends Modul {


public function run()
	{
	set_time_limit(0);

	$this->DB->connect(
		$this->Settings->getValue('sql_host'),
		$this->Settings->getValue('sql_user'),
		$this->Settings->getValue('sql_password'),
		$this->Settings->getValue('sql_database'));
		
	$_SERVER['HTTP_HOST'] = $this->Board->getHost();
	$tables = $this->DB->getColumnSet('SHOW TABLES');
	$this->DB->execute('LOCK TABLES `'.implode('` WRITE, `', $tables).'` WRITE');

	$this->updateDB();
	$this->updateMarkup();

	$this->DB->execute('UNLOCK TABLES');
	}

private function updateDB()
	{
	$this->DB->execute('DROP TABLE `cache`');
	$this->DB->execute
		('CREATE TABLE `search` (
			`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
			`query` varchar(255) NOT NULL,
			`expires` int(10) unsigned NOT NULL,
		PRIMARY KEY (`id`),
		UNIQUE KEY `query` (`query`)
		) ENGINE=MyISAM  DEFAULT CHARSET=utf8');
	$this->DB->execute
		('CREATE TABLE `search_threads` (
			`searchid` int(10) unsigned NOT NULL,
			`threadid` int(10) unsigned NOT NULL,
			`score` double unsigned NOT NULL,
		KEY `searchid` (`searchid`,`threadid`)
		) ENGINE=MyISAM DEFAULT CHARSET=utf8');
	}

private function updateMarkup()
	{
	$totalPosts = $this->DB->getColumn
		('SELECT
			COUNT(*)
		FROM
			posts
		');
	$posts = $this->DB->getRowSet
		('
		SELECT
			id,
			text
		FROM
			posts
		');
	$update = $this->DB->prepare
		('
		UPDATE
			posts
		SET
			text = ?
		WHERE
			id = ?
		');

	$brokenPosts = array();

	$i = 0;
	foreach ($posts as $post)
		{
		$i++;
		if ($i % 1000 == 0)
			{
			echo "Processing $i of $totalPosts...\t";
			}
		$text = $this->OldUnMarkup->fromHtml($post['text']);
		
		$text = str_replace('<angel>', '0:-)', $text);
		$text = str_replace('<blank>', ':-|', $text);
		$text = str_replace('', '€', $text);
		$text = str_replace('', '€', $text);
		$text = mb_convert_encoding($text, 'UTF-8', 'UTF-8');
		try
			{
			$text = $this->Markup->toHtml($text);
			}
		catch (RuntimeException $e)
			{
			$brokenPosts[] = $post['id'];
			AdminFunctions::delPost($post['id']);
			continue;
			}

		$update->bindString($text);
{'fraction_non_alphanumeric': 0.181301421091997, 'fraction_numerical': 0.011219147344801795, 'mean_word_length': 4.084410646387833, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5152850', 'n_tokens_mistral': 2587, 'n_tokens_neox': 2300, 'n_words': 623}