<commit_before><!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body>
<tt>

<?php
///////////////////////////////////////////////////////////////////////////////
// SMF2 to FLARUM migration script
//
// s9e\TextFormatter version
//
// This version performes a migration from SMF2 to Flarum using the 
// TextFormatter parser from s9e\TextFormatter version by PHP
//
// (C) <NAME> - ISAA Technical Team (http://www.isaa.it)
// This file is shared under a Creative Commons "BY" license.
//
// Partially based on script phpbb_to_flarum by robrotheram, VIRUXE, Reflic
// https://github.com/robrotheram/phpbb_to_flarum
//
// License: MIT
///////////////////////////////////////////////////////////////////////////////

// Get the autoloader (unless you have already loaded Composer's)
include __DIR__ . '/vendor/autoload.php';

use s9e\TextFormatter\Bundles\Forum as TextFormatter;

set_time_limit(60*60*24); // 1 day
error_reporting(E_ALL);
ini_set('display_errors',1);
$timestamp_start = time();

///////////////////////////////////////////////////////////////////////////////
// GENERAL SETTINGS ***TO BE TAILORED***
///////////////////////////////////////////////////////////////////////////////
// General settings are in smf2_to_flarum_settings.php
///////////////////////////////////////////////////////////////////////////////
include_once("smf2_to_flarum_settings.php");

///////////////////////////////////////////////////////////////////////////////
// AUXILIARY FUNCTIONS
///////////////////////////////////////////////////////////////////////////////
function slugify($text)
{
	$text = preg_replace('~[^\\pL\d]+~u', '-', $text);
	$text = trim($text, '-');
	$text = iconv('utf-8', 'us-ascii//TRANSLIT', $text);
	$text = strtolower($text);
	$text = preg_replace('~[^-\w]+~', '', $text);

	if (empty($text))
		return 'n-a';

	return $text;
}

// Generates a random color for Tags
function rand_color()
{
	return '#' . str_pad(dechex(mt_rand(0, 0xFFFFFF)), 6, '0', STR_PAD_LEFT);
}

// Used to convert Categories to Tags
function stripBBCode($text_to_search) {
	$pattern = '|[[\/\!]*?[^\[\]]*?]|si';
	$replace = '';
	return preg_replace($pattern, $replace, $text_to_search);
}

// We convert messages bodies into Flarum-compatible XML format
function newFormatText($connection,$text)
{
	
	// We get rid of [html] bbcode
	$text = preg_replace('/(\[html]|\[\/html])/si', '', $text);
	
	// HTML line breaks to \n
	$text = preg_replace('/(<br\s?\/?>)+/is', "\n", $text);
	
	// Replace multiple (one ore more) line breaks with a single one.
	$text = preg_replace('/[\r\n]{2,}/s', "\n", $text);
	
	// Handle some special case here...
	$text = preg_replace('/(&amp;#039;|&#039;)+/',"'", $text);
	
	// Force encoding conversion to UTF-8
	$text = html_entity_decode($text,ENT_COMPAT|ENT_HTML401,'UTF-8');
	
	// 
	$text = TextFormatter::parse($text);
	
	// Delete all non converted bbcode
	$text = preg_replace('/[[\/\!]*?[^\[\]]*?]/si', '', $text);
	
	return $connection->real_escape_string($text);
}

// Older attachments may still use this function.
function getLegacyAttachmentFilename($filename, $attachment_id)
{
	$clean_name = $filename;

	// Remove international characters (windows-1252)<commit_msg>Better management of [html] bbcode<commit_after><!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body>
<tt>

<?php
///////////////////////////////////////////////////////////////////////////////
// SMF2 to FLARUM migration script
//
// s9e\TextFormatter version
//
// This version performes a migration from SMF2 to Flarum using the 
// TextFormatter parser from s9e\TextFormatter version by PHP
//
// (C) <NAME> - ISAA Technical Team (http://www.isaa.it)
// This file is shared under a Creative Commons "BY" license.
//
// Partially based on script phpbb_to_flarum by robrotheram, VIRUXE, Reflic
// https://github.com/robrotheram/phpbb_to_flarum
//
// License: MIT
///////////////////////////////////////////////////////////////////////////////

// Get the autoloader (unless you have already loaded Composer's)
include __DIR__ . '/vendor/autoload.php';

use s9e\TextFormatter\Bundles\Forum as TextFormatter;

set_time_limit(60*60*24); // 1 day
error_reporting(E_ALL);
ini_set('display_errors',1);
$timestamp_start = time();

///////////////////////////////////////////////////////////////////////////////
// GENERAL SETTINGS ***TO BE TAILORED***
///////////////////////////////////////////////////////////////////////////////
// General settings are in smf2_to_flarum_settings.php
///////////////////////////////////////////////////////////////////////////////
include_once("smf2_to_flarum_settings.php");

///////////////////////////////////////////////////////////////////////////////
// AUXILIARY FUNCTIONS
///////////////////////////////////////////////////////////////////////////////
function slugify($text)
{
	$text = preg_replace('~[^\\pL\d]+~u', '-', $text);
	$text = trim($text, '-');
	$text = iconv('utf-8', 'us-ascii//TRANSLIT', $text);
	$text = strtolower($text);
	$text = preg_replace('~[^-\w]+~', '', $text);

	if (empty($text))
		return 'n-a';

	return $text;
}

// Generates a random color for Tags
function rand_color()
{
	return '#' . str_pad(dechex(mt_rand(0, 0xFFFFFF)), 6, '0', STR_PAD_LEFT);
}

// Used to convert Categories to Tags
function stripBBCode($text_to_search) {
	$pattern = '|[[\/\!]*?[^\[\]]*?]|si';
	$replace = '';
	return preg_replace($pattern, $replace, $text_to_search);
}

// We convert messages bodies into Flarum-compatible XML format
function newFormatText($connection,$text)
{
		
	// We get rid of [html] bbcode, saving part of its content.
	if (preg_match('/\[html]/si',$text))
	{
		$text = preg_replace('/\[html]/si', '', $text);
		$text = preg_replace('/\[\/html]/si', '', $text);
		$text = strip_tags($text,"<img>");
		preg_match_all('#src="?(https?:\/\/(\S*?\.\S*?))([\s)\[\]{},;"\':<]|\.\s|$)"?#is',$text,$matches);
		$text = strip_tags($text);
		$text = "[img]".$matches[1][0]."[/img]\n".$text;
	} 
	
	// HTML line breaks to \n
	$text = preg_replace('/(<br\s?\/?>)+/is', "\n", $text);
	
	// Replace multiple (one ore more) line breaks with a single one.
	$text = preg_replace('/[\r\n]{2,}/s', "\n", $text);
	
	// Handle some special case here...
	$text = preg_replace('/(&amp;#039;|&#039;)+/',"'", $text);
	
	// Force encoding conversion to UTF-8
	$text = html_entity_decode($text,ENT_COMPAT|ENT_HTML401,'UTF-8');
	
	// 
	$text = TextFormatter::parse($text);
	
	// Delete all non converted bbcode
	$text = preg_replace('/[[\/\!]*?[^\[\]]*?]/si', '', $text);
	
	return $connection->real_escape_string($text);
}

// Older attachments may still use this function.
function getLegacyAttachmentFilename($filename, $attachment_id)
{
	$clean_name = $filename;

	// Remove international characters (windows-1252)
{'fraction_non_alphanumeric': 0.33043478260869563, 'fraction_numerical': 0.011201179071481208, 'mean_word_length': 2.088757396449704, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '643021', 'n_tokens_mistral': 2208, 'n_tokens_neox': 2073, 'n_words': 492}