<commit_before>#include "sitewindow.h"
#include "ui_sitewindow.h"
#include "mainwindow.h"
#include "functions.h"

extern mainWindow *_mainwindow;



siteWindow::siteWindow(QMap<QString,Site*> *sites, QWidget *parent)
	: QDialog(parent), ui(new Ui::siteWindow)
{
	Q_UNUSED(sites);

	ui->setupUi(this);
	ui->progressBar->hide();
	m_models = new QList<Site*>();

	QStringList dir = QDir(savePath("sites")).entryList(QDir::Dirs | QDir::NoDotAndDotDot);

	for (int i = 0; i < dir.count(); i++)
	{
		QFile file(savePath("sites/"+dir.at(i)+"/model.xml"));
		if (file.open(QIODevice::ReadOnly | QIODevice::Text))
		{
			QString source = file.readAll();
			QDomDocument doc;
			QString errorMsg;
			int errorLine, errorColumn;
			if (!doc.setContent(source, false, &errorMsg, &errorLine, &errorColumn))
			{ log(tr("Erreur lors de l'analyse du fichier XML : %1 (%2 - %3).").arg(errorMsg, QString::number(errorLine), QString::number(errorColumn)), Error); }
			else
			{
				QDomElement docElem = doc.documentElement();
				QMap<QString,QString> detals = domToMap(docElem);
				detals["Model"] = dir[i];
				Site *site = new Site(dir[i], dir[i], detals);
				m_models->append(site);
			}
			file.close();
		}
	}

	QStringList types;
	for (Site *site : *m_models)
	{
		if (!types.contains(site->type()))
		{
			types.append(site->type());
			ui->comboBox->addItem(QIcon(savePath("sites/"+site->type()+"/icon.png")), site->type());
		}
	}

	ui->comboBox->setDisabled(true);
	ui->checkBox->setChecked(true);
}

siteWindow::~siteWindow()
{
	delete ui;
}

void siteWindow::accept()
{
	ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(false);

	QString url = ui->lineEdit->text();
	if (url.startsWith("http://"))
	{ url.remove("http://"); }
	if (url.startsWith("https://"))
	{ url.remove("https://"); }
	if (url.endsWith("/"))
	{ url = url.left(url.size()-1); }

	QString type, name;
	QStringList checked;
	if (ui->checkBox->isChecked())
	{
		ui->progressBar->setValue(0);
		ui->progressBar->setMaximum(m_models->count());
		ui->progressBar->show();

		for (Site *map : *m_models)
		{
			if (!checked.contains(map->type()))
			{
				checked.append(map->type());
				if (map->contains("Check/Url") && map->contains("Check/Regex"))
				{
					QString curr = map->value("Selected");
					curr[0] = curr[0].toUpper();
					QNetworkReply *reply = map->get("http://"+url+map->value("Check/Url"));
					QEventLoop loop;
						connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
					loop.exec();
					QString source = reply->readAll();
					if (reply->error() == 0)
					{
						QRegExp rx(map->value("Check/Regex"));
						if (rx.indexIn(source) != -1)
						{
							type = map->type();
							break;
						}
					}
					else
					{ log(tr("Erreur lors de la récupération de la page de test : %1.").arg(reply->errorString()), Error); }
				}
				ui->progressBar->setValue(checked.size());
			}
		}
		if (type.isEmpty())
		{
			error(this, tr("Impossible de deviner le type du site. Êtes-vous sûr de l'url ?"));
			ui->comboBox->setDisabled(false);
			ui->checkBox->setChecked(false);<commit_msg>Fix source type detection on redirecting sites (usually https)<commit_after>#include "sitewindow.h"
#include "ui_sitewindow.h"
#include "mainwindow.h"
#include "functions.h"

extern mainWindow *_mainwindow;



siteWindow::siteWindow(QMap<QString,Site*> *sites, QWidget *parent)
	: QDialog(parent), ui(new Ui::siteWindow)
{
	Q_UNUSED(sites);

	ui->setupUi(this);
	ui->progressBar->hide();
	m_models = new QList<Site*>();

	QStringList dir = QDir(savePath("sites")).entryList(QDir::Dirs | QDir::NoDotAndDotDot);

	for (int i = 0; i < dir.count(); i++)
	{
		QFile file(savePath("sites/"+dir.at(i)+"/model.xml"));
		if (file.open(QIODevice::ReadOnly | QIODevice::Text))
		{
			QString source = file.readAll();
			QDomDocument doc;
			QString errorMsg;
			int errorLine, errorColumn;
			if (!doc.setContent(source, false, &errorMsg, &errorLine, &errorColumn))
			{ log(tr("Erreur lors de l'analyse du fichier XML : %1 (%2 - %3).").arg(errorMsg, QString::number(errorLine), QString::number(errorColumn)), Error); }
			else
			{
				QDomElement docElem = doc.documentElement();
				QMap<QString,QString> detals = domToMap(docElem);
				detals["Model"] = dir[i];
				Site *site = new Site(dir[i], dir[i], detals);
				m_models->append(site);
			}
			file.close();
		}
	}

	QStringList types;
	for (Site *site : *m_models)
	{
		if (!types.contains(site->type()))
		{
			types.append(site->type());
			ui->comboBox->addItem(QIcon(savePath("sites/"+site->type()+"/icon.png")), site->type());
		}
	}

	ui->comboBox->setDisabled(true);
	ui->checkBox->setChecked(true);
}

siteWindow::~siteWindow()
{
	delete ui;
}

void siteWindow::accept()
{
	ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(false);

	QString url = ui->lineEdit->text();
	if (url.startsWith("http://"))
	{ url.remove("http://"); }
	if (url.startsWith("https://"))
	{ url.remove("https://"); }
	if (url.endsWith("/"))
	{ url = url.left(url.size()-1); }

	QString type, name;
	QStringList checked;
	if (ui->checkBox->isChecked())
	{
		ui->progressBar->setValue(0);
		ui->progressBar->setMaximum(m_models->count());
		ui->progressBar->show();

		for (Site *map : *m_models)
		{
			if (!checked.contains(map->type()))
			{
				checked.append(map->type());
				if (map->contains("Check/Url") && map->contains("Check/Regex"))
				{
					QString curr = map->value("Selected");
					curr[0] = curr[0].toUpper();

					QUrl getUrl("http://" + url+map->value("Check/Url"));
					QNetworkReply *reply;
					do
					{
						reply = map->get(getUrl);
						QEventLoop loop;
							connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
						loop.exec();

						getUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
					} while (!getUrl.isEmpty());
					QString source = reply->readAll();
					if (reply->error() == 0)
					{
						QRegExp rx(map->value("Check/Regex"));
						if (rx.indexIn(source) != -1)
						{
							type = map->type();
							break;
						}
					}
					else
					{ log(tr("Erreur lors de la récupération de la page de test : %1.").arg(reply->errorString()), Error); }
				}
				ui->progressBar->setValue(checked.size());
			}
		}
		if (type.isEmpty())
		{
			error(this, tr("Impossible de deviner le type du site. Êtes-vous sûr de l'url ?"));
			ui->comboBox->setDisabled(false);
			ui->checkBox->setChecked(false);
{'fraction_non_alphanumeric': 0.20118712902218056, 'fraction_numerical': 0.003436426116838488, 'mean_word_length': 3.8434190620272313, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 102, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319108', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2305, 'n_words': 410}