<commit_before>#include "downloader.h"

#include <QDebug>
#include <QtNetwork>
#include <QtXml>
#include <QDir>
#include <QFile>

#define TARGET_URL "http://www.goear.com/"
#define SEARCH_URL "http://www.goear.com/search/"
QString Downloader::DownloadUrl("http://www.goear.com/action/sound/get/");

Downloader::Downloader(QObject *parent) :
    QObject(parent)
{
    m_netAccess = new QNetworkAccessManager(this);

    connect(m_netAccess, SIGNAL(finished(QNetworkReply*)),
            this, SLOT(downloadFinished(QNetworkReply*)));
}

Downloader::~Downloader()
{
    delete m_netAccess;
}

void Downloader::downloadSong(const QString &name, const QString &url)
{
    m_songsToDownload.insert(url, name);
    download(url);
}

void Downloader::download(const QString &urlString)
{
    QUrl url(urlString);
    QNetworkRequest request(url);
    m_nreply = m_netAccess->get(request);
    connect(m_nreply, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(downloadProgressChanged(qint64, qint64)));
}

void Downloader::search(const QString &term)
{
    qDebug() << Q_FUNC_INFO << " " + term;
    download(SEARCH_URL + term);
}

void Downloader::downloadFinished(QNetworkReply *reply)
{
    if (reply->url().toString().startsWith(SEARCH_URL))
        emit searchEnded();

    if (reply->error() != QNetworkReply::NoError) {
        qDebug() << Q_FUNC_INFO << " error downloading "
                 << reply->url().toString() << ":" << reply->errorString();

        if (reply->errorString().contains("Server Error"))
            emit serverError();

        reply->deleteLater();
        return;
    }

    qDebug() << reply->url().toString() << " has been dowloaded";

    QVariant redir = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);

    if (redir.isValid()) {
        QUrl url = redir.toUrl();
        qDebug() << reply->url().toString() << " was redirected to:" << url.toString();

        if (url.isRelative()) {
            url.setScheme(reply->url().scheme());
            /// TODO: check why this doesn't work
            //            url.setEncodedHost(reply->url().encodedHost());
        }

        download(url.toString());
        reply->deleteLater();
        return;
    }

    QString mimeType(reply->header(QNetworkRequest::ContentTypeHeader).toString());

    if (mimeType.contains("text/html")) {
        QString songs(reply->readAll());

        int count = songs.count("listen/");

        QString hasMore = "";
        QString searchTerm = "<ol id=\"new_pagination\">";
        QString closingTerm = "</ol>";

        int termBegins = songs.indexOf(searchTerm);
        int termEnds = songs.indexOf(closingTerm, termBegins);

        if (termBegins != -1) {
            hasMore = songs.mid(termBegins + searchTerm.length(), termEnds - termBegins
                                - searchTerm.length());
        }

        QStringList list = hasMore.simplified().split("<li", QString::SkipEmptyParts);

        for (int i = 0; i < list.length() - 1; ++i) {
            if (list.at(i).simplified().startsWith("class=\"active\">")) {
                emit searchHasMoreResults(reply->url().toString() + "/"
                                          + QString::number(++i));

                break;
            }
        }

        searchTerm = "<ol id=\"search_results\">";
        closingTerm = "</ol>";

        termBegins = songs.indexOf(searchTerm);
        termEnds = songs.indexOf(closingTerm, termBegins);

        songs = songs.mid(termBegins + searchTerm.length(),
                          termEnds - termBegins);

        for (int i = 0; i < count; ++i) {
            songs = decodeHtml(songs);

            searchTerm = "listen/";
            closingTerm = "/";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString code = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<span class=\"song\">";
            closingTerm = "</span>";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString title = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<span class=\"group\">";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString group = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<li class=\"length radius_3\">";
            closingTerm = "</li>";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString length = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<li class=\"description\">";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString comment = songs.mid(termBegins, termEnds - termBegins);

//            searchTerm = "<p class=\"comment\">";
//            closingTerm = "</p>";
//            termBegins = songs.indexOf(searchTerm, listenBegins);
//            termEnds = songs.indexOf(closingTerm, termBegins);

//            QString comment = songs.mid(termBegins + searchTerm.length(), termEnds - termBegins
//                                        - searchTerm.length()).simplified();

            qDebug() << "Song found " << title << group << length << comment << code;
            emit songFound(title, group, length, comment, code);

            songs = songs.mid(songs.indexOf("<li>", termEnds));
//            download(DOWNLOAD_URL + code);

//            songs = songs.mid(listenEnds) + 1;
//            exit(0);
        }
    } else if (mimeType == "text/xml") {
        QString link = reply->readAll();

        int linkBegins = link.indexOf("path=\"") + 6;
        int linkEnds = link.indexOf("\"", linkBegins);
        link = link.mid(linkBegins, linkEnds - linkBegins);

        emit decodedUrl(reply->url().toString().replace(DownloadUrl, ""), link);
    } else if (mimeType == "audio/mpeg") {
        qDebug() << "Writing file";
        QString name(m_songsToDownload.value(reply->url().toString()).toString());

#ifdef Q_OS_BLACKBERRY
        name = "shared/music/" + name;
#endif

#ifdef Q_OS_ANDROID
        QDir musicDir("/sdcard/Music");
        if (!musicDir.exists())
            musicDir.mkdir("/sdcard/Music");

        name = "/sdcard/Music/" + name;
#endif

        bool permission;
        QFile file(name + reply->url().toString().mid(reply->url().toString().lastIndexOf(".")));
        permission = file.open(QIODevice::WriteOnly);<commit_msg>Proper naming when url is redirecting.<commit_after>#include "downloader.h"

#include <QDebug>
#include <QtNetwork>
#include <QtXml>
#include <QDir>
#include <QFile>

#define TARGET_URL "http://www.goear.com/"
#define SEARCH_URL "http://www.goear.com/search/"
QString Downloader::DownloadUrl("http://www.goear.com/action/sound/get/");

Downloader::Downloader(QObject *parent) :
    QObject(parent)
{
    m_netAccess = new QNetworkAccessManager(this);

    connect(m_netAccess, SIGNAL(finished(QNetworkReply*)),
            this, SLOT(downloadFinished(QNetworkReply*)));
}

Downloader::~Downloader()
{
    delete m_netAccess;
}

void Downloader::downloadSong(const QString &name, const QString &url)
{
    m_songsToDownload.insert(url, name);
    download(url);
}

void Downloader::download(const QString &urlString)
{
    QUrl url(urlString);
    QNetworkRequest request(url);
    m_nreply = m_netAccess->get(request);
    connect(m_nreply, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(downloadProgressChanged(qint64, qint64)));
}

void Downloader::search(const QString &term)
{
    qDebug() << Q_FUNC_INFO << " " + term;
    download(SEARCH_URL + term);
}

void Downloader::downloadFinished(QNetworkReply *reply)
{
    if (reply->url().toString().startsWith(SEARCH_URL))
        emit searchEnded();

    if (reply->error() != QNetworkReply::NoError) {
        qDebug() << Q_FUNC_INFO << " error downloading "
                 << reply->url().toString() << ":" << reply->errorString();

        if (reply->errorString().contains("Server Error"))
            emit serverError();

        reply->deleteLater();
        return;
    }

    qDebug() << reply->url().toString() << " has been dowloaded";

    QVariant redir = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);

    if (redir.isValid()) {
        QUrl url = redir.toUrl();
        QString name = m_songsToDownload.value(reply->url().toString()).toString();
        m_songsToDownload.remove(reply->url().toString());
        m_songsToDownload.insert(url.toString(), name);
        qDebug() << reply->url().toString() << " was redirected to:" << url.toString();

        if (url.isRelative()) {
            url.setScheme(reply->url().scheme());
            /// TODO: check why this doesn't work
            //            url.setEncodedHost(reply->url().encodedHost());
        }

        download(url.toString());
        reply->deleteLater();
        return;
    }

    QString mimeType(reply->header(QNetworkRequest::ContentTypeHeader).toString());

    if (mimeType.contains("text/html")) {
        QString songs(reply->readAll());

        int count = songs.count("listen/");

        QString hasMore = "";
        QString searchTerm = "<ol id=\"new_pagination\">";
        QString closingTerm = "</ol>";

        int termBegins = songs.indexOf(searchTerm);
        int termEnds = songs.indexOf(closingTerm, termBegins);

        if (termBegins != -1) {
            hasMore = songs.mid(termBegins + searchTerm.length(), termEnds - termBegins
                                - searchTerm.length());
        }

        QStringList list = hasMore.simplified().split("<li", QString::SkipEmptyParts);

        for (int i = 0; i < list.length() - 1; ++i) {
            if (list.at(i).simplified().startsWith("class=\"active\">")) {
                emit searchHasMoreResults(reply->url().toString() + "/"
                                          + QString::number(++i));

                break;
            }
        }

        searchTerm = "<ol id=\"search_results\">";
        closingTerm = "</ol>";

        termBegins = songs.indexOf(searchTerm);
        termEnds = songs.indexOf(closingTerm, termBegins);

        songs = songs.mid(termBegins + searchTerm.length(),
                          termEnds - termBegins);

        for (int i = 0; i < count; ++i) {
            songs = decodeHtml(songs);

            searchTerm = "listen/";
            closingTerm = "/";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString code = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<span class=\"song\">";
            closingTerm = "</span>";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString title = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<span class=\"group\">";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString group = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<li class=\"length radius_3\">";
            closingTerm = "</li>";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString length = songs.mid(termBegins, termEnds - termBegins);

            searchTerm = "<li class=\"description\">";

            termBegins = songs.indexOf(searchTerm) + searchTerm.length();
            termEnds = songs.indexOf(closingTerm, termBegins);

            QString comment = songs.mid(termBegins, termEnds - termBegins);

//            searchTerm = "<p class=\"comment\">";
//            closingTerm = "</p>";
//            termBegins = songs.indexOf(searchTerm, listenBegins);
//            termEnds = songs.indexOf(closingTerm, termBegins);

//            QString comment = songs.mid(termBegins + searchTerm.length(), termEnds - termBegins
//                                        - searchTerm.length()).simplified();

            qDebug() << "Song found " << title << group << length << comment << code;
            emit songFound(title, group, length, comment, code);

            songs = songs.mid(songs.indexOf("<li>", termEnds));
//            download(DOWNLOAD_URL + code);

//            songs = songs.mid(listenEnds) + 1;
//            exit(0);
        }
    } else if (mimeType == "text/xml") {
        QString link = reply->readAll();

        int linkBegins = link.indexOf("path=\"") + 6;
        int linkEnds = link.indexOf("\"", linkBegins);
        link = link.mid(linkBegins, linkEnds - linkBegins);

        emit decodedUrl(reply->url().toString().replace(DownloadUrl, ""), link);
    } else if (mimeType == "audio/mpeg") {
        qDebug() << "Writing file";
        QString name(m_songsToDownload.value(reply->url().toString()).toString());
        m_songsToDownload.remove(reply->url().toString());

		name.replace("/", "-");

#ifdef Q_OS_BLACKBERRY
        name = "shared/music/" + name;
#endif

#ifdef Q_OS_ANDROID
        QDir musicDir("/sdcard/Music");
        if (!musicDir.exists())
            musicDir.mkdir("/sdcard/Music");

        name = "/sdcard/Music/" + name;
#endif

        bool permission;
        QFile file(name + reply->url().toString().mid(reply->url().toString().lastIndexOf(".")));
        permission = file.open(QIODevice::WriteOnly);
{'fraction_non_alphanumeric': 0.15361794500723588, 'fraction_numerical': 0.0023154848046309695, 'mean_word_length': 2.469997489329651, 'pattern_counts': {'":': 2, '<': 121, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469115', 'n_tokens_mistral': 4145, 'n_tokens_neox': 4034, 'n_words': 752}