<commit_before>#include "httpdownload.h"
#include "../../log/logger.h"
#include "types.h"

#include <QtMath>
#include <numeric>

LOGGER(HTTPDownload);

DownloadThread::DownloadThread (const QUrl &url, const QHostInfo &server, int targetTimeMs, bool cacheTest, QObject *parent)
: QObject(parent)
, url(url)
, server(server)
, targetTime(targetTimeMs)
, avoidCaches(cacheTest)
, socket(NULL)
, tStatus(Inactive)
{
}

DownloadThread::~DownloadThread()
{
    //socket needs to be deleted
    if (socket != NULL)
    {
        //shouldn't happen, but in case it does we are good
        //Internet citizens and behave nicely
        if (socket->state() == QAbstractSocket::ConnectedState)
        {
            socket->close();
        }

        delete socket;
    }
}

DownloadThread::DownloadThreadStatus DownloadThread::threadStatus() const
{
    return tStatus;
}

qint64 DownloadThread::timeToFirstByteInNs() const
{
    return timeToFirstByte;
}

qint64 DownloadThread::startTimeInNs() const
{
    return startTime.toMSecsSinceEpoch() * 1000000;
}

qint64 DownloadThread::endTimeInNs() const
{
    return startTime.toMSecsSinceEpoch() * 1000000 + timeIntervals.last();
}

qint64 DownloadThread::runTimeInNs() const
{
    return timeIntervals.last();
}

void DownloadThread::startTCPConnection()
{
    //each Thread is supposed to first build up the TCP connection,
    //emit the connected signal, and only when all threads have connected
    //do the actual download (coordinated by HTTPDownload)

    //shouldn't happen, check anyway
    if (server.addresses().isEmpty() || (!url.isValid()))
    {
        //invoke the connection tracking code
        tStatus = FinishedError;
        return;
    }

    socket = new QTcpSocket();

    //so let's connect now (if no port as part of the URL use 80 as default)
    socket->connectToHost(server.addresses().first(), url.port(defaultPort));

    tStatus = ConnectingTCP;

    //wait for up to 5 seconds for a successful connection
    if (socket->waitForConnected(tcpConnectTimeout))
    {
        //for the successfully connected sockets, we should track the disconnection
        connect(socket, &QTcpSocket::disconnected, this, &DownloadThread::disconnectionHandling);
        tStatus = ConnectedTCP;
        emit TCPConnected(true);
    }
    else
    {
        //something went wrong
        tStatus = FinishedError;
        emit TCPConnected(false);
        socket->close();
    }
}

void DownloadThread::disconnectionHandling()
{
    // handling premature TCP disconnects
    if(tStatus == DownloadInProgress)
    {
        tStatus = FinishedSuccess;
        emit TCPDisconnected();
    }
}

void DownloadThread::startDownload()
{
    //we can only download, if this thread sucessfully established the
    //TCP connection (all threads will receive the signal)
    if(socket->state() != QAbstractSocket::ConnectedState)
    {
        tStatus = FinishedError;
        emit firstByteReceived(false);
        return;
    }

    QString path = url.path();

    if (avoidCaches)
    {
        path.append(QString("?timestamp=%1").arg(QDateTime::currentDateTime().toString("yy_MM_dd_HH_mm_ss_zzz")));
    }

    QString request = QString("GET %1 HTTP/1.1\r\n"
                              "Host: %2\r\n"
                              "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:31.0) Gecko/20100101 Firefox/31.0\r\n"
                              "Referer: http://www.measure-it.net\r\n\r\n").arg(path).arg(url.host());

    int bytesWritten = 0;

    //log actual time of start
    startTime = QDateTime::currentDateTime();

    //send the HTTP GET
    while(bytesWritten < request.length())
    {
        bytesWritten += socket->write(request.mid(bytesWritten).toLatin1());

        //on error
        if(bytesWritten < 0)
        {
            socket->close();
            tStatus = FinishedError;
            emit firstByteReceived(false);
            return;
        }
    }

    //start eplapsed timer for calculating the time slots
    measurementTimer.start();

    tStatus = AwaitingFirstByte;

    //do a blocking read for the first bytes or time-out if nothing is arriving
    if (socket->waitForReadyRead(firstByteReceivedTimeout))
    {
        //TODO: check for HTTP status code and act intelligently on it
        //currently, a 404 etc. is simply to little data
        //to generate results, but a better checking would be great
        timeToFirstByte = measurementTimer.nsecsElapsed();
        tStatus = DownloadInProgress;
        //connect readyRead of the socket with read() for further reads
        connect(socket, &QTcpSocket::readyRead, this, &DownloadThread::read);
        //call read
        read();
        emit firstByteReceived(true);
    }
    else
    {
        tStatus = FinishedError;
        socket->close();
        emit firstByteReceived(false);<commit_msg>Two fixes for the HTTPDownload<commit_after>#include "httpdownload.h"
#include "../../log/logger.h"
#include "types.h"

#include <QtMath>
#include <numeric>

LOGGER(HTTPDownload);

DownloadThread::DownloadThread (const QUrl &url, const QHostInfo &server, int targetTimeMs, bool cacheTest, QObject *parent)
: QObject(parent)
, url(url)
, server(server)
, targetTime(targetTimeMs)
, avoidCaches(cacheTest)
, socket(NULL)
, tStatus(Inactive)
{
}

DownloadThread::~DownloadThread()
{
    //socket needs to be deleted
    if (socket != NULL)
    {
        //shouldn't happen, but in case it does we are good
        //Internet citizens and behave nicely
        if (socket->state() == QAbstractSocket::ConnectedState)
        {
            socket->close();
        }

        delete socket;
    }
}

DownloadThread::DownloadThreadStatus DownloadThread::threadStatus() const
{
    return tStatus;
}

qint64 DownloadThread::timeToFirstByteInNs() const
{
    return timeToFirstByte;
}

qint64 DownloadThread::startTimeInNs() const
{
    return startTime.toMSecsSinceEpoch() * 1000000;
}

qint64 DownloadThread::endTimeInNs() const
{
    return startTime.toMSecsSinceEpoch() * 1000000 + timeIntervals.last();
}

qint64 DownloadThread::runTimeInNs() const
{
    return timeIntervals.last();
}

void DownloadThread::startTCPConnection()
{
    //each Thread is supposed to first build up the TCP connection,
    //emit the connected signal, and only when all threads have connected
    //do the actual download (coordinated by HTTPDownload)

    //shouldn't happen, check anyway
    if (server.addresses().isEmpty() || (!url.isValid()))
    {
        //invoke the connection tracking code
        tStatus = FinishedError;
        emit TCPConnected(false);
        return;
    }

    socket = new QTcpSocket();

    //so let's connect now (if no port as part of the URL use 80 as default)
    socket->connectToHost(server.addresses().first(), url.port(defaultPort));

    tStatus = ConnectingTCP;

    //wait for up to 5 seconds for a successful connection
    if (socket->waitForConnected(tcpConnectTimeout))
    {
        //for the successfully connected sockets, we should track the disconnection
        connect(socket, &QTcpSocket::disconnected, this, &DownloadThread::disconnectionHandling);
        tStatus = ConnectedTCP;
        emit TCPConnected(true);
    }
    else
    {
        //something went wrong
        tStatus = FinishedError;
        emit TCPConnected(false);
        socket->close();
    }
}

void DownloadThread::disconnectionHandling()
{
    // handling premature TCP disconnects
    if(tStatus == DownloadInProgress)
    {
        tStatus = FinishedSuccess;
        emit TCPDisconnected();
    }
}

void DownloadThread::startDownload()
{
    //we can only download, if this thread sucessfully established the
    //TCP connection (all threads will receive the signal)
    if(socket->state() != QAbstractSocket::ConnectedState)
    {
        tStatus = FinishedError;
        emit firstByteReceived(false);
        return;
    }

    QString path = url.path();

    if (avoidCaches)
    {
        path.append(QString("?timestamp=%1").arg(QDateTime::currentDateTime().toString("yy_MM_dd_HH_mm_ss_zzz")));
    }

    QString request = QString("GET %1 HTTP/1.1\r\n"
                              "Host: %2\r\n"
                              "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:31.0) Gecko/20100101 Firefox/31.0\r\n"
                              "Referer: http://www.measure-it.net\r\n\r\n").arg(path).arg(url.host());

    qint64 bytesWritten = 0;

    //log actual time of start
    startTime = QDateTime::currentDateTime();

    //send the HTTP GET
    while(bytesWritten < request.length())
    {
        qint64 writeResult = socket->write(request.mid(bytesWritten).toLatin1());

        //on error
        if(writeResult < 0)
        {
            socket->close();
            tStatus = FinishedError;
            emit firstByteReceived(false);
            return;
        }

        bytesWritten += writeResult;
    }

    //start eplapsed timer for calculating the time slots
    measurementTimer.start();

    tStatus = AwaitingFirstByte;

    //do a blocking read for the first bytes or time-out if nothing is arriving
    if (socket->waitForReadyRead(firstByteReceivedTimeout))
    {
        //TODO: check for HTTP status code and act intelligently on it
        //currently, a 404 etc. is simply to little data
        //to generate results, but a better checking would be great
        timeToFirstByte = measurementTimer.nsecsElapsed();
        tStatus = DownloadInProgress;
        //connect readyRead of the socket with read() for further reads
        connect(socket, &QTcpSocket::readyRead, this, &DownloadThread::read);
        //call read
        read();
        emit firstByteReceived(true);
    }
    else
    {
        tStatus = FinishedError;
        socket->close();
        emit firstByteReceived(false);
{'fraction_non_alphanumeric': 0.10818237329533889, 'fraction_numerical': 0.011601872582943212, 'mean_word_length': 2.7251705837755877, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2005059', 'n_tokens_mistral': 2907, 'n_tokens_neox': 2788, 'n_words': 760}