<commit_before>#include <QDebug>

#include <sstream>
#include <iostream>

#include "DownloadManager.h"

#include "Util.h"
#include "Downloader.h"
using namespace efdl;

DownloadManager::DownloadManager(bool connProg)
  : connProg{connProg}, chksum{false}, conns{0}, chunksAmount{0},
  chunksFinished{0}, size{0}, offset{0}, bytesDown{0},
  hashAlg{QCryptographicHash::Sha3_512}, downloader{nullptr}
{ }

DownloadManager::~DownloadManager() {
  cleanup();
  qDeleteAll(queue);
}

void DownloadManager::add(Downloader *entry) {
  queue << entry;
}

void DownloadManager::createChecksum(QCryptographicHash::Algorithm hashAlg) {
  this->hashAlg = hashAlg;
  chksum = true;
}

void DownloadManager::start() {
  next();
}

void DownloadManager::next() {
  static bool first{true};
  if (!first) {
    // Update progress with total download time with no connection
    // lines.
    bool tmp{connProg};
    connProg = false;
    updateProgress();
    connProg = tmp;

    if (chksum) printChecksum();

    // Separate each download with a newline.
    if (!queue.isEmpty()) qDebug();
  }
  if (first) first = false;
  
  if (queue.isEmpty()) {
    emit finished();
    return;
  }

  cleanup();

  downloader = queue.dequeue();
  qDebug() << "Downloading" << qPrintable(downloader->getUrl().toString());
  connect(downloader, &Downloader::finished, this, &DownloadManager::next);
  connect(downloader, &Downloader::information,
          this, &DownloadManager::onInformation);
  connect(downloader, &Downloader::chunkStarted,
          this, &DownloadManager::onChunkStarted);
  connect(downloader, &Downloader::chunkProgress,
          this, &DownloadManager::onChunkProgress);
  connect(downloader, &Downloader::chunkFinished,
          this, &DownloadManager::onChunkFinished);
  connect(downloader, &Downloader::chunkFailed,
          this, &DownloadManager::onChunkFailed);
  downloader->start();

  started = QDateTime::currentDateTime();
}
void DownloadManager::onInformation(const QString &outputPath, qint64 size,
                                    int chunksAmount, int conns,
                                    qint64 offset) {
  //qDebug() << "CHUNKS AMOUNT:" << total;
  this->outputPath = outputPath;
  this->size = size;
  this->chunksAmount = chunksAmount;
  this->conns = conns;
  this->offset = offset;
  updateProgress();
}

void DownloadManager::onChunkStarted(int num) {
  //qDebug() << "CHUNK STARTED:" << num;
  connsMap[num] = Range{0, 0};
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkProgress(int num, qint64 received, qint64 total) {
  //qDebug() << "CHUNK PROGRESS:" << num << received << total;
  connsMap[num] = Range{received, total};
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkFinished(int num, Range range) {
  //qDebug() << "CHUNK FINISHED:" << num << range;
  chunksFinished++;
  bytesDown += (range.second - range.first + 1);
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkFailed(int num, Range range, int httpCode,
                                    QNetworkReply::NetworkError error) {
  //qDebug() << "CHUNK FAILED:" << num << range << httpCode << error;
}

void DownloadManager::cleanup() {
  chunksAmount = chunksFinished = size = offset = bytesDown = 0;<commit_msg>Clear connections map and removed unused debug info.<commit_after>#include <QDebug>

#include <sstream>
#include <iostream>

#include "DownloadManager.h"

#include "Util.h"
#include "Downloader.h"
using namespace efdl;

DownloadManager::DownloadManager(bool connProg)
  : connProg{connProg}, chksum{false}, conns{0}, chunksAmount{0},
  chunksFinished{0}, size{0}, offset{0}, bytesDown{0},
  hashAlg{QCryptographicHash::Sha3_512}, downloader{nullptr}
{ }

DownloadManager::~DownloadManager() {
  cleanup();
  qDeleteAll(queue);
}

void DownloadManager::add(Downloader *entry) {
  queue << entry;
}

void DownloadManager::createChecksum(QCryptographicHash::Algorithm hashAlg) {
  this->hashAlg = hashAlg;
  chksum = true;
}

void DownloadManager::start() {
  next();
}

void DownloadManager::next() {
  static bool first{true};
  if (!first) {
    // Update progress with total download time with no connection
    // lines.
    bool tmp{connProg};
    connProg = false;
    updateProgress();
    connProg = tmp;

    if (chksum) printChecksum();

    // Separate each download with a newline.
    if (!queue.isEmpty()) qDebug();
  }
  if (first) first = false;
  
  if (queue.isEmpty()) {
    emit finished();
    return;
  }

  cleanup();

  downloader = queue.dequeue();
  qDebug() << "Downloading" << qPrintable(downloader->getUrl().toString());
  connect(downloader, &Downloader::finished, this, &DownloadManager::next);
  connect(downloader, &Downloader::information,
          this, &DownloadManager::onInformation);
  connect(downloader, &Downloader::chunkStarted,
          this, &DownloadManager::onChunkStarted);
  connect(downloader, &Downloader::chunkProgress,
          this, &DownloadManager::onChunkProgress);
  connect(downloader, &Downloader::chunkFinished,
          this, &DownloadManager::onChunkFinished);
  connect(downloader, &Downloader::chunkFailed,
          this, &DownloadManager::onChunkFailed);
  downloader->start();

  started = QDateTime::currentDateTime();
}
void DownloadManager::onInformation(const QString &outputPath, qint64 size,
                                    int chunksAmount, int conns,
                                    qint64 offset) {
  this->outputPath = outputPath;
  this->size = size;
  this->chunksAmount = chunksAmount;
  this->conns = conns;
  this->offset = offset;
  updateProgress();
}

void DownloadManager::onChunkStarted(int num) {
  connsMap[num] = Range{0, 0};
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkProgress(int num, qint64 received, qint64 total) {
  connsMap[num] = Range{received, total};
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkFinished(int num, Range range) {
  chunksFinished++;
  bytesDown += (range.second - range.first + 1);
  updateConnsMap();
  updateProgress();
}

void DownloadManager::onChunkFailed(int num, Range range, int httpCode,
                                    QNetworkReply::NetworkError error) {
  //qDebug() << "CHUNK FAILED:" << num << range << httpCode << error;
}

void DownloadManager::cleanup() {
  chunksAmount = chunksFinished = size = offset = bytesDown = 0;
  connsMap.clear();
{'fraction_non_alphanumeric': 0.13921294135946494, 'fraction_numerical': 0.0068439881785658735, 'mean_word_length': 4.091053048297704, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 90, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675798', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1969, 'n_words': 430}