<commit_before>#include "QNETag.h"
#include <QString>
#include <QFile>
#include <QByteArray>
#include <QCryptographicHash>
#include <QFileInfo>
#include <QDataStream>
#include <QDebug>
QNETag::QNETag()
{
}
QNETag::~QNETag()
{
}

QString QNETag::UrlSafeBase64Encode(QByteArray data)
{
    QByteArray encodedData = data.toBase64();
    QString encodedStr = QString(encodedData);
    encodedStr = encodedStr.replace("+","-").replace("/","_");
    return encodedStr;
}

QByteArray QNETag::Sha1(QByteArray data)
{
    QCryptographicHash sha1Hash(QCryptographicHash::Sha1);
    sha1Hash.addData(data);
    return sha1Hash.result();
}

/*
    This method will return an non-empty string if the file exists and readable
*/
QString QNETag::CalcETag(QString fileName)
{
    QString etag = "";
    QFileInfo fi(fileName);
    if(fi.exists() && fi.permission(QFile::ReadUser))
    {
        QFile fh(fileName);
        bool opened=fh.open(QIODevice::ReadOnly);
        if (opened){
            qint64 fileLen = fh.size();
            if(fileLen <= CHUNK_SIZE)
            {
                QByteArray fileData=fh.readAll();
                QByteArray sha1Data=Sha1(fileData);
                QByteArray hashData=sha1Data;
                hashData.insert(0,0x16);
                etag=UrlSafeBase64Encode(hashData);
            }
            else
            {
                int chunkCount=fileLen/CHUNK_SIZE;
                if(fileLen%CHUNK_SIZE!=0)
                {
                    chunkCount+=1;
                }
                QByteArray sha1AllData;
                for(int i=0;i<chunkCount;i++)
                {
                    qint64 chunkSize=CHUNK_SIZE;
                    if(i==chunkCount-1)
                    {
                        chunkSize=fileLen-(chunkCount-1)*CHUNK_SIZE;
                    }
                    QByteArray buf=QByteArray(chunkSize,' ');
                    qint64 readCnt=fh.read(buf.data(),chunkSize);
                    fh.seek(readCnt);
                    QByteArray sha1Data=Sha1(buf);
                    sha1AllData.append(sha1Data);
                }
                QByteArray hashData=Sha1(sha1AllData);
                hashData.insert(0,0x96);
                etag=UrlSafeBase64Encode(hashData);
            }
            fh.close();
        }
    }
    return etag;
}<commit_msg>Fix a bug in etag calc.<commit_after>#include "QNETag.h"
#include <QString>
#include <QFile>
#include <QByteArray>
#include <QCryptographicHash>
#include <QFileInfo>
#include <QDataStream>
#include <QDebug>
QNETag::QNETag()
{
}
QNETag::~QNETag()
{
}

QString QNETag::UrlSafeBase64Encode(QByteArray data)
{
    QByteArray encodedData = data.toBase64();
    QString encodedStr = QString(encodedData);
    encodedStr = encodedStr.replace("+","-").replace("/","_");
    return encodedStr;
}

QByteArray QNETag::Sha1(QByteArray data)
{
    QCryptographicHash sha1Hash(QCryptographicHash::Sha1);
    sha1Hash.addData(data);
    return sha1Hash.result();
}

/*
    This method will return an non-empty string if the file exists and readable
*/
QString QNETag::CalcETag(QString fileName)
{
    QString etag = "";
    QFileInfo fi(fileName);
    if(fi.exists() && fi.permission(QFile::ReadUser))
    {
        QFile fh(fileName);
        bool opened = fh.open(QIODevice::ReadOnly);
        if (opened){
            qint64 fileLen = fh.size();
            if(fileLen <= CHUNK_SIZE)
            {
                QByteArray fileData = fh.readAll();
                QByteArray sha1Data = Sha1(fileData);
                QByteArray hashData = sha1Data;
                hashData.insert(0,0x16);
                etag = UrlSafeBase64Encode(hashData);
            }
            else
            {
                int chunkCount = fileLen / CHUNK_SIZE;
                if(fileLen % CHUNK_SIZE != 0)
                {
                    chunkCount += 1;
                }
                QByteArray sha1AllData;
                for(int i=0; i<chunkCount; i++)
                {
                    qint64 chunkSize = CHUNK_SIZE;
                    if(i == chunkCount-1)
                    {
                        chunkSize = fileLen-(chunkCount-1)*CHUNK_SIZE;
                    }
                    QByteArray buf = QByteArray(chunkSize,' ');
                    fh.read(buf.data(),chunkSize);
                    QByteArray sha1Data = Sha1(buf);
                    sha1AllData.append(sha1Data);
                }
                QByteArray hashData = Sha1(sha1AllData);
                hashData.insert(0,0x96);
                etag = UrlSafeBase64Encode(hashData);
            }
            fh.close();
        }
    }
    return etag;
}
{'fraction_non_alphanumeric': 0.11151619129315891, 'fraction_numerical': 0.017585245550075058, 'mean_word_length': 1.7597633136094675, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '161777', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1427, 'n_words': 238}