<commit_before>//////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2007-2017 musikcube team
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//    * Redistributions of source code must retain the above copyright notice,
//      this list of conditions and the following disclaimer.
//
//    * Redistributions in binary form must reproduce the above copyright
//      notice, this list of conditions and the following disclaimer in the
//      documentation and/or other materials provided with the distribution.
//
//    * Neither the name of the author nor the names of other contributors may
//      be used to endorse or promote products derived from this software
//      without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////////

#include "pch.hpp"

#include <core/library/track/IndexerTrack.h>

#include <core/support/Common.h>
#include <core/db/Connection.h>
#include <core/db/Statement.h>
#include <core/library/LocalLibrary.h>
#include <core/io/DataStreamFactory.h>

#include <boost/lexical_cast.hpp>
#include <unordered_map>

using namespace musik::core;
using namespace musik::core::sdk;

#define GENRE_TRACK_COLUMN_NAME "genre"
#define GENRES_TABLE_NAME "genres"
#define GENRE_TRACK_JUNCTION_TABLE_NAME "track_genres"
#define GENRE_TRACK_FOREIGN_KEY "genre_id"

#define ARTIST_TRACK_COLUMN_NAME "artist"
#define ARTISTS_TABLE_NAME "artists"
#define ARTIST_TRACK_JUNCTION_TABLE_NAME "track_artists"
#define ARTIST_TRACK_FOREIGN_KEY "artist_id"

std::mutex IndexerTrack::sharedWriteMutex;
static std::unordered_map<std::string, int64_t> metadataIdCache;

void IndexerTrack::ResetIdCache() {
    metadataIdCache.clear();
}

IndexerTrack::IndexerTrack(int64_t id)
: internalMetadata(new IndexerTrack::InternalMetadata())
, id(id)
{
}

IndexerTrack::~IndexerTrack() {
    delete this->internalMetadata;
    this->internalMetadata  = nullptr;
}

std::string IndexerTrack::GetString(const char* metakey) {
    if (metakey && this->internalMetadata) {
        MetadataMap::iterator metavalue = this->internalMetadata->metadata.find(metakey);
        if (metavalue != this->internalMetadata->metadata.end()) {
            return metavalue->second;
        }
    }

    return "";
}

long long IndexerTrack::GetInt64(const char* key, long long defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stoll(GetString(key));
        }
    } catch (...) {
    }
    return defaultValue;
}

int IndexerTrack::GetInt32(const char* key, unsigned int defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stol(GetString(key));
        }
    }
    catch (...) {
    }
    return defaultValue;
}

double IndexerTrack::GetDouble(const char* key, double defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stod(GetString(key));
        }
    } catch (...) {
    }
    return defaultValue;
}

void IndexerTrack::SetValue(const char* metakey, const char* value) {
    if (metakey && value) {
        this->internalMetadata->metadata.insert(
            std::pair<std::string, std::string>(metakey,value));
    }
}

void IndexerTrack::ClearValue(const char* metakey) {
    if (this->internalMetadata) {
        this->internalMetadata->metadata.erase(metakey);
    }
}

bool IndexerTrack::Contains(const char* metakey) {
    auto md = this->internalMetadata;
    return md && md->metadata.find(metakey) != md->metadata.end();
}

void IndexerTrack::SetThumbnail(const char *data, long size) {
    if (this->internalMetadata->thumbnailData) {
        delete[] this->internalMetadata->thumbnailData;
    }

    this->internalMetadata->thumbnailData = new char[size];
    this->internalMetadata->thumbnailSize = size;

    memcpy(this->internalMetadata->thumbnailData, data, size);
}

void IndexerTrack::SetReplayGain(const ReplayGain& replayGain) {
    this->internalMetadata->replayGain.reset();
    this->internalMetadata->replayGain = std::make_shared<ReplayGain>();
    memcpy(this->internalMetadata->replayGain.get(), &replayGain, sizeof(ReplayGain));
}

std::string IndexerTrack::Uri() {
    return this->GetString("filename");
}

int IndexerTrack::GetString(const char* key, char* dst, int size) {
    return CopyString(this->GetString(key), dst, size);
}

int IndexerTrack::Uri(char* dst, int size) {
    return CopyString(this->Uri(), dst, size);
}

Track::MetadataIteratorRange IndexerTrack::GetValues(const char* metakey) {
    if (this->internalMetadata) {
        return this->internalMetadata->metadata.equal_range(metakey);
    }

    return Track::MetadataIteratorRange();
}

Track::MetadataIteratorRange IndexerTrack::GetAllValues() {
    if (this->internalMetadata) {
        return Track::MetadataIteratorRange(
            this->internalMetadata->metadata.begin(),
            this->internalMetadata->metadata.end());
    }

    return Track::MetadataIteratorRange();
}

int64_t IndexerTrack::GetId() {
    return this->id;
}

bool IndexerTrack::NeedsToBeIndexed(
    const boost::filesystem::path &file,
    db::Connection &dbConnection)
{
    try {
        this->SetValue("path", file.string().c_str());
        this->SetValue("filename", file.string().c_str());

        size_t lastDot = file.leaf().string().find_last_of(".");
        if (lastDot != std::string::npos) {
            this->SetValue("extension", file.leaf().string().substr(lastDot + 1).c_str());
        }

        size_t fileSize = (size_t) boost::filesystem::file_size(file);
        size_t fileTime = (size_t) boost::filesystem::last_write_time(file);

        this->SetValue("filesize", boost::lexical_cast<std::string>(fileSize).c_str());
        this->SetValue("filetime", boost::lexical_cast<std::string>(fileTime).c_str());

        db::Statement stmt(
            "SELECT id, filename, filesize, filetime " \
            "FROM tracks t " \
            "WHERE filename=?", dbConnection);

        stmt.BindText(0, this->GetString("filename"));

        bool fileDifferent = true;

        if (stmt.Step() == db::Row) {
            this->id = stmt.ColumnInt64(0);
            int dbFileSize = stmt.ColumnInt32(2);
            int dbFileTime = stmt.ColumnInt32(3);

            if (fileSize == dbFileSize && fileTime == dbFileTime) {
                return false;
            }
        }
    }
    catch (...) {
    }

    return true;
}

static int64_t writeToTracksTable(
    db::Connection &dbConnection,
    IndexerTrack& track)
{
    std::string externalId = track.GetString("external_id");
    int64_t id = track.GetId();

    if (externalId.size() == 0) {
        return 0;
    }

    int sourceId = track.GetInt32("source_id", 0);

    /* if there's no ID specified, but we have an external ID, let's
    see if we can find the corresponding ID. this can happen when
    IInputSource plugins are reading/writing track data. */
    if (id == 0) {
        if (sourceId == 0) {
            db::Statement stmt("SELECT id FROM tracks WHERE source_id=? AND external_id=?", dbConnection);
            stmt.BindInt32(0, sourceId);
            stmt.BindText(1, externalId);
            if (stmt.Step() == db::Row) {
                track.SetId(stmt.ColumnInt64(0));
            }
        }
        else {
            std::string fn = track.GetString("filename");
            if (fn.size()) {
                db::Statement stmt("SELECT id, external_id FROM tracks WHERE filename=?", dbConnection);
                stmt.BindText(0, track.GetString("filename"));
                if (stmt.Step() == db::Row) {
                    id = stmt.ColumnInt64(0);
                    track.SetId(id);
                    track.SetValue("external_id", stmt.ColumnText(1));
                }
            }
        }
    }

    std::string query;

    if (id != 0) {
        query =
            "UPDATE tracks "
            "SET track=?, disc=?, bpm=?, duration=?, filesize=?, "
            "    title=?, filename=?, filetime=?, path_id=?, external_id=? "
            "WHERE id=?";
    }
    else {
        query =
            "INSERT INTO tracks "
            "(track, disc, bpm, duration, filesize, title, filename, filetime, path_id, external_id) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
    }

    db::Statement stmt(query.c_str(), dbConnection);

    stmt.BindText(0, track.GetString("track"));
    stmt.BindText(1, track.GetString("disc"));
    stmt.BindText(2, track.GetString("bpm"));
    stmt.BindInt32(3, track.GetInt32("duration"));
    stmt.BindInt32(4, track.GetInt32("filesize"));
    stmt.BindText(5, track.GetString("title"));
    stmt.BindText(6, track.GetString("filename"));
    stmt.BindInt32(7, track.GetInt32("filetime"));
    stmt.BindInt64(8, track.GetInt64("path_id"));
    stmt.BindText(9, track.GetString("external_id"));

    if (id != 0) {
        stmt.BindInt64(11, id);
    }

    if (stmt.Step() == db::Done) {
        if (id == 0) {
            return dbConnection.LastInsertedId();
        }
    }

    return id;
}
<commit_msg>Fixed bug in IndexerTrack update logic -- sometimes some properties weren't getting properly saved (fallout from removal of 'year' column in 'tracks' table).<commit_after>//////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2007-2017 musikcube team
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//    * Redistributions of source code must retain the above copyright notice,
//      this list of conditions and the following disclaimer.
//
//    * Redistributions in binary form must reproduce the above copyright
//      notice, this list of conditions and the following disclaimer in the
//      documentation and/or other materials provided with the distribution.
//
//    * Neither the name of the author nor the names of other contributors may
//      be used to endorse or promote products derived from this software
//      without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////////

#include "pch.hpp"

#include <core/library/track/IndexerTrack.h>

#include <core/support/Common.h>
#include <core/db/Connection.h>
#include <core/db/Statement.h>
#include <core/library/LocalLibrary.h>
#include <core/io/DataStreamFactory.h>

#include <boost/lexical_cast.hpp>
#include <unordered_map>

using namespace musik::core;
using namespace musik::core::sdk;

#define GENRE_TRACK_COLUMN_NAME "genre"
#define GENRES_TABLE_NAME "genres"
#define GENRE_TRACK_JUNCTION_TABLE_NAME "track_genres"
#define GENRE_TRACK_FOREIGN_KEY "genre_id"

#define ARTIST_TRACK_COLUMN_NAME "artist"
#define ARTISTS_TABLE_NAME "artists"
#define ARTIST_TRACK_JUNCTION_TABLE_NAME "track_artists"
#define ARTIST_TRACK_FOREIGN_KEY "artist_id"

std::mutex IndexerTrack::sharedWriteMutex;
static std::unordered_map<std::string, int64_t> metadataIdCache;

void IndexerTrack::ResetIdCache() {
    metadataIdCache.clear();
}

IndexerTrack::IndexerTrack(int64_t id)
: internalMetadata(new IndexerTrack::InternalMetadata())
, id(id)
{
}

IndexerTrack::~IndexerTrack() {
    delete this->internalMetadata;
    this->internalMetadata  = nullptr;
}

std::string IndexerTrack::GetString(const char* metakey) {
    if (metakey && this->internalMetadata) {
        MetadataMap::iterator metavalue = this->internalMetadata->metadata.find(metakey);
        if (metavalue != this->internalMetadata->metadata.end()) {
            return metavalue->second;
        }
    }

    return "";
}

long long IndexerTrack::GetInt64(const char* key, long long defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stoll(GetString(key));
        }
    } catch (...) {
    }
    return defaultValue;
}

int IndexerTrack::GetInt32(const char* key, unsigned int defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stol(GetString(key));
        }
    }
    catch (...) {
    }
    return defaultValue;
}

double IndexerTrack::GetDouble(const char* key, double defaultValue) {
    try {
        std::string value = GetString(key);
        if (value.size()) {
            return std::stod(GetString(key));
        }
    } catch (...) {
    }
    return defaultValue;
}

void IndexerTrack::SetValue(const char* metakey, const char* value) {
    if (metakey && value) {
        this->internalMetadata->metadata.insert(
            std::pair<std::string, std::string>(metakey,value));
    }
}

void IndexerTrack::ClearValue(const char* metakey) {
    if (this->internalMetadata) {
        this->internalMetadata->metadata.erase(metakey);
    }
}

bool IndexerTrack::Contains(const char* metakey) {
    auto md = this->internalMetadata;
    return md && md->metadata.find(metakey) != md->metadata.end();
}

void IndexerTrack::SetThumbnail(const char *data, long size) {
    if (this->internalMetadata->thumbnailData) {
        delete[] this->internalMetadata->thumbnailData;
    }

    this->internalMetadata->thumbnailData = new char[size];
    this->internalMetadata->thumbnailSize = size;

    memcpy(this->internalMetadata->thumbnailData, data, size);
}

void IndexerTrack::SetReplayGain(const ReplayGain& replayGain) {
    this->internalMetadata->replayGain.reset();
    this->internalMetadata->replayGain = std::make_shared<ReplayGain>();
    memcpy(this->internalMetadata->replayGain.get(), &replayGain, sizeof(ReplayGain));
}

std::string IndexerTrack::Uri() {
    return this->GetString("filename");
}

int IndexerTrack::GetString(const char* key, char* dst, int size) {
    return CopyString(this->GetString(key), dst, size);
}

int IndexerTrack::Uri(char* dst, int size) {
    return CopyString(this->Uri(), dst, size);
}

Track::MetadataIteratorRange IndexerTrack::GetValues(const char* metakey) {
    if (this->internalMetadata) {
        return this->internalMetadata->metadata.equal_range(metakey);
    }

    return Track::MetadataIteratorRange();
}

Track::MetadataIteratorRange IndexerTrack::GetAllValues() {
    if (this->internalMetadata) {
        return Track::MetadataIteratorRange(
            this->internalMetadata->metadata.begin(),
            this->internalMetadata->metadata.end());
    }

    return Track::MetadataIteratorRange();
}

int64_t IndexerTrack::GetId() {
    return this->id;
}

bool IndexerTrack::NeedsToBeIndexed(
    const boost::filesystem::path &file,
    db::Connection &dbConnection)
{
    try {
        this->SetValue("path", file.string().c_str());
        this->SetValue("filename", file.string().c_str());

        size_t lastDot = file.leaf().string().find_last_of(".");
        if (lastDot != std::string::npos) {
            this->SetValue("extension", file.leaf().string().substr(lastDot + 1).c_str());
        }

        size_t fileSize = (size_t) boost::filesystem::file_size(file);
        size_t fileTime = (size_t) boost::filesystem::last_write_time(file);

        this->SetValue("filesize", boost::lexical_cast<std::string>(fileSize).c_str());
        this->SetValue("filetime", boost::lexical_cast<std::string>(fileTime).c_str());

        db::Statement stmt(
            "SELECT id, filename, filesize, filetime " \
            "FROM tracks t " \
            "WHERE filename=?", dbConnection);

        stmt.BindText(0, this->GetString("filename"));

        bool fileDifferent = true;

        if (stmt.Step() == db::Row) {
            this->id = stmt.ColumnInt64(0);
            int dbFileSize = stmt.ColumnInt32(2);
            int dbFileTime = stmt.ColumnInt32(3);

            if (fileSize == dbFileSize && fileTime == dbFileTime) {
                return false;
            }
        }
    }
    catch (...) {
    }

    return true;
}

static int64_t writeToTracksTable(
    db::Connection &dbConnection,
    IndexerTrack& track)
{
    std::string externalId = track.GetString("external_id");
    int64_t id = track.GetId();

    if (externalId.size() == 0) {
        return 0;
    }

    int sourceId = track.GetInt32("source_id", 0);

    /* if there's no ID specified, but we have an external ID, let's
    see if we can find the corresponding ID. this can happen when
    IInputSource plugins are reading/writing track data. */
    if (id == 0) {
        if (sourceId == 0) {
            db::Statement stmt("SELECT id FROM tracks WHERE source_id=? AND external_id=?", dbConnection);
            stmt.BindInt32(0, sourceId);
            stmt.BindText(1, externalId);
            if (stmt.Step() == db::Row) {
                track.SetId(stmt.ColumnInt64(0));
            }
        }
        else {
            std::string fn = track.GetString("filename");
            if (fn.size()) {
                db::Statement stmt("SELECT id, external_id FROM tracks WHERE filename=?", dbConnection);
                stmt.BindText(0, track.GetString("filename"));
                if (stmt.Step() == db::Row) {
                    id = stmt.ColumnInt64(0);
                    track.SetId(id);
                    track.SetValue("external_id", stmt.ColumnText(1));
                }
            }
        }
    }

    std::string query;

    if (id != 0) {
        query =
            "UPDATE tracks "
            "SET track=?, disc=?, bpm=?, duration=?, filesize=?, "
            "    title=?, filename=?, filetime=?, path_id=?, external_id=? "
            "WHERE id=?";
    }
    else {
        query =
            "INSERT INTO tracks "
            "(track, disc, bpm, duration, filesize, title, filename, filetime, path_id, external_id) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
    }

    db::Statement stmt(query.c_str(), dbConnection);

    stmt.BindText(0, track.GetString("track"));
    stmt.BindText(1, track.GetString("disc"));
    stmt.BindText(2, track.GetString("bpm"));
    stmt.BindInt32(3, track.GetInt32("duration"));
    stmt.BindInt32(4, track.GetInt32("filesize"));
    stmt.BindText(5, track.GetString("title"));
    stmt.BindText(6, track.GetString("filename"));
    stmt.BindInt32(7, track.GetInt32("filetime"));
    stmt.BindInt64(8, track.GetInt64("path_id"));
    stmt.BindText(9, track.GetString("external_id"));

    if (id != 0) {
        stmt.BindInt64(10, id);
    }

    if (stmt.Step() == db::Done) {
        if (id == 0) {
            return dbConnection.LastInsertedId();
        }
    }

    return id;
}

{'fraction_non_alphanumeric': 0.1472108709566245, 'fraction_numerical': 0.008369848850376643, 'mean_word_length': 2.869689464659935, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1872230', 'n_tokens_mistral': 6219, 'n_tokens_neox': 5952, 'n_words': 1449}