<commit_before>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

/**
 * @file EntityWithSources.cpp
 * @brief Implementation of all methods of the class EntityWithSources.
 */


#include <pandora/Util.hpp>
#include <pandora/Group.hpp>
#include <pandora/DataSet.hpp>
#include <pandora/PSize.hpp>
#include <pandora/File.hpp>
#include <pandora/Block.hpp>
#include <pandora/Source.hpp>
#include <pandora/EntityWithSources.hpp>

using namespace std;

namespace pandora {


EntityWithSources::EntityWithSources(File file, Block block, Group group, string id)
  : EntityWithMetadata(file, group, id), block(block), sources_refs(group, "sources")
{
}


EntityWithSources::EntityWithSources(File file, Block block, Group group, string id, time_t time)
  : EntityWithMetadata(file, group, id, time), block(block), sources_refs(group, "sources")
{
}


std::vector<std::string> EntityWithSources::source_ids() const
{
  vector<string> ids;

  if (group.hasData("sources")) {
    DataSet ds = group.openData("sources");
    ds.read(ids, true);
  }

  return ids;
}


size_t EntityWithSources::sourceCount() const {
  size_t count = 0;
  if (group.hasData("sources")) {
    DataSet dataset = group.openData("sources");
    PSize size = dataset.size();
    count = size[0];
  }
  return count;
}


bool EntityWithSources::hasSource(const Source &source) const {
  return sources_refs.has(source);
}


bool EntityWithSources::hasSource(string id) const {
  return sources_refs.has(id);
}


vector<Source> EntityWithSources::sources() const {
  vector<string> ids = sources_refs.get();

  vector<Source> source_obj = block.findSources([&](const Source &source) {
    return std::find(ids.begin(), ids.end(), source.id()) != ids.end();
  });

  if (source_obj.size() != ids.size()) {<commit_msg>EntityWithSources: use ReferenceList in sourceCount()<commit_after>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

/**
 * @file EntityWithSources.cpp
 * @brief Implementation of all methods of the class EntityWithSources.
 */


#include <pandora/Util.hpp>
#include <pandora/Group.hpp>
#include <pandora/DataSet.hpp>
#include <pandora/PSize.hpp>
#include <pandora/File.hpp>
#include <pandora/Block.hpp>
#include <pandora/Source.hpp>
#include <pandora/EntityWithSources.hpp>

using namespace std;

namespace pandora {


EntityWithSources::EntityWithSources(File file, Block block, Group group, string id)
  : EntityWithMetadata(file, group, id), block(block), sources_refs(group, "sources")
{
}


EntityWithSources::EntityWithSources(File file, Block block, Group group, string id, time_t time)
  : EntityWithMetadata(file, group, id, time), block(block), sources_refs(group, "sources")
{
}


std::vector<std::string> EntityWithSources::source_ids() const
{
  vector<string> ids;

  if (group.hasData("sources")) {
    DataSet ds = group.openData("sources");
    ds.read(ids, true);
  }

  return ids;
}


size_t EntityWithSources::sourceCount() const {
  return sources_refs.count();
}


bool EntityWithSources::hasSource(const Source &source) const {
  return sources_refs.has(source);
}


bool EntityWithSources::hasSource(string id) const {
  return sources_refs.has(id);
}


vector<Source> EntityWithSources::sources() const {
  vector<string> ids = sources_refs.get();

  vector<Source> source_obj = block.findSources([&](const Source &source) {
    return std::find(ids.begin(), ids.end(), source.id()) != ids.end();
  });

  if (source_obj.size() != ids.size()) {
{'fraction_non_alphanumeric': 0.13915940543311123, 'fraction_numerical': 0.0025627883136852894, 'mean_word_length': 4.949695121951219, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '484834', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1219, 'n_words': 327}