<commit_before>/* Copyright 2016 Endless Mobile, Inc. */

/* This file is part of eos-shard.
 *
 * eos-shard is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * eos-shard is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with eos-shard.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include "eos-shard-shard-file-impl-v2.h"

#include <errno.h>
#include <endian.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "eos-shard-shard-file.h"
#include "eos-shard-shard-file-impl.h"
#include "eos-shard-enums.h"
#include "eos-shard-blob.h"
#include "eos-shard-record.h"

#include "eos-shard-format-v2.h"

struct _EosShardShardFileImplV2
{
  GObject parent;

  EosShardShardFile *shard_file;
  int fd;

  struct eos_shard_v2_hdr hdr;
  struct eos_shard_v2_record *records;
};

static void shard_file_impl_init (EosShardShardFileImplInterface *iface);

G_DEFINE_TYPE_WITH_CODE (EosShardShardFileImplV2, eos_shard_shard_file_impl_v2, G_TYPE_OBJECT,
                         G_IMPLEMENT_INTERFACE (EOS_SHARD_TYPE_SHARD_FILE_IMPL, shard_file_impl_init))

static void
eos_shard_shard_file_impl_v2_finalize (GObject *object)
{
  EosShardShardFileImplV2 *self = EOS_SHARD_SHARD_FILE_IMPL_V2 (object);

  g_clear_pointer (&self->records, g_free);

  G_OBJECT_CLASS (eos_shard_shard_file_impl_v2_parent_class)->finalize (object);
}

static void
eos_shard_shard_file_impl_v2_class_init (EosShardShardFileImplV2Class *klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);

  gobject_class->finalize = eos_shard_shard_file_impl_v2_finalize;
}

static void
eos_shard_shard_file_impl_v2_init (EosShardShardFileImplV2 *shard_file)
{
}

EosShardShardFileImpl *
_eos_shard_shard_file_impl_v2_new (EosShardShardFile *shard_file,
                                   int fd,
                                   GError **error)
{
  g_autoptr(EosShardShardFileImplV2) self = g_object_new (EOS_SHARD_TYPE_SHARD_FILE_IMPL_V2, NULL);
  self->fd = fd;
  self->shard_file = shard_file;

  read (self->fd, &self->hdr, sizeof (self->hdr));

  if (memcmp (self->hdr.magic, EOS_SHARD_V2_MAGIC, sizeof (self->hdr.magic)) != 0)
    goto error;

  int buf_size = self->hdr.records_length * sizeof (*self->records);
  self->records = g_malloc (buf_size);

  if (pread (self->fd, self->records, buf_size, self->hdr.records_start) != buf_size)<commit_msg>impl-v2: Make sure to check the result of read<commit_after>/* Copyright 2016 Endless Mobile, Inc. */

/* This file is part of eos-shard.
 *
 * eos-shard is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * eos-shard is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with eos-shard.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include "eos-shard-shard-file-impl-v2.h"

#include <errno.h>
#include <endian.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "eos-shard-shard-file.h"
#include "eos-shard-shard-file-impl.h"
#include "eos-shard-enums.h"
#include "eos-shard-blob.h"
#include "eos-shard-record.h"

#include "eos-shard-format-v2.h"

struct _EosShardShardFileImplV2
{
  GObject parent;

  EosShardShardFile *shard_file;
  int fd;

  struct eos_shard_v2_hdr hdr;
  struct eos_shard_v2_record *records;
};

static void shard_file_impl_init (EosShardShardFileImplInterface *iface);

G_DEFINE_TYPE_WITH_CODE (EosShardShardFileImplV2, eos_shard_shard_file_impl_v2, G_TYPE_OBJECT,
                         G_IMPLEMENT_INTERFACE (EOS_SHARD_TYPE_SHARD_FILE_IMPL, shard_file_impl_init))

static void
eos_shard_shard_file_impl_v2_finalize (GObject *object)
{
  EosShardShardFileImplV2 *self = EOS_SHARD_SHARD_FILE_IMPL_V2 (object);

  g_clear_pointer (&self->records, g_free);

  G_OBJECT_CLASS (eos_shard_shard_file_impl_v2_parent_class)->finalize (object);
}

static void
eos_shard_shard_file_impl_v2_class_init (EosShardShardFileImplV2Class *klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);

  gobject_class->finalize = eos_shard_shard_file_impl_v2_finalize;
}

static void
eos_shard_shard_file_impl_v2_init (EosShardShardFileImplV2 *shard_file)
{
}

EosShardShardFileImpl *
_eos_shard_shard_file_impl_v2_new (EosShardShardFile *shard_file,
                                   int fd,
                                   GError **error)
{
  g_autoptr(EosShardShardFileImplV2) self = g_object_new (EOS_SHARD_TYPE_SHARD_FILE_IMPL_V2, NULL);
  self->fd = fd;
  self->shard_file = shard_file;

  if (read (self->fd, &self->hdr, sizeof (self->hdr)) != sizeof (self->hdr))
    goto error;

  if (memcmp (self->hdr.magic, EOS_SHARD_V2_MAGIC, sizeof (self->hdr.magic)) != 0)
    goto error;

  int buf_size = self->hdr.records_length * sizeof (*self->records);
  self->records = g_malloc (buf_size);

  if (pread (self->fd, self->records, buf_size, self->hdr.records_start) != buf_size)
{'fraction_non_alphanumeric': 0.09970470731283655, 'fraction_numerical': 0.009553586937641132, 'mean_word_length': 4.356279069767442, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7372167', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2151, 'n_words': 492}