<commit_before>/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
 *
 * Copyright (C) 2013,2014 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; either version 2 of the licence or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General
 * Public License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#include "config.h"

#include <string.h>
#include <glib-unix.h>
#include <json-glib/json-glib.h>
#include <gio/gunixoutputstream.h>
#include <rpm/rpmsq.h>
#include <libhif.h>
#include <libhif/hif-utils.h>
#include <stdio.h>
#include <libglnx.h>
#include <rpm/rpmmacro.h>

#include "rpmostree-compose-builtins.h"
#include "rpmostree-util.h"
#include "rpmostree-json-parsing.h"
#include "rpmostree-cleanup.h"
#include "rpmostree-postprocess.h"
#include "rpmostree-passwd-util.h"
#include "rpmostree-rpm-util.h"

#include "libgsystem.h"

static char *opt_workdir;
static gboolean opt_workdir_tmpfs;
static char *opt_cachedir;
static gboolean opt_force_nocache;
static char *opt_proxy;
static char *opt_output_repodata_dir;
static char **opt_metadata_strings;
static char *opt_repo;
static char **opt_override_pkg_repos;
static char *opt_touch_if_changed;
static gboolean opt_print_only;

static GOptionEntry option_entries[] = {
  { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Append given key and value (in string format) to metadata", "KEY=VALUE" },
  { "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
  { "workdir-tmpfs", 0, 0, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL },
  { "output-repodata-dir", 0, 0, G_OPTION_ARG_STRING, &opt_output_repodata_dir, "Save downloaded repodata in DIR", "DIR" },
  { "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
  { "force-nocache", 0, 0, G_OPTION_ARG_NONE, &opt_force_nocache, "Always create a new OSTree commit, even if nothing appears to have changed", NULL },
  { "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
  { "add-override-pkg-repo", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_override_pkg_repos, "Include an additional package repository from DIRECTORY", "DIRECTORY" },
  { "proxy", 0, 0, G_OPTION_ARG_STRING, &opt_proxy, "HTTP proxy", "PROXY" },
  { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" },
  { "print-only", 0, 0, G_OPTION_ARG_NONE, &opt_print_only, "Just expand any includes and print treefile", NULL },
  { NULL }
};

/* FIXME: This is a copy of ot_admin_checksum_version */
static char *
checksum_version (GVariant *checksum)
{
  gs_unref_variant GVariant *metadata = NULL;
  const char *ret = NULL;

  metadata = g_variant_get_child_value (checksum, 0);

  if (!g_variant_lookup (metadata, "version", "&s", &ret))
    return NULL;

  return g_strdup (ret);
}

typedef struct {
  GPtrArray *treefile_context_dirs;
  
  GFile *workdir;
  int workdir_dfd;
  OstreeRepo *repo;
  char *previous_checksum;

  GBytes *serialized_treefile;
} RpmOstreeTreeComposeContext;

static int
ptrarray_sort_compare_strings (gconstpointer ap,
                               gconstpointer bp)
{
  char **asp = (gpointer)ap;
  char **bsp = (gpointer)bp;
  return strcmp (*asp, *bsp);
}

static gboolean
compute_checksum_from_treefile_and_goal (RpmOstreeTreeComposeContext   *self,
                                         HyGoal                         goal,
                                         char                        **out_checksum,
                                         GError                      **error)
{
  gboolean ret = FALSE;
  gs_free char *ret_checksum = NULL;
  GChecksum *checksum = g_checksum_new (G_CHECKSUM_SHA256);
  
  /* Hash in the raw treefile; this means reordering the input packages
   * or adding a comment will cause a recompose, but let's be conservative
   * here.
   */
  { gsize len;
    const guint8* buf = g_bytes_get_data (self->serialized_treefile, &len);

    g_checksum_update (checksum, buf, len);
  }

  /* FIXME; we should also hash the post script */

  /* Hash in each package */
  { _cleanup_hypackagelist_ HyPackageList pkglist = NULL;
    HyPackage pkg;
    guint i;
    gs_unref_ptrarray GPtrArray *nevras = g_ptr_array_new_with_free_func (g_free);

    pkglist = hy_goal_list_installs (goal);

    FOR_PACKAGELIST(pkg, pkglist, i)
      {
        g_ptr_array_add (nevras, hy_package_get_nevra (pkg));
      }

    g_ptr_array_sort (nevras, ptrarray_sort_compare_strings);
    
    for (i = 0; i < nevras->len; i++)
      {
        const char *nevra = nevras->pdata[i];
        g_checksum_update (checksum, (guint8*)nevra, strlen (nevra));
      }
  }

  ret_checksum = g_strdup (g_checksum_get_string (checksum));

  ret = TRUE;
  gs_transfer_out_value (out_checksum, &ret_checksum);
  if (checksum) g_checksum_free (checksum);
  return ret;
}


static void
on_hifstate_percentage_changed (HifState   *hifstate,
                                guint       percentage,
                                gpointer    user_data)
{
  const char *text = user_data;
  glnx_console_progress_text_percent (text, percentage);
}

static gboolean
install_packages_in_root (RpmOstreeTreeComposeContext  *self,
                          JsonObject      *treedata,
                          GFile           *yumroot,
                          char           **packages,
                          gboolean        *out_unmodified,
                          char           **out_new_inputhash,
                          GCancellable    *cancellable,
                          GError         **error)
{
  gboolean ret = FALSE;
  guint progress_sigid;
  char **strviter;
  GFile *contextdir = self->treefile_context_dirs->pdata[0];
  gs_unref_object HifContext *hifctx = NULL;
  gs_free char *cachedir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                             "cache",
                                             NULL);
  gs_free char *solvdir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                            "solv",
                                            NULL);
  gs_free char *lockdir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                            "lock",
                                            NULL);
  gs_free char *ret_new_inputhash = NULL;

  /* We can be control-c'd at any time */
#if BUILDOPT_HAVE_RPMSQ_SET_INTERRUPT_SAFETY
  rpmsqSetInterruptSafety (FALSE);
#endif

  /* Apparently there's only one process-global macro context;
   * realistically, we're going to have to refactor all of the RPM
   * stuff to a subprocess.
   */
  hifctx = hif_context_new ();
  hif_context_set_http_proxy (hifctx, opt_proxy ? opt_proxy : g_getenv ("http_proxy"));

  hif_context_set_install_root (hifctx, gs_file_get_path_cached (yumroot));

  hif_context_set_cache_dir (hifctx, cachedir);
  hif_context_set_solv_dir (hifctx, solvdir);
  hif_context_set_lock_dir (hifctx, lockdir);
  hif_context_set_check_disk_space (hifctx, FALSE);
  hif_context_set_check_transaction (hifctx, FALSE);

  hif_context_set_yumdb_enabled (hifctx, FALSE);

  hif_context_set_repo_dir (hifctx, gs_file_get_path_cached (contextdir));

  { JsonNode *install_langs_n =
      json_object_get_member (treedata, "install-langs");

    if (install_langs_n != NULL)<commit_msg>compose: Fix "too old" error for "old" file:/// repos<commit_after>/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
 *
 * Copyright (C) 2013,2014 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; either version 2 of the licence or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General
 * Public License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#include "config.h"

#include <string.h>
#include <glib-unix.h>
#include <json-glib/json-glib.h>
#include <gio/gunixoutputstream.h>
#include <rpm/rpmsq.h>
#include <libhif.h>
#include <libhif/hif-utils.h>
#include <stdio.h>
#include <libglnx.h>
#include <rpm/rpmmacro.h>

#include "rpmostree-compose-builtins.h"
#include "rpmostree-util.h"
#include "rpmostree-json-parsing.h"
#include "rpmostree-cleanup.h"
#include "rpmostree-postprocess.h"
#include "rpmostree-passwd-util.h"
#include "rpmostree-rpm-util.h"

#include "libgsystem.h"

static char *opt_workdir;
static gboolean opt_workdir_tmpfs;
static char *opt_cachedir;
static gboolean opt_force_nocache;
static char *opt_proxy;
static char *opt_output_repodata_dir;
static char **opt_metadata_strings;
static char *opt_repo;
static char **opt_override_pkg_repos;
static char *opt_touch_if_changed;
static gboolean opt_print_only;

static GOptionEntry option_entries[] = {
  { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Append given key and value (in string format) to metadata", "KEY=VALUE" },
  { "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
  { "workdir-tmpfs", 0, 0, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL },
  { "output-repodata-dir", 0, 0, G_OPTION_ARG_STRING, &opt_output_repodata_dir, "Save downloaded repodata in DIR", "DIR" },
  { "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
  { "force-nocache", 0, 0, G_OPTION_ARG_NONE, &opt_force_nocache, "Always create a new OSTree commit, even if nothing appears to have changed", NULL },
  { "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
  { "add-override-pkg-repo", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_override_pkg_repos, "Include an additional package repository from DIRECTORY", "DIRECTORY" },
  { "proxy", 0, 0, G_OPTION_ARG_STRING, &opt_proxy, "HTTP proxy", "PROXY" },
  { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" },
  { "print-only", 0, 0, G_OPTION_ARG_NONE, &opt_print_only, "Just expand any includes and print treefile", NULL },
  { NULL }
};

/* FIXME: This is a copy of ot_admin_checksum_version */
static char *
checksum_version (GVariant *checksum)
{
  gs_unref_variant GVariant *metadata = NULL;
  const char *ret = NULL;

  metadata = g_variant_get_child_value (checksum, 0);

  if (!g_variant_lookup (metadata, "version", "&s", &ret))
    return NULL;

  return g_strdup (ret);
}

typedef struct {
  GPtrArray *treefile_context_dirs;
  
  GFile *workdir;
  int workdir_dfd;
  OstreeRepo *repo;
  char *previous_checksum;

  GBytes *serialized_treefile;
} RpmOstreeTreeComposeContext;

static int
ptrarray_sort_compare_strings (gconstpointer ap,
                               gconstpointer bp)
{
  char **asp = (gpointer)ap;
  char **bsp = (gpointer)bp;
  return strcmp (*asp, *bsp);
}

static gboolean
compute_checksum_from_treefile_and_goal (RpmOstreeTreeComposeContext   *self,
                                         HyGoal                         goal,
                                         char                        **out_checksum,
                                         GError                      **error)
{
  gboolean ret = FALSE;
  gs_free char *ret_checksum = NULL;
  GChecksum *checksum = g_checksum_new (G_CHECKSUM_SHA256);
  
  /* Hash in the raw treefile; this means reordering the input packages
   * or adding a comment will cause a recompose, but let's be conservative
   * here.
   */
  { gsize len;
    const guint8* buf = g_bytes_get_data (self->serialized_treefile, &len);

    g_checksum_update (checksum, buf, len);
  }

  /* FIXME; we should also hash the post script */

  /* Hash in each package */
  { _cleanup_hypackagelist_ HyPackageList pkglist = NULL;
    HyPackage pkg;
    guint i;
    gs_unref_ptrarray GPtrArray *nevras = g_ptr_array_new_with_free_func (g_free);

    pkglist = hy_goal_list_installs (goal);

    FOR_PACKAGELIST(pkg, pkglist, i)
      {
        g_ptr_array_add (nevras, hy_package_get_nevra (pkg));
      }

    g_ptr_array_sort (nevras, ptrarray_sort_compare_strings);
    
    for (i = 0; i < nevras->len; i++)
      {
        const char *nevra = nevras->pdata[i];
        g_checksum_update (checksum, (guint8*)nevra, strlen (nevra));
      }
  }

  ret_checksum = g_strdup (g_checksum_get_string (checksum));

  ret = TRUE;
  gs_transfer_out_value (out_checksum, &ret_checksum);
  if (checksum) g_checksum_free (checksum);
  return ret;
}


static void
on_hifstate_percentage_changed (HifState   *hifstate,
                                guint       percentage,
                                gpointer    user_data)
{
  const char *text = user_data;
  glnx_console_progress_text_percent (text, percentage);
}

static gboolean
install_packages_in_root (RpmOstreeTreeComposeContext  *self,
                          JsonObject      *treedata,
                          GFile           *yumroot,
                          char           **packages,
                          gboolean        *out_unmodified,
                          char           **out_new_inputhash,
                          GCancellable    *cancellable,
                          GError         **error)
{
  gboolean ret = FALSE;
  guint progress_sigid;
  char **strviter;
  GFile *contextdir = self->treefile_context_dirs->pdata[0];
  gs_unref_object HifContext *hifctx = NULL;
  gs_free char *cachedir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                             "cache",
                                             NULL);
  gs_free char *solvdir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                            "solv",
                                            NULL);
  gs_free char *lockdir = g_build_filename (gs_file_get_path_cached (self->workdir),
                                            "lock",
                                            NULL);
  gs_free char *ret_new_inputhash = NULL;

  /* We can be control-c'd at any time */
#if BUILDOPT_HAVE_RPMSQ_SET_INTERRUPT_SAFETY
  rpmsqSetInterruptSafety (FALSE);
#endif

  /* Apparently there's only one process-global macro context;
   * realistically, we're going to have to refactor all of the RPM
   * stuff to a subprocess.
   */
  hifctx = hif_context_new ();
  hif_context_set_http_proxy (hifctx, opt_proxy ? opt_proxy : g_getenv ("http_proxy"));

  hif_context_set_install_root (hifctx, gs_file_get_path_cached (yumroot));

  hif_context_set_cache_dir (hifctx, cachedir);
  hif_context_set_cache_age (hifctx, G_MAXUINT);
  hif_context_set_solv_dir (hifctx, solvdir);
  hif_context_set_lock_dir (hifctx, lockdir);
  hif_context_set_check_disk_space (hifctx, FALSE);
  hif_context_set_check_transaction (hifctx, FALSE);

  hif_context_set_yumdb_enabled (hifctx, FALSE);

  hif_context_set_repo_dir (hifctx, gs_file_get_path_cached (contextdir));

  { JsonNode *install_langs_n =
      json_object_get_member (treedata, "install-langs");

    if (install_langs_n != NULL)
{'fraction_non_alphanumeric': 0.09775929489168891, 'fraction_numerical': 0.006455216932530569, 'mean_word_length': 3.0049714143673874, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4726707', 'n_tokens_mistral': 5474, 'n_tokens_neox': 5301, 'n_words': 1378}