<commit_before>/*
 * libsocialweb - social data store
 * Copyright (C) 2008 - 2009 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include "sw-view.h"
#include "sw-view-ginterface.h"

#include <libsocialweb/sw-core.h>
#include <libsocialweb/sw-debug.h>
#include <libsocialweb/sw-item.h>
#include <libsocialweb/sw-set.h>
#include <libsocialweb/sw-utils.h>
#include <libsocialweb/sw-cache.h>
#include <libsocialweb/sw-online.h>

static void view_iface_init (gpointer g_iface, gpointer iface_data);
G_DEFINE_TYPE_WITH_CODE (SwView, sw_view, G_TYPE_OBJECT,
                         G_IMPLEMENT_INTERFACE (SW_TYPE_VIEW_IFACE,
                                                view_iface_init));

#define GET_PRIVATE(o) \
  (G_TYPE_INSTANCE_GET_PRIVATE ((o), SW_TYPE_VIEW, SwViewPrivate))

/* Refresh every 5 minutes */
#define REFRESH_TIMEOUT (5 * 60)

/* Wait 200ms between queued recalculates before recalculating */
#define RECALCULATE_DELAY 200

struct _SwViewPrivate {
  SwCore *core;
  /* List of SwService objects */
  GList *services;
  /* The maximum number of items in the view */
  guint count;
  /* The refresh timeout id */
  guint refresh_timeout_id;
  /* The set of all items currently being considered for the client */
  SwSet *all_items;
  /* The items we've sent to the client */
  SwSet *current;
  /* Whether we're currently running */
  gboolean running;
  /* For recalculate queue */
  guint recalculate_timeout_id;
  /* The last time we recalculated. Use for changed items. */
  time_t last_recalculate_time;
};

enum {
  PROP_0,
  PROP_CORE,
  PROP_COUNT,
};

static int
get_service_count (GHashTable *hash,
                   SwService  *service)
{
  return GPOINTER_TO_INT (g_hash_table_lookup (hash, service));
}

static void
inc_service_count (GHashTable *hash,
                   SwService  *service)
{
  int count;
  count = GPOINTER_TO_INT (g_hash_table_lookup (hash, service));
  ++count;
  g_hash_table_insert (hash, service, GINT_TO_POINTER (count));
}

static SwSet *
munge_items (SwView *view)
{
  SwViewPrivate *priv = view->priv;
  GList *list, *l = NULL;
  int count, service_max;
  GHashTable *counts;
  SwSet *new;
  SwItem *item;

  /* The magic */
  list = sw_set_as_list (priv->all_items);

  while (list) {
    item = (SwItem *)list->data;
    if (!sw_core_is_item_banned (priv->core, item)) {
      if (sw_item_get_ready (item)) {
        l = g_list_prepend (l, item);
        SW_DEBUG (VIEWS, "Item ready: %s",
                      sw_item_get (item, "id"));
      } else {
        SW_DEBUG (VIEWS, "Item not ready. Skipping: %s",
                      sw_item_get (item, "id"));
      }
    }
    list = g_list_delete_link (list, list);
  }
  list = l;

  list = g_list_sort (list, (GCompareFunc)sw_item_compare_date_newer);

  counts = g_hash_table_new (NULL, NULL);
  service_max = ceil ((float)priv->count / g_list_length (priv->services));

  count = 0;
  new = sw_item_set_new ();

  /* We manipulate list in place instead of using a temporary GList* because
   * need to keep track of the new head.  This loop exits if we have ran out of
   * items from the sources (list == NULL), we have filled up the result list
   * (count >= priv->count), or if we have reached the end of the list but have
   * not yet filled up the result list.
  */
  /* TODO: use gsequence instead for clarity? */
  while (list && count < priv->count) {
    SwItem *item;
    SwService *service;

    item = list->data;
    service = sw_item_get_service (item);

    if (get_service_count (counts, service) >= service_max) {
      if (list->next) {
        list = list->next;
        continue;
      } else {
        /* Here we have got to the end of the source list but haven't yet filled
           up the target list */
        /* TODO: put the fill-in logic here? */
        break;
      }
    }

    /* New list steals the reference */
    count++;
    sw_set_add (new, (GObject*)item);
    g_object_unref (item);
    list = g_list_delete_link (list, list);

    inc_service_count (counts, service);
  }

  /* If we still have items left in the pending list, rewind back to the
     beginning and add more */
  list = g_list_first (list);
  while (list && count < priv->count) {
    GObject *item = list->data;
    count++;
    sw_set_add (new, item);
    g_object_unref (item);
    list = g_list_delete_link (list, list);
  }

  /* Clean up */
  while (list) {
    g_object_unref (list->data);
    list = g_list_delete_link (list, list);
  }

  return new;
}

static gboolean
remove_service (GObject  *object,
                gpointer  user_data)
{
  SwItem *item = SW_ITEM (object);
  SwService *service = SW_SERVICE (user_data);

  return sw_item_get_service (item) == service;
}

static void
sw_view_removed_items (SwView *view,
                       SwSet  *removed_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (removed_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = g_value_array_new (2);

    value_array = g_value_array_append (value_array, NULL);
    g_value_init (g_value_array_get_nth (value_array, 0), G_TYPE_STRING);
    g_value_set_string (g_value_array_get_nth (value_array, 0),
                        sw_service_get_name (sw_item_get_service (item)));

    value_array = g_value_array_append (value_array, NULL);
    g_value_init (g_value_array_get_nth (value_array, 1), G_TYPE_STRING);
    g_value_set_string (g_value_array_get_nth (value_array, 1),
                        sw_item_get (item, "id"));

    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_removed (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}

static void
sw_view_added_items (SwView *view,
                     SwSet  *added_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (added_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = _sw_item_to_value_array (item);
    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_added (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}


static void
sw_view_changed_items (SwView *view,
                       SwSet  *added_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (added_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = _sw_item_to_value_array (item);
    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_changed (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}

static gboolean
_filter_only_changed_cb (SwSet  *set,
                         SwItem *item,
                         SwView *view)
{
  if (sw_item_get_mtime (item) >= view->priv->last_recalculate_time)
    return TRUE;
  else
    return FALSE;
}

void
sw_view_recalculate (SwView *view)
{
  SwViewPrivate *priv;
  SwSet *old_items, *new_items;
  SwSet *removed_items, *added_items, *changed_items = NULL;

  g_return_if_fail (SW_IS_VIEW (view));

  priv = view->priv;
  if (!priv->running)
    return;

  old_items = priv->current;
  new_items = munge_items (view);

  removed_items = sw_set_difference (old_items, new_items);
  added_items = sw_set_difference (new_items, old_items);

  if (priv->last_recalculate_time != 0)
  {
    changed_items = sw_set_filter (new_items,
                                   (SwSetFilterFunc)_filter_only_changed_cb,
                                   view);
  }

  if (!sw_set_is_empty (removed_items))
    sw_view_removed_items (view, removed_items);

  if (!sw_set_is_empty (added_items))
    sw_view_added_items (view, added_items);

  if (changed_items && !sw_set_is_empty (changed_items))
    sw_view_changed_items (view, changed_items);

  sw_set_unref (removed_items);
  sw_set_unref (added_items);

  if (changed_items)
    sw_set_unref (changed_items);

  sw_set_unref (priv->current);
  priv->current = new_items;
  priv->last_recalculate_time = time (NULL);
}

static gboolean
recalculate_timeout_cb (SwView *view)
{
  view->priv->recalculate_timeout_id = 0;
  sw_view_recalculate (view);

  return FALSE;
}

static void
sw_view_queue_recalculate (SwView *view)
{
  SwViewPrivate *priv;

  priv = view->priv;

  if (priv->recalculate_timeout_id)
  {
    g_source_remove (priv->recalculate_timeout_id);
  }

  priv->recalculate_timeout_id =
    g_timeout_add (RECALCULATE_DELAY,
                   (GSourceFunc)recalculate_timeout_cb,
                   view);
}

static void
_item_ready_notify_cb (SwItem     *item,
                       GParamSpec *pspec,
                       SwView     *view)
{
  /* TODO: Use a timeout to rate limit this */
  if (sw_item_get_ready (item)) {
    SW_DEBUG (VIEWS, "Item became ready: %s.",
                  sw_item_get (item, "id"));
    sw_view_queue_recalculate (view);
    g_signal_handlers_disconnect_by_func (item,
                                          _item_ready_notify_cb,
                                          view);
  }
}

static void
setup_ready_handler (gpointer object, gpointer user_data)
{
  SwItem *item = SW_ITEM (object);
  SwView *view = SW_VIEW (user_data);

  if (sw_item_get_ready (item))
    return;

  SW_DEBUG (VIEWS, "Item not ready. Setting up signal handler: %s.",
                sw_item_get (item, "id"));

  g_signal_connect (item,
                    "notify::ready",
                    (GCallback)_item_ready_notify_cb,
                    view);
}

static void
service_updated (SwService *service,
                 SwSet     *set,
                 gpointer   user_data)
{
  SwView *view = SW_VIEW (user_data);
  SwViewPrivate *priv = view->priv;

  if (set) {
    sw_cache_save (service, set);<commit_msg>view: Remove "notify::ready" signal handler when view is disposed<commit_after>/*
 * libsocialweb - social data store
 * Copyright (C) 2008 - 2009 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include "sw-view.h"
#include "sw-view-ginterface.h"

#include <libsocialweb/sw-core.h>
#include <libsocialweb/sw-debug.h>
#include <libsocialweb/sw-item.h>
#include <libsocialweb/sw-set.h>
#include <libsocialweb/sw-utils.h>
#include <libsocialweb/sw-cache.h>
#include <libsocialweb/sw-online.h>

static void view_iface_init (gpointer g_iface, gpointer iface_data);
G_DEFINE_TYPE_WITH_CODE (SwView, sw_view, G_TYPE_OBJECT,
                         G_IMPLEMENT_INTERFACE (SW_TYPE_VIEW_IFACE,
                                                view_iface_init));

#define GET_PRIVATE(o) \
  (G_TYPE_INSTANCE_GET_PRIVATE ((o), SW_TYPE_VIEW, SwViewPrivate))

/* Refresh every 5 minutes */
#define REFRESH_TIMEOUT (5 * 60)

/* Wait 200ms between queued recalculates before recalculating */
#define RECALCULATE_DELAY 200

struct _SwViewPrivate {
  SwCore *core;
  /* List of SwService objects */
  GList *services;
  /* The maximum number of items in the view */
  guint count;
  /* The refresh timeout id */
  guint refresh_timeout_id;
  /* The set of all items currently being considered for the client */
  SwSet *all_items;
  /* The items we've sent to the client */
  SwSet *current;
  /* Whether we're currently running */
  gboolean running;
  /* For recalculate queue */
  guint recalculate_timeout_id;
  /* The last time we recalculated. Use for changed items. */
  time_t last_recalculate_time;
};

enum {
  PROP_0,
  PROP_CORE,
  PROP_COUNT,
};

static int
get_service_count (GHashTable *hash,
                   SwService  *service)
{
  return GPOINTER_TO_INT (g_hash_table_lookup (hash, service));
}

static void
inc_service_count (GHashTable *hash,
                   SwService  *service)
{
  int count;
  count = GPOINTER_TO_INT (g_hash_table_lookup (hash, service));
  ++count;
  g_hash_table_insert (hash, service, GINT_TO_POINTER (count));
}

static SwSet *
munge_items (SwView *view)
{
  SwViewPrivate *priv = view->priv;
  GList *list, *l = NULL;
  int count, service_max;
  GHashTable *counts;
  SwSet *new;
  SwItem *item;

  /* The magic */
  list = sw_set_as_list (priv->all_items);

  while (list) {
    item = (SwItem *)list->data;
    if (!sw_core_is_item_banned (priv->core, item)) {
      if (sw_item_get_ready (item)) {
        l = g_list_prepend (l, item);
        SW_DEBUG (VIEWS, "Item ready: %s",
                      sw_item_get (item, "id"));
      } else {
        SW_DEBUG (VIEWS, "Item not ready. Skipping: %s",
                      sw_item_get (item, "id"));
      }
    }
    list = g_list_delete_link (list, list);
  }
  list = l;

  list = g_list_sort (list, (GCompareFunc)sw_item_compare_date_newer);

  counts = g_hash_table_new (NULL, NULL);
  service_max = ceil ((float)priv->count / g_list_length (priv->services));

  count = 0;
  new = sw_item_set_new ();

  /* We manipulate list in place instead of using a temporary GList* because
   * need to keep track of the new head.  This loop exits if we have ran out of
   * items from the sources (list == NULL), we have filled up the result list
   * (count >= priv->count), or if we have reached the end of the list but have
   * not yet filled up the result list.
  */
  /* TODO: use gsequence instead for clarity? */
  while (list && count < priv->count) {
    SwItem *item;
    SwService *service;

    item = list->data;
    service = sw_item_get_service (item);

    if (get_service_count (counts, service) >= service_max) {
      if (list->next) {
        list = list->next;
        continue;
      } else {
        /* Here we have got to the end of the source list but haven't yet filled
           up the target list */
        /* TODO: put the fill-in logic here? */
        break;
      }
    }

    /* New list steals the reference */
    count++;
    sw_set_add (new, (GObject*)item);
    g_object_unref (item);
    list = g_list_delete_link (list, list);

    inc_service_count (counts, service);
  }

  /* If we still have items left in the pending list, rewind back to the
     beginning and add more */
  list = g_list_first (list);
  while (list && count < priv->count) {
    GObject *item = list->data;
    count++;
    sw_set_add (new, item);
    g_object_unref (item);
    list = g_list_delete_link (list, list);
  }

  /* Clean up */
  while (list) {
    g_object_unref (list->data);
    list = g_list_delete_link (list, list);
  }

  return new;
}

static gboolean
remove_service (GObject  *object,
                gpointer  user_data)
{
  SwItem *item = SW_ITEM (object);
  SwService *service = SW_SERVICE (user_data);

  return sw_item_get_service (item) == service;
}

static void
sw_view_removed_items (SwView *view,
                       SwSet  *removed_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (removed_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = g_value_array_new (2);

    value_array = g_value_array_append (value_array, NULL);
    g_value_init (g_value_array_get_nth (value_array, 0), G_TYPE_STRING);
    g_value_set_string (g_value_array_get_nth (value_array, 0),
                        sw_service_get_name (sw_item_get_service (item)));

    value_array = g_value_array_append (value_array, NULL);
    g_value_init (g_value_array_get_nth (value_array, 1), G_TYPE_STRING);
    g_value_set_string (g_value_array_get_nth (value_array, 1),
                        sw_item_get (item, "id"));

    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_removed (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}

static void
sw_view_added_items (SwView *view,
                     SwSet  *added_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (added_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = _sw_item_to_value_array (item);
    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_added (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}


static void
sw_view_changed_items (SwView *view,
                       SwSet  *added_items)
{
  GValueArray *value_array;
  GPtrArray *ptr_array;
  GList *items, *l;

  items = sw_set_as_list (added_items);
  ptr_array = g_ptr_array_new ();

  for (l = items; l; l = l->next)
  {
    SwItem *item = (SwItem *)l->data;
    value_array = _sw_item_to_value_array (item);
    g_ptr_array_add (ptr_array, value_array);
  }

  sw_view_iface_emit_items_changed (view, ptr_array);

  g_ptr_array_foreach (ptr_array, (GFunc)g_value_array_free, NULL);
  g_ptr_array_free (ptr_array, TRUE);
}

static gboolean
_filter_only_changed_cb (SwSet  *set,
                         SwItem *item,
                         SwView *view)
{
  if (sw_item_get_mtime (item) >= view->priv->last_recalculate_time)
    return TRUE;
  else
    return FALSE;
}

void
sw_view_recalculate (SwView *view)
{
  SwViewPrivate *priv;
  SwSet *old_items, *new_items;
  SwSet *removed_items, *added_items, *changed_items = NULL;

  g_return_if_fail (SW_IS_VIEW (view));

  priv = view->priv;
  if (!priv->running)
    return;

  old_items = priv->current;
  new_items = munge_items (view);

  removed_items = sw_set_difference (old_items, new_items);
  added_items = sw_set_difference (new_items, old_items);

  if (priv->last_recalculate_time != 0)
  {
    changed_items = sw_set_filter (new_items,
                                   (SwSetFilterFunc)_filter_only_changed_cb,
                                   view);
  }

  if (!sw_set_is_empty (removed_items))
    sw_view_removed_items (view, removed_items);

  if (!sw_set_is_empty (added_items))
    sw_view_added_items (view, added_items);

  if (changed_items && !sw_set_is_empty (changed_items))
    sw_view_changed_items (view, changed_items);

  sw_set_unref (removed_items);
  sw_set_unref (added_items);

  if (changed_items)
    sw_set_unref (changed_items);

  sw_set_unref (priv->current);
  priv->current = new_items;
  priv->last_recalculate_time = time (NULL);
}

static gboolean
recalculate_timeout_cb (SwView *view)
{
  view->priv->recalculate_timeout_id = 0;
  sw_view_recalculate (view);

  return FALSE;
}

static void
sw_view_queue_recalculate (SwView *view)
{
  SwViewPrivate *priv;

  priv = view->priv;

  if (priv->recalculate_timeout_id)
  {
    g_source_remove (priv->recalculate_timeout_id);
  }

  priv->recalculate_timeout_id =
    g_timeout_add (RECALCULATE_DELAY,
                   (GSourceFunc)recalculate_timeout_cb,
                   view);
}

static void
_item_ready_notify_cb (SwItem     *item,
                       GParamSpec *pspec,
                       SwView     *view)
{
  /* TODO: Use a timeout to rate limit this */
  if (sw_item_get_ready (item)) {
    SW_DEBUG (VIEWS, "Item became ready: %s.",
                  sw_item_get (item, "id"));
    sw_view_queue_recalculate (view);
    g_signal_handlers_disconnect_by_func (item,
                                          _item_ready_notify_cb,
                                          view);
  }
}

static void
_item_ready_weak_notify_cb (gpointer  data,
                            GObject  *dead_object)
{
    g_signal_handlers_disconnect_by_func (data,
                                          _item_ready_notify_cb,
                                          dead_object);
}

static void
setup_ready_handler (gpointer object, gpointer user_data)
{
  SwItem *item = SW_ITEM (object);
  SwView *view = SW_VIEW (user_data);

  if (sw_item_get_ready (item))
    return;

  SW_DEBUG (VIEWS, "Item not ready. Setting up signal handler: %s.",
                sw_item_get (item, "id"));

  g_signal_connect (item,
                    "notify::ready",
                    (GCallback)_item_ready_notify_cb,
                    view);
  g_object_weak_ref ((GObject *)view,
                     _item_ready_weak_notify_cb,
                     item);
}

static void
service_updated (SwService *service,
                 SwSet     *set,
                 gpointer   user_data)
{
  SwView *view = SW_VIEW (user_data);
  SwViewPrivate *priv = view->priv;

  if (set) {
    sw_cache_save (service, set);
{'fraction_non_alphanumeric': 0.10141509433962265, 'fraction_numerical': 0.00362844702467344, 'mean_word_length': 2.935915744377008, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4701895', 'n_tokens_mistral': 7676, 'n_tokens_neox': 7462, 'n_words': 1933}