<commit_before>/*
 * GeeXboX Valhalla: tiny media scanner API.
 * Copyright (C) 2009 <NAME> <<EMAIL>>
 *
 * This file is part of libvalhalla.
 *
 * libvalhalla is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * libvalhalla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with libvalhalla; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "valhalla.h"
#include "valhalla_internals.h"
#include "utils.h"
#include "url_utils.h"
#include "fifo_queue.h"
#include "logs.h"
#include "thread_utils.h"
#include "dispatcher.h"
#include "downloader.h"

struct downloader_s {
  valhalla_t   *valhalla;
  pthread_t     thread;
  fifo_queue_t *fifo;
  int           priority;

  int             run;
  pthread_mutex_t mutex_run;

  url_t *url_handler;
  char **dl_list;
};


static inline int
downloader_is_stopped (downloader_t *downloader)
{
  int run;
  pthread_mutex_lock (&downloader->mutex_run);
  run = downloader->run;
  pthread_mutex_unlock (&downloader->mutex_run);
  return !run;
}

static void *
downloader_thread (void *arg)
{
  int res;
  int e;
  void *data = NULL;
  file_data_t *pdata;
  downloader_t *downloader = arg;

  if (!downloader)
    pthread_exit (NULL);

  vh_setpriority (downloader->priority);

  do
  {
    e = ACTION_NO_OPERATION;
    data = NULL;

    res = fifo_queue_pop (downloader->fifo, &e, &data);
    if (res || e == ACTION_NO_OPERATION)
      continue;

    if (e == ACTION_KILL_THREAD)
      break;

    pdata = data;

    if (pdata->list_downloader)
    {
      file_dl_t *it = pdata->list_downloader;
      for (; it; it = it->next)
      {
        char *dest;
        size_t len;
        valhalla_dl_t dst = it->dst;
        int err;

        if (downloader_is_stopped (downloader))
          break;

        if (!it->url || dst >= VALHALLA_DL_LAST || dst < 0)
          continue;

        if (!downloader->dl_list[dst] || !downloader->dl_list[dst][0])
          dst = VALHALLA_DL_DEFAULT;

        if (!downloader->dl_list[dst] || !downloader->dl_list[dst][0])
          continue;

        len = strlen (downloader->dl_list[dst]) + strlen (it->name) + 2;
        dest = malloc (len);
        if (!dest)
          continue;

        snprintf (dest, len, "%s%s%s",
                  downloader->dl_list[dst],
                  *(strrchr (downloader->dl_list[dst], '\0') - 1) == '/'
                  ? "" : "/",
                  it->name);
        err = url_save_to_disk (downloader->url_handler, it->url, dest);
        if (!err)
          valhalla_log (VALHALLA_MSG_VERBOSE, "[%s] %s saved to %s",
                        __FUNCTION__, it->url, downloader->dl_list[dst]);
        free (dest);
      }
    }

    file_data_step_increase (pdata, &e);
    dispatcher_action_send (downloader->valhalla->dispatcher, e, pdata);
  }
  while (!downloader_is_stopped (downloader));
<commit_msg>check if url is not actually a local file and, if so, only copy the file instead of downloading it through curl (which would fail)<commit_after>/*
 * GeeXboX Valhalla: tiny media scanner API.
 * Copyright (C) 2009 <NAME> <<EMAIL>>
 *
 * This file is part of libvalhalla.
 *
 * libvalhalla is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * libvalhalla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with libvalhalla; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "valhalla.h"
#include "valhalla_internals.h"
#include "utils.h"
#include "url_utils.h"
#include "fifo_queue.h"
#include "logs.h"
#include "thread_utils.h"
#include "dispatcher.h"
#include "downloader.h"

struct downloader_s {
  valhalla_t   *valhalla;
  pthread_t     thread;
  fifo_queue_t *fifo;
  int           priority;

  int             run;
  pthread_mutex_t mutex_run;

  url_t *url_handler;
  char **dl_list;
};


static inline int
downloader_is_stopped (downloader_t *downloader)
{
  int run;
  pthread_mutex_lock (&downloader->mutex_run);
  run = downloader->run;
  pthread_mutex_unlock (&downloader->mutex_run);
  return !run;
}

static void *
downloader_thread (void *arg)
{
  int res;
  int e;
  void *data = NULL;
  file_data_t *pdata;
  downloader_t *downloader = arg;

  if (!downloader)
    pthread_exit (NULL);

  vh_setpriority (downloader->priority);

  do
  {
    e = ACTION_NO_OPERATION;
    data = NULL;

    res = fifo_queue_pop (downloader->fifo, &e, &data);
    if (res || e == ACTION_NO_OPERATION)
      continue;

    if (e == ACTION_KILL_THREAD)
      break;

    pdata = data;

    if (pdata->list_downloader)
    {
      file_dl_t *it = pdata->list_downloader;
      for (; it; it = it->next)
      {
        char *dest;
        size_t len;
        valhalla_dl_t dst = it->dst;
        int err;

        if (downloader_is_stopped (downloader))
          break;

        if (!it->url || dst >= VALHALLA_DL_LAST || dst < 0)
          continue;

        if (!downloader->dl_list[dst] || !downloader->dl_list[dst][0])
          dst = VALHALLA_DL_DEFAULT;

        if (!downloader->dl_list[dst] || !downloader->dl_list[dst][0])
          continue;

        len = strlen (downloader->dl_list[dst]) + strlen (it->name) + 2;
        dest = malloc (len);
        if (!dest)
          continue;

        snprintf (dest, len, "%s%s%s",
                  downloader->dl_list[dst],
                  *(strrchr (downloader->dl_list[dst], '\0') - 1) == '/'
                  ? "" : "/",
                  it->name);

        if (*it->url == '/')
          err = file_copy (it->url, dest);
        else
          err = url_save_to_disk (downloader->url_handler, it->url, dest);
        if (!err)
          valhalla_log (VALHALLA_MSG_VERBOSE, "[%s] %s saved to %s",
                        __FUNCTION__, it->url, downloader->dl_list[dst]);
        free (dest);
      }
    }

    file_data_step_increase (pdata, &e);
    dispatcher_action_send (downloader->valhalla->dispatcher, e, pdata);
  }
  while (!downloader_is_stopped (downloader));

{'fraction_non_alphanumeric': 0.11788675003512716, 'fraction_numerical': 0.006463397498946185, 'mean_word_length': 2.736482939632546, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 61, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4392915', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2350, 'n_words': 679}