<commit_before>/*
 * GeeXboX Valhalla: tiny media scanner API.
 * Copyright (C) 2009 <NAME> <<EMAIL>>
 *
 * This file is part of libvalhalla.
 *
 * libvalhalla is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * libvalhalla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with libvalhalla; if not, write to the Free Software
 * Foundation, Inc, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#ifndef VALHALLA_GRABBER_COMMON_H
#define VALHALLA_GRABBER_COMMON_H

/**
 * \file grabber_common.h
 *
 * GeeXboX Valhalla Grabber private API header.
 *
 * To add a new grabber, a good approach is to copy an existing grabber like
 * grabber_dummy.[ch] in order to have at least the structure. A grabber must
 * not use global/static variables. A grabber must be thread-safe in the case
 * where more than one instance of Valhalla are running concurrently. But, the
 * functions in one grabber are not called in concurrency in one instance
 * of Valhalla.
 *
 * Some others points to consider:
 *  - grabber_list_t::init() and grabber_list_t::uninit() functions are called
 *    only one time by a Valhalla instance.
 *  - grabber_list_t::grab() function is called only between the
 *    grabber_list_t::init() and grabber_list_t::uninit() functions.
 *  - grabber_list_t::loop() function is called only between two scanner loops.
 *    The function is not called if only one loop is configured for the instance
 *    of Valhalla.
 *
 * Some utils are available for the grabbers:
 *  - xml_utils.h for XML parsing (based on libxml2)
 *  - url_utils.h for downloading (based on libcurl)
 *  - logs.h      for logging
 *  - md5.h       to compute the MD5 sum
 *  - list.h      to handle very simple linked-lists
 *
 * Main header for all grabbers:
 *  - grabber_common.h
 *    - metadata.h to save metadata retrieved by grabber_list_t::grab()
 *    - utils.h    to prepare files (images, ...) for downloading
 *
 * \see grabber_list_t for details on the functions.
 */

#include "utils.h"

/** \name Flags for the capabilities of the grabbers.
 * @{
 */
#define GRABBER_CAP_AUDIO  (1 << 0) /**< \brief audio metadata: id3tags,...   */
#define GRABBER_CAP_VIDEO  (1 << 1) /**< \brief video metadata: fourccs,...   */
#define GRABBER_CAP_IMAGE  (1 << 2) /**< \brief photo metadata: exif,...      */
#define GRABBER_CAP_COVER  (1 << 3) /**< \brief covers, thumbs, snapshots,... */
/**
 *@}
 */

/**
 * \brief Structure for a grabber.
 */
typedef struct grabber_list_s {
  struct grabber_list_s *next;

  /** \brief Textual identification of the grabber. */
  const char *name;
  /** \brief Flags to define the capabilities of the grabber. */
  int caps_flag;

  /**
   * \brief Init function for the grabber.
   *
   * This initialization is called only at the init of an instance of valhalla.
   * The private structure (\p priv) must be created before this initialization.
   *
   * \param[in] priv      Private structure registered with the grabber.
   * \return 0 for success, != 0 on error.
   */
  int (*init) (void *priv);

  /**
   * \brief Uninit function for the grabber.
   *
   * This unititialization is called only at the uninit of an instance of
   * valhalla. The private structure (\p priv) must be released in this
   * function.
   *
   * \param[in] priv      Private structure registered with the grabber.
   */
  void (*uninit) (void *priv);

  /**
   * \brief Grabbing function for the grabber.
   *
   * This function is called in order to populate the attributes \p meta_grabber
   * and \p list_downloader in the \p data structure. All others attributes must
   * be considered as read-only! Only them are thread-safe for writing.
   *
   * To add new metadata in the database, the function metadata_add() must be
   * used on \p meta_grabber.
   *
   * It is proibited to download files (images for example) with this function.
   * Only textual metadata are proceeded here. But the reference on an image
   * can be saved in the meta_grabber attribute.
   * To download a file, the URL and the destination must be prepared for the
   * downloader with the function file_dl_add() in order to populate the
   * \p list_downloader attribute. The files will be downloaded after the
   * grabbing step.
   *
   * To read \p meta (attribute populated by the parser), you must use the
   * function metadata_get().
   *<commit_msg>missing one meta > meta_parser (see r90)<commit_after>/*
 * GeeXboX Valhalla: tiny media scanner API.
 * Copyright (C) 2009 <NAME> <<EMAIL>>
 *
 * This file is part of libvalhalla.
 *
 * libvalhalla is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * libvalhalla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with libvalhalla; if not, write to the Free Software
 * Foundation, Inc, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#ifndef VALHALLA_GRABBER_COMMON_H
#define VALHALLA_GRABBER_COMMON_H

/**
 * \file grabber_common.h
 *
 * GeeXboX Valhalla Grabber private API header.
 *
 * To add a new grabber, a good approach is to copy an existing grabber like
 * grabber_dummy.[ch] in order to have at least the structure. A grabber must
 * not use global/static variables. A grabber must be thread-safe in the case
 * where more than one instance of Valhalla are running concurrently. But, the
 * functions in one grabber are not called in concurrency in one instance
 * of Valhalla.
 *
 * Some others points to consider:
 *  - grabber_list_t::init() and grabber_list_t::uninit() functions are called
 *    only one time by a Valhalla instance.
 *  - grabber_list_t::grab() function is called only between the
 *    grabber_list_t::init() and grabber_list_t::uninit() functions.
 *  - grabber_list_t::loop() function is called only between two scanner loops.
 *    The function is not called if only one loop is configured for the instance
 *    of Valhalla.
 *
 * Some utils are available for the grabbers:
 *  - xml_utils.h for XML parsing (based on libxml2)
 *  - url_utils.h for downloading (based on libcurl)
 *  - logs.h      for logging
 *  - md5.h       to compute the MD5 sum
 *  - list.h      to handle very simple linked-lists
 *
 * Main header for all grabbers:
 *  - grabber_common.h
 *    - metadata.h to save metadata retrieved by grabber_list_t::grab()
 *    - utils.h    to prepare files (images, ...) for downloading
 *
 * \see grabber_list_t for details on the functions.
 */

#include "utils.h"

/** \name Flags for the capabilities of the grabbers.
 * @{
 */
#define GRABBER_CAP_AUDIO  (1 << 0) /**< \brief audio metadata: id3tags,...   */
#define GRABBER_CAP_VIDEO  (1 << 1) /**< \brief video metadata: fourccs,...   */
#define GRABBER_CAP_IMAGE  (1 << 2) /**< \brief photo metadata: exif,...      */
#define GRABBER_CAP_COVER  (1 << 3) /**< \brief covers, thumbs, snapshots,... */
/**
 *@}
 */

/**
 * \brief Structure for a grabber.
 */
typedef struct grabber_list_s {
  struct grabber_list_s *next;

  /** \brief Textual identification of the grabber. */
  const char *name;
  /** \brief Flags to define the capabilities of the grabber. */
  int caps_flag;

  /**
   * \brief Init function for the grabber.
   *
   * This initialization is called only at the init of an instance of valhalla.
   * The private structure (\p priv) must be created before this initialization.
   *
   * \param[in] priv      Private structure registered with the grabber.
   * \return 0 for success, != 0 on error.
   */
  int (*init) (void *priv);

  /**
   * \brief Uninit function for the grabber.
   *
   * This unititialization is called only at the uninit of an instance of
   * valhalla. The private structure (\p priv) must be released in this
   * function.
   *
   * \param[in] priv      Private structure registered with the grabber.
   */
  void (*uninit) (void *priv);

  /**
   * \brief Grabbing function for the grabber.
   *
   * This function is called in order to populate the attributes \p meta_grabber
   * and \p list_downloader in the \p data structure. All others attributes must
   * be considered as read-only! Only them are thread-safe for writing.
   *
   * To add new metadata in the database, the function metadata_add() must be
   * used on \p meta_grabber.
   *
   * It is proibited to download files (images for example) with this function.
   * Only textual metadata are proceeded here. But the reference on an image
   * can be saved in the meta_grabber attribute.
   * To download a file, the URL and the destination must be prepared for the
   * downloader with the function file_dl_add() in order to populate the
   * \p list_downloader attribute. The files will be downloaded after the
   * grabbing step.
   *
   * To read \p meta_parser (attribute populated by the parser), you must use
   * the function metadata_get().
   *
{'fraction_non_alphanumeric': 0.08634206725809779, 'fraction_numerical': 0.006602021869197442, 'mean_word_length': 3.1734825656478693, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '618549', 'n_tokens_mistral': 2931, 'n_tokens_neox': 2830, 'n_words': 1257}