<commit_before> /* This program is free software. It comes without any warranty, to
 * the extent permitted by applicable law. You can redistribute it
 * and/or modify it under the terms of the Do What The Fuck You Want
 * To Public License, Version 2, as published by Sam Hocevar. See
 * http://sam.zoy.org/wtfpl/COPYING for more details. */ 


#define CPLANET_VERSION "0.1"
#include <stdio.h>
#include <unistd.h>
#include <err.h>
#include <string.h>
#include <time.h>
#include <mrss.h>
#include <cs/cs.h>
#include <iconv.h>
#include <util/neo_misc.h>
#include <util/neo_hdf.h>


/* function to create the atom feed */
/*void 
create_atom(HDF *hdf)
{
    mrss_t *d;
    mrss_error_t err;
    HDF *subhdf;
    d=NULL; // ->this is important! If d!=NULL, mrss_new doesn't alloc memory.
    mrss_new(&d);

    err=mrss_set (d,
	    MRSS_FLAG_VERSION, MRSS_VERSION_2_0,
	    MRSS_FLAG_TITLE, hdf_get_valuef(hdf,"CPlanet.Name"),
	    MRSS_FLAG_TTL, 12,
	    MRSS_FLAG_END);

    if(err!=MRSS_OK) warn("bla%s\n",mrss_strerror(err));

    subhdf = hdf_get_obj(hdf,"CPlanet.Posts.0");
    while ((subhdf = hdf_obj_next(subhdf)) != NULL) {
	mrss_t *item = 
    }

    err=mrss_write_file( d, "/home/bapt/planet/www/index.atom");
    if(err!=MRSS_OK) warn("%s\n",mrss_strerror(err));
}
*/
/* convert RFC822 to epoch time */

int
date_to_unix(char *s)
{
    struct tm date;
    char date_s [16];
    strptime(s,"%a, %d %b %Y %T",&date);
    strftime (date_s, sizeof date_s, "%s", &date);
    return atoi(date_s);
}

/* sort posts by date */

int 
sort_obj_by_date(const void *a, const void *b) {
    HDF **ha = (HDF **)a;
    HDF **hb = (HDF **)b;
    int atime = date_to_unix(hdf_get_valuef(*ha,"Date"));
    int btime = date_to_unix(hdf_get_valuef(*hb,"Date"));
    if  (atime<btime) return 1;
    if (atime==btime) return 0;
    return -1;
}

/* prepare the string to be written to the html file */

static NEOERR 
*output (void *ctx, char *s)
{
    STRING *str = (STRING *)ctx;
    NEOERR *err;

    err = nerr_pass(string_append(str, s));
    return err;
}

char * 
convert_to_Unicode(char *source_encoding, char *str)
{
    size_t insize;
    size_t outsize;
    size_t ret;
    char *output;
    if (!strcmp(source_encoding,"UTF-8"))
	return str;
    
    iconv_t conv;
    const char * inptr = str;
    conv = iconv_open("UTF-8",source_encoding);
    if (conv == (iconv_t) -1) {
	warn("conversion from '%s' to UTF-8ot available",source_encoding);
	return str;
    }
    if (str == NULL)
	return str;
    insize = strlen(str);
    outsize = 4 * insize + 2;
    output=malloc(outsize);
    memset(output,0,outsize);
    char *outputptr = output;
    ret = iconv (conv,(const char**) &inptr,&insize,&outputptr,&outsize);
    if(ret ==(size_t) -1) {
	warn("Conversion Failed");
	free(output);
	iconv_close(conv);
	return str;
    }
    iconv_close(conv);
    return output;
}

/* retreive ports and prepare the dataset for the template */

int 
get_posts(HDF *hdf_cfg, HDF* hdf_dest, int pos, int days)
{
    mrss_t *feed;
    mrss_error_t ret;
    mrss_item_t *item;
    CURLcode code;
    char *encoding;
    char *name=hdf_get_valuef(hdf_cfg,"Name");
    ret = mrss_parse_url_with_options_and_error (hdf_get_valuef(hdf_cfg,"URL"), &feed, NULL, &code);
    if(ret)
    {
	warn( "MRSS return error: %s, %s\n",
		ret ==
		MRSS_ERR_DOWNLOAD ? mrss_curl_strerror (code) :
		mrss_strerror (ret),name);
	return pos;
    }
    encoding=feed->encoding;
    for (item = feed->item;item;item=item->next)
    {
	struct tm date;
	time_t t;
	char date_now[16];
	char date_comp[16];
	strptime(item->pubDate,"%a, %d %b %Y %T",&date);
	strftime (date_comp, sizeof date_comp, "%s", &date);
	time(&t);
	strftime(date_now, sizeof date_now,"%s",localtime(&t));
	if ( ( atoi(date_now) - atoi(date_comp) ) >= days ) {
	    continue;
	}
	mrss_category_t *tags;
	mrss_tag_t *content; 
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Name=%s",pos,name);
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.FeedName=%s",pos,convert_to_Unicode(encoding,feed->title));
	if(item->author != NULL)
	    hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Author=%s",pos,convert_to_Unicode(encoding,item->author));
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Title=%s",pos,convert_to_Unicode(encoding,item->title));
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Link=%s",pos,item->link);
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Date=%s",pos,item->pubDate);
	/* Description is only for description tag, we want content if exists */
	if(feed->version == MRSS_VERSION_ATOM_0_3 || feed->version == MRSS_VERSION_ATOM_1_0 ) {
	    if ((mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content) == MRSS_OK && content) || 
		    (mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content) == MRSS_OK && content)) {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,content->value));
	    } else {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,item->description));
	    }
	} else {
	    if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,content->value));
	    } else {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,item->description));
	    }
	}
	mrss_free(content);
	/* Get the categories/tags */
	int nb_tags=0;
	for (tags = item->category;tags;tags=tags->next) 
	{
	    hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Tags.%i.Tag=%s",pos,nb_tags,convert_to_Unicode(encoding,tags->category));
	    nb_tags++;
	}
	mrss_free(tags);
	pos++;
    }
    mrss_free(item);

    return pos;
}

static void
usage()
{
errx(1,"usage: cplanet -c conf.hdf\n");
}

int 
main (int argc, char *argv[])
{
    NEOERR *err;
    CSPARSE *parse;
    STRING cs_output_data;
    HDF *hdf;
    HDF *feed_hdf;
    int pos=0;
    int days=0;
    int ch=0;
    char *cspath, *cs_output, *hdf_file=NULL;
    if (argc == 1)
	usage();
    while ((ch = getopt(argc, argv, "c:h")) != -1)
	switch (ch) {  
	    case 'h':
		usage();
		break;
	    case 'c':
		hdf_file=optarg;
		/* TODO check file existence here */
		break;
	    case '?':<commit_msg>Reformat the C code, and add automatic vim configuration<commit_after>/* This program is free software. It comes without any warranty, to
 * the extent permitted by applicable law. You can redistribute it
 * and/or modify it under the terms of the Do What The Fuck You Want
 * To Public License, Version 2, as published by Sam Hocevar. See
 * http://sam.zoy.org/wtfpl/COPYING for more details. */ 

/*
 * vim: set s=4 sw=4 sts=4:
 */
#define CPLANET_VERSION "0.1"
#include <stdio.h>
#include <unistd.h>
#include <err.h>
#include <string.h>
#include <time.h>
#include <mrss.h>
#include <cs/cs.h>
#include <iconv.h>
#include <util/neo_misc.h>
#include <util/neo_hdf.h>


/* function to create the atom feed */
/*void 
  create_atom(HDF *hdf)
  {
  mrss_t *d;
  mrss_error_t err;
  HDF *subhdf;
  d=NULL; // ->this is important! If d!=NULL, mrss_new doesn't alloc memory.
  mrss_new(&d);

  err=mrss_set (d,
  MRSS_FLAG_VERSION, MRSS_VERSION_2_0,
  MRSS_FLAG_TITLE, hdf_get_valuef(hdf,"CPlanet.Name"),
  MRSS_FLAG_TTL, 12,
  MRSS_FLAG_END);

  if(err!=MRSS_OK) warn("bla%s\n",mrss_strerror(err));

  subhdf = hdf_get_obj(hdf,"CPlanet.Posts.0");
  while ((subhdf = hdf_obj_next(subhdf)) != NULL) {
  mrss_t *item = 
  }

  err=mrss_write_file( d, "/home/bapt/planet/www/index.atom");
  if(err!=MRSS_OK) warn("%s\n",mrss_strerror(err));
  }
  */
/* convert RFC822 to epoch time */

int
date_to_unix(char *s)
{
    struct tm date;
    char date_s [16];
    strptime(s,"%a, %d %b %Y %T",&date);
    strftime (date_s, sizeof date_s, "%s", &date);
    return atoi(date_s);
}

/* sort posts by date */

int 
sort_obj_by_date(const void *a, const void *b) {
    HDF **ha = (HDF **)a;
    HDF **hb = (HDF **)b;
    int atime = date_to_unix(hdf_get_valuef(*ha,"Date"));
    int btime = date_to_unix(hdf_get_valuef(*hb,"Date"));
    if  (atime<btime) return 1;
    if (atime==btime) return 0;
    return -1;
}

/* prepare the string to be written to the html file */

static NEOERR *
output (void *ctx, char *s)
{
    STRING *str = (STRING *)ctx;
    NEOERR *err;

    err = nerr_pass(string_append(str, s));
    return err;
}

char * 
convert_to_Unicode(char *source_encoding, char *str)
{
    size_t insize;
    size_t outsize;
    size_t ret;
    char *output;
    if (!strcmp(source_encoding,"UTF-8"))
	return str;

    iconv_t conv;
    const char * inptr = str;
    conv = iconv_open("UTF-8",source_encoding);
    if (conv == (iconv_t) -1) {
	warn("conversion from '%s' to UTF-8ot available",source_encoding);
	return str;
    }
    if (str == NULL)
	return str;
    insize = strlen(str);
    outsize = 4 * insize + 2;
    output=malloc(outsize);
    memset(output,0,outsize);
    char *outputptr = output;
    ret = iconv (conv,(const char**) &inptr,&insize,&outputptr,&outsize);
    if(ret ==(size_t) -1) {
	warn("Conversion Failed");
	free(output);
	iconv_close(conv);
	return str;
    }
    iconv_close(conv);
    return output;
}

/* retreive ports and prepare the dataset for the template */

int 
get_posts(HDF *hdf_cfg, HDF* hdf_dest, int pos, int days)
{
    mrss_t *feed;
    mrss_error_t ret;
    mrss_item_t *item;
    CURLcode code;
    char *encoding;
    char *name=hdf_get_valuef(hdf_cfg,"Name");
    ret = mrss_parse_url_with_options_and_error (hdf_get_valuef(hdf_cfg,"URL"), &feed, NULL, &code);
    if(ret)
    {
	warn( "MRSS return error: %s, %s\n",
		ret ==
		MRSS_ERR_DOWNLOAD ? mrss_curl_strerror (code) :
		mrss_strerror (ret),name);
	return pos;
    }
    encoding=feed->encoding;
    for (item = feed->item;item;item=item->next)
    {
	struct tm date;
	time_t t;
	char date_now[16];
	char date_comp[16];
	strptime(item->pubDate,"%a, %d %b %Y %T",&date);
	strftime (date_comp, sizeof date_comp, "%s", &date);
	time(&t);
	strftime(date_now, sizeof date_now,"%s",localtime(&t));
	if ( ( atoi(date_now) - atoi(date_comp) ) >= days ) {
	    continue;
	}
	mrss_category_t *tags;
	mrss_tag_t *content; 
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Name=%s",pos,name);
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.FeedName=%s",pos,convert_to_Unicode(encoding,feed->title));
	if(item->author != NULL)
	    hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Author=%s",pos,convert_to_Unicode(encoding,item->author));
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Title=%s",pos,convert_to_Unicode(encoding,item->title));
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Link=%s",pos,item->link);
	hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Date=%s",pos,item->pubDate);
	/* Description is only for description tag, we want content if exists */
	if(feed->version == MRSS_VERSION_ATOM_0_3 || feed->version == MRSS_VERSION_ATOM_1_0 ) {
	    if ((mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content) == MRSS_OK && content) || 
		    (mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content) == MRSS_OK && content)) {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,content->value));
	    } else {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,item->description));
	    }
	} else {
	    if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,content->value));
	    } else {
		hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Description=%s",pos,convert_to_Unicode(encoding,item->description));
	    }
	}
	mrss_free(content);
	/* Get the categories/tags */
	int nb_tags=0;
	for (tags = item->category;tags;tags=tags->next) 
	{
	    hdf_set_valuef(hdf_dest,"CPlanet.Posts.%i.Tags.%i.Tag=%s",pos,nb_tags,convert_to_Unicode(encoding,tags->category));
	    nb_tags++;
	}
	mrss_free(tags);
	pos++;
    }
    mrss_free(item);

    return pos;
}

static void
usage()
{
    errx(1,"usage: cplanet -c conf.hdf\n");
}

int 
main (int argc, char *argv[])
{
    NEOERR *err;
    CSPARSE *parse;
    STRING cs_output_data;
    HDF *hdf;
    HDF *feed_hdf;
    int pos=0;
    int days=0;
    int ch=0;
    char *cspath, *cs_output, *hdf_file=NULL;
    if (argc == 1)
	usage();
    while ((ch = getopt(argc, argv, "c:h")) != -1)
	switch (ch) {  
	    case 'h':
		usage();
		break;
	    case 'c':
		hdf_file=optarg;
		/* TODO check file existence here */
		break;
	    case '?':
{'fraction_non_alphanumeric': 0.16363057831516983, 'fraction_numerical': 0.0075570758094025935, 'mean_word_length': 3.988888888888889, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4058068', 'n_tokens_mistral': 5269, 'n_tokens_neox': 5097, 'n_words': 1007}