<commit_before>// API workflow?

#include <stdlib.h>

#include <curl/curl.h>
#include <jansson.h>

#include "se_query_builder.h"
#include "stackexchange.h"

// temp file naming scheme
static char TEMP_FILE_FORMAT[] = "/tmp/staq-working-XXXX";

// the shared CURL handle
static int g_curl = 0;

void SEInit() {
   if(!g_curl) {
      curl_global_init(CURL_GLOBAL_DEFAULT);
      g_curl = 1;
   }
}

void SECleanup() {
   if(g_curl) {
      curl_global_cleanup();
      g_curl = 0;
   }
}

/**
 * Assemble a single answer from its JSON
 */
static SEError SEPopulateAnswer(SEAnswer* answer, json_t* json) {
   if(!answer || !json) {
      fprintf(stderr, "PopulateAnswer: bad params\n");
      return SE_ERROR;
   }

   if(!json_is_object(json)){
      fprintf(stderr, "error: root is not an object\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   json_error_t jerror;
   if(json_unpack_ex(json, &jerror, 0,
            "{s:b s:i s:i s:i s?:i s:i s:s s:s}",
            "is_accepted",          &answer->isAccepted,
            "score",                &answer->score,
            "last_activity_date",   &answer->dateLastActivity,
            "creation_date",        &answer->dateCreation,
            "last_edit_date",       &answer->dateLastEdit,
            "answer_id",            &answer->answerId,
            "body",                 &answer->body,
            "body_markdown",        &answer->bodyMarkdown
            )) {
      fprintf(stderr, "error: on line %d: %s\n", jerror.line, jerror.text);
      return SE_JSON_ERROR;
   }
   return SE_OK;
}

/**
 * Assemble a single question from its JSON (including any answers)
 */
static SEError SEPopulateQuestion(SEQuestion* question, json_t* json) {
   if(!question || !json) {
      fprintf(stderr, "PopulateQuestion: bad params\n");
      return SE_ERROR;
   }

   if(!json_is_object(json)){
      fprintf(stderr, "error: root is not an object\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   json_error_t jerror;
   if(json_unpack_ex(json, &jerror, 0,
            "{s:i s:b s?:i s:i s:i s:i s:i s?:i s:s s:s s:s s:s}",
            "question_id",          &question->questionId,
            "is_answered",          &question->isAnswered,
            "accepted_answer_id",   &question->acceptedAnswerId,
            "answer_count",         &question->answerCount,
            "score",                &question->score,
            "last_activity_date",   &question->dateLastActivity,
            "creation_date",        &question->dateCreation,
            "last_edit_date",       &question->dateLastEdit,
            "link",                 &question->url,
            "title",                &question->title,
            "body",                 &question->body,
            "body_markdown",        &question->bodyMarkdown
            )) {
      fprintf(stderr, "error: on line %d: %s\n", jerror.line, jerror.text);
      return SE_JSON_ERROR;
   }

   printf("old qid = %d\n", question->questionId);

   json_t* janswers = json_object_get(json, "answers");
   if(!janswers) {
      fprintf(stderr, "error: couldn't get question's answer items\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   int numAnswers = json_array_size(janswers);
   printf("\t%d answers\n", numAnswers);
   question->answers = (SEAnswer*)malloc(numAnswers * sizeof(SEAnswer));

   json_t* jcur;
   for(int i = 0; i < numAnswers; i++) {
      jcur = json_array_get(janswers, i);

      if(SEPopulateAnswer(&(question->answers[i]), jcur) != SE_OK) {
         fprintf(stderr, "error: populating answer %d\n", i);
         json_decref(jcur);
         return SE_ERROR;
      }
   }

   return SE_OK;
}

/**
 * Generic 'run a query and return the decoded JSON of the response'
 */
static SEError SEQueryAdvanced(json_t** json, SEStructuredQuery* query, SEQueryOptions* seqo) {
   CURLcode res;

   if(!query) {
      return SE_BAD_PARAM;
   }

   *json = NULL;

   // instead of writing the curl response into memory, spool it to a temp file
   int cfd = mkstemp(TEMP_FILE_FORMAT);
   if(cfd == -1) {
      perror("opening the temp file");
      return SE_ERROR;
   }

   FILE* cfile = fdopen(cfd, "w+");
   if(!cfile) {
      perror("opening the temp file");
      return SE_ERROR;
   }

   CURL* curl = curl_easy_init();

   // follow redirects
   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
   // stack exchange gives us gzipped content, so ask curl to gunzip for us
   curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
   // set the url
   curl_easy_setopt(curl, CURLOPT_URL, query->url);
   // ask curl to write to our file
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, cfile);

   // Perform the actual request, res will get the return code
   res = curl_easy_perform(curl);

   /* Check for errors */ 
   if(res != CURLE_OK) {
      fprintf(stderr, "curl_easy_perform() failed: %s\n",
            curl_easy_strerror(res));
      return SE_CURL_ERROR;
   }

   // reset output stream to start
   fseek(cfile, 0, SEEK_SET);

   // check for errors and return status
   json_error_t jerr;
   *json = json_loadf(cfile, 0, &jerr);

   // we don't need this curl session anymore
   curl_easy_cleanup(curl);

   // TODO unlink temp file

   if(!*json) {
      fprintf(stderr, "error: on line %d: %s\n", jerr.line, jerr.text);
      return SE_JSON_ERROR;
   }

   return SE_OK;<commit_msg>switched to a more sane tempfile function<commit_after>// API workflow?

#include <stdlib.h>

#include <curl/curl.h>
#include <jansson.h>

#include "se_query_builder.h"
#include "stackexchange.h"

// the shared CURL handle
static int g_curl = 0;

void SEInit() {
   if(!g_curl) {
      curl_global_init(CURL_GLOBAL_DEFAULT);
      g_curl = 1;
   }
}

void SECleanup() {
   if(g_curl) {
      curl_global_cleanup();
      g_curl = 0;
   }
}

/**
 * Assemble a single answer from its JSON
 */
static SEError SEPopulateAnswer(SEAnswer* answer, json_t* json) {
   if(!answer || !json) {
      fprintf(stderr, "PopulateAnswer: bad params\n");
      return SE_ERROR;
   }

   if(!json_is_object(json)){
      fprintf(stderr, "error: root is not an object\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   json_error_t jerror;
   if(json_unpack_ex(json, &jerror, 0,
            "{s:b s:i s:i s:i s?:i s:i s:s s:s}",
            "is_accepted",          &answer->isAccepted,
            "score",                &answer->score,
            "last_activity_date",   &answer->dateLastActivity,
            "creation_date",        &answer->dateCreation,
            "last_edit_date",       &answer->dateLastEdit,
            "answer_id",            &answer->answerId,
            "body",                 &answer->body,
            "body_markdown",        &answer->bodyMarkdown
            )) {
      fprintf(stderr, "error: on line %d: %s\n", jerror.line, jerror.text);
      return SE_JSON_ERROR;
   }
   return SE_OK;
}

/**
 * Assemble a single question from its JSON (including any answers)
 */
static SEError SEPopulateQuestion(SEQuestion* question, json_t* json) {
   if(!question || !json) {
      fprintf(stderr, "PopulateQuestion: bad params\n");
      return SE_ERROR;
   }

   if(!json_is_object(json)){
      fprintf(stderr, "error: root is not an object\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   json_error_t jerror;
   if(json_unpack_ex(json, &jerror, 0,
            "{s:i s:b s?:i s:i s:i s:i s:i s?:i s:s s:s s:s s:s}",
            "question_id",          &question->questionId,
            "is_answered",          &question->isAnswered,
            "accepted_answer_id",   &question->acceptedAnswerId,
            "answer_count",         &question->answerCount,
            "score",                &question->score,
            "last_activity_date",   &question->dateLastActivity,
            "creation_date",        &question->dateCreation,
            "last_edit_date",       &question->dateLastEdit,
            "link",                 &question->url,
            "title",                &question->title,
            "body",                 &question->body,
            "body_markdown",        &question->bodyMarkdown
            )) {
      fprintf(stderr, "error: on line %d: %s\n", jerror.line, jerror.text);
      return SE_JSON_ERROR;
   }

   printf("old qid = %d\n", question->questionId);

   json_t* janswers = json_object_get(json, "answers");
   if(!janswers) {
      fprintf(stderr, "error: couldn't get question's answer items\n");
      json_decref(json);
      return SE_JSON_ERROR;
   }

   int numAnswers = json_array_size(janswers);
   printf("\t%d answers\n", numAnswers);
   question->answers = (SEAnswer*)malloc(numAnswers * sizeof(SEAnswer));

   json_t* jcur;
   for(int i = 0; i < numAnswers; i++) {
      jcur = json_array_get(janswers, i);

      if(SEPopulateAnswer(&(question->answers[i]), jcur) != SE_OK) {
         fprintf(stderr, "error: populating answer %d\n", i);
         json_decref(jcur);
         return SE_ERROR;
      }
   }

   return SE_OK;
}

/**
 * Generic 'run a query and return the decoded JSON of the response'
 */
static SEError SEQueryAdvanced(json_t** json, SEStructuredQuery* query, SEQueryOptions* seqo) {
   CURLcode res;

   if(!query) {
      return SE_BAD_PARAM;
   }

   *json = NULL;

   FILE* cfile = tmpfile();
   if(!cfile) {
      perror("opening the temp file");
      return SE_ERROR;
   }

   CURL* curl = curl_easy_init();

   // follow redirects
   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
   // stack exchange gives us gzipped content, so ask curl to gunzip for us
   curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
   // set the url
   curl_easy_setopt(curl, CURLOPT_URL, query->url);
   // ask curl to write to our file
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, cfile);

   // Perform the actual request, res will get the return code
   res = curl_easy_perform(curl);

   /* Check for errors */ 
   if(res != CURLE_OK) {
      fprintf(stderr, "curl_easy_perform() failed: %s\n",
            curl_easy_strerror(res));
      return SE_CURL_ERROR;
   }

   // reset output stream to start
   fseek(cfile, 0, SEEK_SET);

   // check for errors and return status
   json_error_t jerr;
   *json = json_loadf(cfile, 0, &jerr);

   // we don't need this curl session anymore
   curl_easy_cleanup(curl);

   if(!*json) {
      fprintf(stderr, "error: on line %d: %s\n", jerr.line, jerr.text);
      return SE_JSON_ERROR;
   }

   return SE_OK;
{'fraction_non_alphanumeric': 0.12893352462936394, 'fraction_numerical': 0.0018173122907699664, 'mean_word_length': 2.46684350132626, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405717', 'n_tokens_mistral': 3691, 'n_tokens_neox': 3499, 'n_words': 817}