<commit_before>#include <typhoeus_multi.h>

static void multi_read_info(VALUE self, CURLM *multi_handle);

static void dealloc(CurlMulti *curl_multi) {
  curl_multi_cleanup(curl_multi->multi);
  free(curl_multi);
}

static VALUE multi_add_handle(VALUE self, VALUE easy) {
  CurlEasy *curl_easy;
  Data_Get_Struct(easy, CurlEasy, curl_easy);
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);
  CURLMcode mcode;

  mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    rb_raise(rb_eRuntimeError, "An error occured adding the handle: %d", mcode);
  }

  curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
  curl_multi->active++;

  if (mcode == CURLM_CALL_MULTI_PERFORM) {
    curl_multi_perform(curl_multi->multi, &(curl_multi->running));
  }
  // 
  // if (curl_multi->running) {
  //     printf("call read_info on add<br/>");
  //   multi_read_info(self, curl_multi->multi);
  // }

  return easy;
}

static VALUE multi_remove_handle(VALUE self, VALUE easy) {
  CurlEasy *curl_easy;
  Data_Get_Struct(easy, CurlEasy, curl_easy);
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);

  curl_multi->active--;
  curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);

  return easy;
}

static void multi_read_info(VALUE self, CURLM *multi_handle) {
  int msgs_left, result;
  CURLMsg *msg;
  CURLcode ecode;
  CURL *easy_handle;
  VALUE easy;

  /* check for finished easy handles and remove from the multi handle */
  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {

    if (msg->msg != CURLMSG_DONE) {
      continue;
    }

    easy_handle = msg->easy_handle;
    result = msg->data.result;
    if (easy_handle) {
      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
      if (ecode != 0) {
        rb_raise(rb_eRuntimeError, "error getting easy object:%d", ecode);
      }

      long response_code = -1;
      curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);

      // TODO: find out what the real problem is here and fix it.
      // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
      // fail intermittently and return this result number. However, it will succeed if you try it a few
      // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!

      // Sandofsky says:
      // This is caused by OS X first attempting to resolve using IPV6.
      // Hack solution: connect to yourself with 127.0.0.1, not localhost
      // http://curl.haxx.se/mail/tracker-2009-09/0018.html
      if (result == 7) {
        VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
        if (max_retries != Qtrue) {
          multi_remove_handle(self, easy);
          multi_add_handle(self, easy);
          CurlMulti *curl_multi;
          Data_Get_Struct(self, CurlMulti, curl_multi);
          curl_multi_perform(curl_multi->multi, &(curl_multi->running));

          rb_funcall(easy, rb_intern("increment_retries"), 0);

          continue;
        }
      }
      multi_remove_handle(self, easy);

      if (result != 0) {
        rb_funcall(easy, rb_intern("failure"), 0);
      }
      else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
        rb_funcall(easy, rb_intern("success"), 0);
      }
      else if (response_code >= 300 && response_code < 600) {
        rb_funcall(easy, rb_intern("failure"), 0);
      }
    }
  }
}

/* called by multi_perform and fire_and_forget */
static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
  CURLMcode mcode;

  do {
    mcode = curl_multi_perform(multi_handle, still_running);
  } while (mcode == CURLM_CALL_MULTI_PERFORM);

  if (mcode != CURLM_OK) {
    rb_raise(rb_eRuntimeError, "an error occured while running perform: %d", mcode);
  }

  multi_read_info( self, multi_handle );
}

static VALUE fire_and_forget(VALUE self) {
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);
  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
}

static VALUE multi_perform(VALUE self) {
  CURLMcode mcode;
  CurlMulti *curl_multi;
  int maxfd, rc;
  fd_set fdread, fdwrite, fdexcep;

  long timeout;
  struct timeval tv = {0, 0};

  Data_Get_Struct(self, CurlMulti, curl_multi);

  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
  while(curl_multi->running) {
    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* get the curl suggested time out */
    mcode = curl_multi_timeout(curl_multi->multi, &timeout);
    if (mcode != CURLM_OK) {
      rb_raise(rb_eRuntimeError, "an error occured getting the timeout: %d", mcode);
          }

    if (timeout == 0) { /* no delay */
      rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
      continue;<commit_msg>need to return a value<commit_after>#include <typhoeus_multi.h>

static void multi_read_info(VALUE self, CURLM *multi_handle);

static void dealloc(CurlMulti *curl_multi) {
  curl_multi_cleanup(curl_multi->multi);
  free(curl_multi);
}

static VALUE multi_add_handle(VALUE self, VALUE easy) {
  CurlEasy *curl_easy;
  Data_Get_Struct(easy, CurlEasy, curl_easy);
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);
  CURLMcode mcode;

  mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    rb_raise(rb_eRuntimeError, "An error occured adding the handle: %d", mcode);
  }

  curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
  curl_multi->active++;

  if (mcode == CURLM_CALL_MULTI_PERFORM) {
    curl_multi_perform(curl_multi->multi, &(curl_multi->running));
  }
  // 
  // if (curl_multi->running) {
  //     printf("call read_info on add<br/>");
  //   multi_read_info(self, curl_multi->multi);
  // }

  return easy;
}

static VALUE multi_remove_handle(VALUE self, VALUE easy) {
  CurlEasy *curl_easy;
  Data_Get_Struct(easy, CurlEasy, curl_easy);
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);

  curl_multi->active--;
  curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);

  return easy;
}

static void multi_read_info(VALUE self, CURLM *multi_handle) {
  int msgs_left, result;
  CURLMsg *msg;
  CURLcode ecode;
  CURL *easy_handle;
  VALUE easy;

  /* check for finished easy handles and remove from the multi handle */
  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {

    if (msg->msg != CURLMSG_DONE) {
      continue;
    }

    easy_handle = msg->easy_handle;
    result = msg->data.result;
    if (easy_handle) {
      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
      if (ecode != 0) {
        rb_raise(rb_eRuntimeError, "error getting easy object:%d", ecode);
      }

      long response_code = -1;
      curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);

      // TODO: find out what the real problem is here and fix it.
      // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
      // fail intermittently and return this result number. However, it will succeed if you try it a few
      // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!

      // Sandofsky says:
      // This is caused by OS X first attempting to resolve using IPV6.
      // Hack solution: connect to yourself with 127.0.0.1, not localhost
      // http://curl.haxx.se/mail/tracker-2009-09/0018.html
      if (result == 7) {
        VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
        if (max_retries != Qtrue) {
          multi_remove_handle(self, easy);
          multi_add_handle(self, easy);
          CurlMulti *curl_multi;
          Data_Get_Struct(self, CurlMulti, curl_multi);
          curl_multi_perform(curl_multi->multi, &(curl_multi->running));

          rb_funcall(easy, rb_intern("increment_retries"), 0);

          continue;
        }
      }
      multi_remove_handle(self, easy);

      if (result != 0) {
        rb_funcall(easy, rb_intern("failure"), 0);
      }
      else if ((response_code >= 200 && response_code < 300) || response_code == 0) {
        rb_funcall(easy, rb_intern("success"), 0);
      }
      else if (response_code >= 300 && response_code < 600) {
        rb_funcall(easy, rb_intern("failure"), 0);
      }
    }
  }
}

/* called by multi_perform and fire_and_forget */
static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
  CURLMcode mcode;

  do {
    mcode = curl_multi_perform(multi_handle, still_running);
  } while (mcode == CURLM_CALL_MULTI_PERFORM);

  if (mcode != CURLM_OK) {
    rb_raise(rb_eRuntimeError, "an error occured while running perform: %d", mcode);
  }

  multi_read_info( self, multi_handle );
}

static VALUE fire_and_forget(VALUE self) {
  CurlMulti *curl_multi;
  Data_Get_Struct(self, CurlMulti, curl_multi);
  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );

  return Qnil;
}

static VALUE multi_perform(VALUE self) {
  CURLMcode mcode;
  CurlMulti *curl_multi;
  int maxfd, rc;
  fd_set fdread, fdwrite, fdexcep;

  long timeout;
  struct timeval tv = {0, 0};

  Data_Get_Struct(self, CurlMulti, curl_multi);

  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
  while(curl_multi->running) {
    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* get the curl suggested time out */
    mcode = curl_multi_timeout(curl_multi->multi, &timeout);
    if (mcode != CURLM_OK) {
      rb_raise(rb_eRuntimeError, "an error occured getting the timeout: %d", mcode);
          }

    if (timeout == 0) { /* no delay */
      rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
      continue;
{'fraction_non_alphanumeric': 0.11658786842369982, 'fraction_numerical': 0.008449854139422593, 'mean_word_length': 3.5273224043715845, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4125283', 'n_tokens_mistral': 3666, 'n_tokens_neox': 3395, 'n_words': 815}