<commit_before>/*
 * htFollow - follow an http url until content is reached - implementation
 * Copyright(c) 2005 of wave++ (<NAME>)
 * Distributed under GNU LGPL without ANY warranty.
 */

// interface and local headers
#include "htfollow.hh"
#include "hdrparse.hh"
#include "sanitize.hh"
#include "msg.hh"
using std::string;
using std::map;

// system headers
#include <memory>
using std::auto_ptr;

#include <stdexcept>
using std::runtime_error;


// implementation
string
itos(const int i)
{
  char buf[16];
  snprintf(buf, sizeof(buf), "%d", i);
  return string(buf);
}


Socket*
htFollow(map<string, string>& pReply, const URL& url,
    const Http::Header qHeaders, size_t limit)
{
  URL buf = url;

  // connection loop
  auto_ptr<Socket> s;
  for(;; --limit)
  {
    msg("connecting to (%s %d)", buf.server.c_str(), buf.port);
    Http::Http httpc(buf.server.c_str(), buf.port);

    msg("requesting data on (%s)", buf.path.c_str());
    Http::Header aHeaders;
    Http::Reply reply(&aHeaders);
    s.reset(httpc.get(buf.path.c_str(), reply, &qHeaders));

    // validate the reply code
    if(reply.code != Http::Proto::ok &&
	reply.code != Http::Proto::found &&
	reply.code != Http::Proto::other)
      throw runtime_error(string("unexpected reply: ") +
	  itos(reply.code) + " " + sanitize_esc((reply.description.size()?
		  reply.description: reply.proto)).c_str());

    // parse the headers
    pReply = Http::hdrParse(aHeaders);
    if(reply.code == Http::Proto::ok)
      break;

    // recursion
    if(!limit)
      throw runtime_error(string("hit redirect follow limit: ") + itos(limit));

    map<string, string>::iterator urlPos = pReply.find(Http::Proto::location);
    if(urlPos == pReply.end())
      throw runtime_error("redirection didn't contain an url");<commit_msg>Sanitize the urls as redirection is unthrusted.<commit_after>/*
 * htFollow - follow an http url until content is reached - implementation
 * Copyright(c) 2005 of wave++ (<NAME>)
 * Distributed under GNU LGPL without ANY warranty.
 */

// interface and local headers
#include "htfollow.hh"
#include "hdrparse.hh"
#include "sanitize.hh"
#include "msg.hh"
using std::string;
using std::map;

// system headers
#include <memory>
using std::auto_ptr;

#include <stdexcept>
using std::runtime_error;


// implementation
string
itos(const int i)
{
  char buf[16];
  snprintf(buf, sizeof(buf), "%d", i);
  return string(buf);
}


Socket*
htFollow(map<string, string>& pReply, const URL& url,
    const Http::Header qHeaders, size_t limit)
{
  URL buf = url;

  // connection loop
  auto_ptr<Socket> s;
  for(;; --limit)
  {
    msg("connecting to (%s %d)", sanitize_esc(buf.server).c_str(), buf.port);
    Http::Http httpc(sanitize_esc(buf.server).c_str(), buf.port);

    msg("requesting data on (%s)", sanitize_esc(buf.path).c_str());
    Http::Header aHeaders;
    Http::Reply reply(&aHeaders);
    s.reset(httpc.get(buf.path.c_str(), reply, &qHeaders));

    // validate the reply code
    if(reply.code != Http::Proto::ok &&
	reply.code != Http::Proto::found &&
	reply.code != Http::Proto::other)
      throw runtime_error(string("unexpected reply: ") +
	  itos(reply.code) + " " + sanitize_esc((reply.description.size()?
		  reply.description: reply.proto)).c_str());

    // parse the headers
    pReply = Http::hdrParse(aHeaders);
    if(reply.code == Http::Proto::ok)
      break;

    // recursion
    if(!limit)
      throw runtime_error(string("hit redirect follow limit: ") + itos(limit));

    map<string, string>::iterator urlPos = pReply.find(Http::Proto::location);
    if(urlPos == pReply.end())
      throw runtime_error("redirection didn't contain an url");
{'fraction_non_alphanumeric': 0.16239082969432314, 'fraction_numerical': 0.0032751091703056767, 'mean_word_length': 4.006830601092896, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5149839', 'n_tokens_mistral': 1232, 'n_tokens_neox': 1193, 'n_words': 303}