<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <string>
#include <sstream>
#include <assert.h>
#include <iostream>

#include <zorba/base64.h>
#include <zorba/config.h>
#include <zorba/diagnostic_list.h>
#include <zorba/error.h>
#include <zorba/item.h>
#include <zorba/item_factory.h>
#include <zorba/transcode_stream.h>
#include <zorba/xmldatamanager.h>
#include <zorba/xquery_exception.h>
#include <zorba/xquery_exception.h>
#include <zorba/xquery_functions.h>

#include "http_response_parser.h"
#include "http_request_handler.h"
#include "curl_stream_buffer.h"

namespace zorba {

static void parse_content_type( std::string const &s, std::string *mime_type,
                                std::string *charset ) {
  std::string::size_type pos = s.find( ';' );
  *mime_type = s.substr( 0, pos );

  // The HTTP/1.1 spec says that the default charset is ISO-8859-1.
  *charset = "ISO-8859-1";

  if ( pos != std::string::npos ) {
    //
    // Parse: charset="?XXXXX"?[ (comment)]
    //
    if ( (pos = s.find( '=' )) != std::string::npos ) {
      std::string t = s.substr( pos + 1 );
      if ( !t.empty() ) {
        if ( t[0] == '"' ) {
          t.erase( 0, 1 );
          if ( (pos = t.find( '"' )) != std::string::npos )
            t.erase( pos );
        } else {
          if ( (pos = t.find( ' ' )) != std::string::npos )
            t.erase( pos );
        }
        *charset = t;
      } 
    }
  }
}

namespace http_client {
  
  HttpResponseParser::HttpResponseParser(RequestHandler& aHandler, CURL* aCurl,
                                         ErrorThrower& aErrorThrower,
                                         std::string aOverridenContentType,
                                         bool aStatusOnly) : 
  theHandler(aHandler),
  theCurl(aCurl),
  theErrorThrower(aErrorThrower),
  theCurrentCharset("ISO-8859-1"),      // HTTP/1.1 says this is the default
  theStatus(-1),
  theStreamBuffer(0),
  theInsideRead(false),
  theOverridenContentType(aOverridenContentType),
  theStatusOnly(aStatusOnly),
  theSelfContained(true)
  {
    registerHandler();
    theStreamBuffer = new zorba::curl::streambuf(theCurl);
  }

  HttpResponseParser::~HttpResponseParser()
  {
    delete theStreamBuffer;
  }

  int HttpResponseParser::parse()
  {
    theStreamBuffer->setInformer(this);
    theHandler.begin();
    int lCode = 0;
    lCode = theStreamBuffer->multi_perform();
    if (lCode)
      return lCode; 
    if (!theStatusOnly) {

      if (!theOverridenContentType.empty()) {
        parse_content_type(
          theOverridenContentType, &theCurrentContentType, &theCurrentCharset
        );
      }

      std::auto_ptr<std::istream> lStream;
      try {
        if ( transcode::is_necessary( theCurrentCharset.c_str() ) ) {
          lStream.reset(
            new transcode::stream<std::istream>(
              theCurrentCharset.c_str(), theStreamBuffer<commit_msg>No longer setting the charset of an overridden media type to a default value.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <string>
#include <sstream>
#include <assert.h>
#include <iostream>

#include <zorba/base64.h>
#include <zorba/config.h>
#include <zorba/diagnostic_list.h>
#include <zorba/error.h>
#include <zorba/item.h>
#include <zorba/item_factory.h>
#include <zorba/transcode_stream.h>
#include <zorba/xmldatamanager.h>
#include <zorba/xquery_exception.h>
#include <zorba/xquery_exception.h>
#include <zorba/xquery_functions.h>

#include "http_response_parser.h"
#include "http_request_handler.h"
#include "curl_stream_buffer.h"

namespace zorba {

static void parse_content_type( std::string const &s, std::string *mime_type,
                                std::string *charset ) {
  std::string::size_type pos = s.find( ';' );
  *mime_type = s.substr( 0, pos );

  charset->clear();

  if ( pos != std::string::npos ) {
    //
    // Parse: charset="?XXXXX"?[ (comment)]
    //
    if ( (pos = s.find( '=' )) != std::string::npos ) {
      std::string t = s.substr( pos + 1 );
      if ( !t.empty() ) {
        if ( t[0] == '"' ) {
          t.erase( 0, 1 );
          if ( (pos = t.find( '"' )) != std::string::npos )
            t.erase( pos );
        } else {
          if ( (pos = t.find( ' ' )) != std::string::npos )
            t.erase( pos );
        }
        *charset = t;
      } 
    }
  }
}

namespace http_client {
  
  HttpResponseParser::HttpResponseParser(RequestHandler& aHandler, CURL* aCurl,
                                         ErrorThrower& aErrorThrower,
                                         std::string aOverridenContentType,
                                         bool aStatusOnly) : 
  theHandler(aHandler),
  theCurl(aCurl),
  theErrorThrower(aErrorThrower),
  theCurrentCharset("ISO-8859-1"),      // HTTP/1.1 says this is the default
  theStatus(-1),
  theStreamBuffer(0),
  theInsideRead(false),
  theOverridenContentType(aOverridenContentType),
  theStatusOnly(aStatusOnly),
  theSelfContained(true)
  {
    registerHandler();
    theStreamBuffer = new zorba::curl::streambuf(theCurl);
  }

  HttpResponseParser::~HttpResponseParser()
  {
    delete theStreamBuffer;
  }

  int HttpResponseParser::parse()
  {
    theStreamBuffer->setInformer(this);
    theHandler.begin();
    int lCode = 0;
    lCode = theStreamBuffer->multi_perform();
    if (lCode)
      return lCode; 
    if (!theStatusOnly) {

      if (!theOverridenContentType.empty()) {
        parse_content_type(
          theOverridenContentType, &theCurrentContentType, &theCurrentCharset
        );
      }

      std::auto_ptr<std::istream> lStream;
      try {
        if ( !theCurrentCharset.empty() &&
             transcode::is_necessary( theCurrentCharset.c_str() ) ) {
          lStream.reset(
            new transcode::stream<std::istream>(
              theCurrentCharset.c_str(), theStreamBuffer
{'fraction_non_alphanumeric': 0.127537889619674, 'fraction_numerical': 0.010008578781812983, 'mean_word_length': 2.7446466809421843, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6990745', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2209, 'n_words': 512}