<commit_before>// @(#)root/net:$Id$
// Author: <NAME>   17/01/97

/*************************************************************************
 * Copyright (C) 1995-2000, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TWebFile                                                             //
//                                                                      //
// A TWebFile is like a normal TFile except that it reads its data      //
// via a standard apache web server. A TWebFile is a read-only file.    //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TWebFile.h"
#include "TROOT.h"
#include "TSocket.h"
#include "Bytes.h"
#include "TError.h"
#include "TSystem.h"
#include "TBase64.h"
#include "TVirtualPerfStats.h"

#include <errno.h>
#include <stdlib.h>

#ifdef WIN32
#define EADDRINUSE  10048
#define EISCONN     10056
#endif

static const char *gUserAgent = "User-Agent: ROOT-TWebFile/1.1";

TUrl TWebFile::fgProxy;


// Internal class used to manage the socket that may stay open between
// calls when HTTP/1.1 protocol is used
class TWebSocket {
private:
   TWebFile *fWebFile;           // associated web file
public:
   TWebSocket(TWebFile *f);
   ~TWebSocket();
   void ReOpen();
};

//______________________________________________________________________________
TWebSocket::TWebSocket(TWebFile *f)
{
   // Open web file socket.

   fWebFile = f;
   if (!f->fSocket)
      ReOpen();
}

//______________________________________________________________________________
TWebSocket::~TWebSocket()
{
   // Close socket in case not HTTP/1.1 protocol or when explicitly requested.

   if (!fWebFile->fHTTP11) {
      delete fWebFile->fSocket;
      fWebFile->fSocket = 0;
   }
}

//______________________________________________________________________________
void TWebSocket::ReOpen()
{
   // Re-open web file socket.

   if (fWebFile->fSocket)
      delete fWebFile->fSocket;

   TUrl connurl;
   if (fWebFile->fProxy.IsValid())
      connurl = fWebFile->fProxy;
   else
      connurl = fWebFile->fUrl;

   for (Int_t i = 0; i < 5; i++) {
      fWebFile->fSocket = new TSocket(connurl.GetHost(), connurl.GetPort());
      if (!fWebFile->fSocket->IsValid()) {
         delete fWebFile->fSocket;
         fWebFile->fSocket = 0;
         if (gSystem->GetErrno() == EADDRINUSE || gSystem->GetErrno() == EISCONN) {
            gSystem->Sleep(i*10);
         } else {
            ::Error("TWebSocket::ReOpen", "cannot connect to host %s (errno=%d)",
                    fWebFile->fUrl.GetHost(), gSystem->GetErrno());
            return;
         }
      } else
         return;
   }
}


ClassImp(TWebFile)

//______________________________________________________________________________
TWebFile::TWebFile(const char *url, Option_t *opt) : TFile(url, "WEB")
{
   // Create a Web file object. A web file is the same as a read-only
   // TFile except that it is being read via a HTTP server. The url
   // argument must be of the form: http://host.dom.ain/file.root.
   // The opt can be "NOPROXY", to bypass any set "http_proxy" shell
   // variable. The proxy can be specified as (in sh, or equivalent csh):
   //   export http_proxy=http://pcsalo.cern.ch:3128
   // The proxy can also be specified via the static method TWebFile::SetProxy().
   // Basic authentication (AuthType Basic) is supported. The user name and
   // passwd can be specified in the url like this:
   //   http://username:mypasswd@pcsalo.cern.ch/files/aap.root
   // If the file specified in the URL does not exist or is not accessible
   // the kZombie bit will be set in the TWebFile object. Use IsZombie()
   // to see if the file is accessible. The preferred interface to this
   // constructor is via TFile::Open().

   TString option = opt;
   fNoProxy = kFALSE;
   if (option.Contains("NOPROXY", TString::kIgnoreCase))
      fNoProxy = kTRUE;
   CheckProxy();

   Bool_t headOnly = kFALSE;
   if (option.Contains("HEADONLY", TString::kIgnoreCase))
      headOnly = kTRUE;

   Init(headOnly);
}

//______________________________________________________________________________
TWebFile::TWebFile(TUrl url, Option_t *opt) : TFile(url.GetUrl(), "WEB")
{
   // Create a Web file object. A web file is the same as a read-only
   // TFile except that it is being read via a HTTP server. Make sure url
   // is a valid TUrl object.
   // The opt can be "NOPROXY", to bypass any set "http_proxy" shell
   // variable. The proxy can be specified as (in sh, or equivalent csh):
   //   export http_proxy=http://pcsalo.cern.ch:3128
   // The proxy can also be specified via the static method TWebFile::SetProxy().
   // Basic authentication (AuthType Basic) is supported. The user name and
   // passwd can be specified in the url like this:
   //   http://username:mypasswd@pcsalo.cern.ch/files/aap.root
   // If the file specified in the URL does not exist or is not accessible
   // the kZombie bit will be set in the TWebFile object. Use IsZombie()
   // to see if the file is accessible.

   TString option = opt;
   fNoProxy = kFALSE;
   if (option.Contains("NOPROXY", TString::kIgnoreCase))
      fNoProxy = kTRUE;
   CheckProxy();

   Bool_t headOnly = kFALSE;
   if (option.Contains("HEADONLY", TString::kIgnoreCase))
      headOnly = kTRUE;

   Init(headOnly);
}

//______________________________________________________________________________
TWebFile::~TWebFile()
{
   // Cleanup.

   delete fSocket;
}

//______________________________________________________________________________
void TWebFile::Init(Bool_t readHeadOnly)
{
   // Initialize a TWebFile object.

   char buf[4];
   int  err;

   fSocket     = 0;
   fSize       = -1;
   fHasModRoot = kFALSE;
   fHTTP11     = kFALSE;

   SetMsgReadBuffer10();

   if ((err = GetHead()) < 0) {
      if (readHeadOnly) {
         fD = -1;
         fWritten = err;
         return;
      }
      if (err == -2) {
         Error("TWebFile", "%s does not exist", fBasicUrl.Data());
         MakeZombie();
         gDirectory = gROOT;
         return;
      }
      // err == -3 HEAD not supported, fall through and try ReadBuffer()
   }
   if (readHeadOnly) {
      fD = -1;
      return;
   }

   if (fIsRootFile) {
      Seek(0);
      if (ReadBuffer(buf, 4)) {
         MakeZombie();
         gDirectory = gROOT;
         return;
      }

      if (strncmp(buf, "root", 4) && strncmp(buf, "PK", 2)) {  // PK is zip file
         Error("TWebFile", "%s is not a ROOT file", fBasicUrl.Data());
         MakeZombie();
         gDirectory = gROOT;
         return;
      }
   }

   TFile::Init(kFALSE);
   fD = -2;   // so TFile::IsOpen() will return true when in TFile::~TFile
}

//______________________________________________________________________________
void TWebFile::SetMsgReadBuffer10(const char *redirectLocation, Bool_t tempRedirect)
{
   // Set GET command for use by ReadBuffer(s)10(), handle redirection if
   // needed. Give full URL so Apache's virtual hosts solution works.

   TUrl oldUrl;
   TString oldBasicUrl;

   if (redirectLocation) {
      if (tempRedirect) { // temp redirect
         fUrlOrg      = fUrl;
         fBasicUrlOrg = fBasicUrl;
      } else {             // permanent redirect
         fUrlOrg      = "";
         fBasicUrlOrg = "";
      }

      oldUrl = fUrl;
      oldBasicUrl = fBasicUrl;

      fUrl.SetUrl(redirectLocation);
      fBasicUrl = fUrl.GetProtocol();
      fBasicUrl += "://";
      fBasicUrl += fUrl.GetHost();
      fBasicUrl += ":";
      fBasicUrl += fUrl.GetPort();
      fBasicUrl += "/";
      fBasicUrl += fUrl.GetFile();
   }

   if (fMsgReadBuffer10 != "") {
      // patch up existing command
      if (oldBasicUrl != "") {
         // change to redirection location
         fMsgReadBuffer10.ReplaceAll(oldBasicUrl, fBasicUrl);
         fMsgReadBuffer10.ReplaceAll(TString("Host: ")+oldUrl.GetHost(), TString("Host: ")+fUrl.GetHost());
      } else if (fBasicUrlOrg != "") {
         // change back from temp redirection location
         fMsgReadBuffer10.ReplaceAll(fBasicUrl, fBasicUrlOrg);
         fMsgReadBuffer10.ReplaceAll(TString("Host: ")+fUrl.GetHost(), TString("Host: ")+fUrlOrg.GetHost());
         fUrl         = fUrlOrg;
         fBasicUrl    = fBasicUrlOrg;
         fUrlOrg      = "";
         fBasicUrlOrg = "";
      }
   }

   if (fBasicUrl == "") {
      fBasicUrl += fUrl.GetProtocol();
      fBasicUrl += "://";
      fBasicUrl += fUrl.GetHost();
      fBasicUrl += ":";
      fBasicUrl += fUrl.GetPort();
      fBasicUrl += "/";
      fBasicUrl += fUrl.GetFile();
   }

   if (fMsgReadBuffer10 == "") {
      fMsgReadBuffer10 = "GET ";
      fMsgReadBuffer10 += fBasicUrl;
      if (fHTTP11)
         fMsgReadBuffer10 += " HTTP/1.1";
      else
         fMsgReadBuffer10 += " HTTP/1.0";
      fMsgReadBuffer10 += "\r\n";
      if (fHTTP11) {
         fMsgReadBuffer10 += "Host: ";
         fMsgReadBuffer10 += fUrl.GetHost();
         fMsgReadBuffer10 += "\r\n";
      }
      fMsgReadBuffer10 += BasicAuthentication();
      fMsgReadBuffer10 += gUserAgent;
      fMsgReadBuffer10 += "\r\n";
      fMsgReadBuffer10 += "Range: bytes=";
   }
}

//______________________________________________________________________________
void TWebFile::CheckProxy()
{
   // Check if shell var "http_proxy" has been set and should be used.

   if (fNoProxy)
      return;

   if (fgProxy.IsValid()) {
      fProxy = fgProxy;
      return;
   }

   TString proxy = gSystem->Getenv("http_proxy");
   if (proxy != "") {
      TUrl p(proxy);
      if (strcmp(p.GetProtocol(), "http")) {
         Error("CheckProxy", "protocol must be HTTP in proxy URL %s",
               proxy.Data());
         return;
      }
      fProxy = p;
   }
}

//______________________________________________________________________________
Bool_t TWebFile::IsOpen() const
{
   // A TWebFile that has been correctly constructed is always considered open.

   return IsZombie() ? kFALSE : kTRUE;
}

//______________________________________________________________________________
Int_t TWebFile::ReOpen(Option_t *mode)
{
   // Reopen a file with a different access mode, like from READ to
   // UPDATE or from NEW, CREATE, RECREATE, UPDATE to READ. Thus the
   // mode argument can be either "READ" or "UPDATE". The method returns
   // 0 in case the mode was successfully modified, 1 in case the mode
   // did not change (was already as requested or wrong input arguments)
   // and -1 in case of failure, in which case the file cannot be used
   // anymore. A TWebFile cannot be reopened in update mode.

   TString opt = mode;
   opt.ToUpper();

   if (opt != "READ" && opt != "UPDATE")
      Error("ReOpen", "mode must be either READ or UPDATE, not %s", opt.Data());

   if (opt == "UPDATE")
      Error("ReOpen", "update mode not allowed for a TWebFile");

   return 1;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer(char *buf, Int_t len)
{
   // Read specified byte range from remote file via HTTP daemon. This
   // routine connects to the remote host, sends the request and returns
   // the buffer. Returns kTRUE in case of error.

   Int_t st;
   if ((st = ReadBufferViaCache(buf, len))) {
      if (st == 2)
         return kTRUE;
      return kFALSE;
   }

   if (!fHasModRoot)
      return ReadBuffer10(buf, len);

   // Give full URL so Apache's virtual hosts solution works.
   // Use protocol 0.9 for efficiency, we are not interested in the 1.0 headers.
   if (fMsgReadBuffer == "") {
      fMsgReadBuffer = "GET ";
      fMsgReadBuffer += fBasicUrl;
      fMsgReadBuffer += "?";
   }
   TString msg = fMsgReadBuffer;
   msg += fOffset;
   msg += ":";
   msg += len;
   msg += "\r\n";

   if (GetFromWeb(buf, len, msg) == -1)
      return kTRUE;

   fOffset += len;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer(char *buf, Long64_t pos, Int_t len)
{
   // Read specified byte range from remote file via HTTP daemon. This
   // routine connects to the remote host, sends the request and returns
   // the buffer. Returns kTRUE in case of error.

   SetOffset(pos);
   return ReadBuffer(buf, len);
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer10(char *buf, Int_t len)
{
   // Read specified byte range from remote file via HTTP 1.0 daemon (without
   // mod-root installed). This routine connects to the remote host, sends the
   // request and returns the buffer. Returns kTRUE in case of error.

   SetMsgReadBuffer10();

   TString msg = fMsgReadBuffer10;
   msg += fOffset;
   msg += "-";
   msg += fOffset+len-1;
   msg += "\r\n\r\n";

   Int_t n = GetFromWeb10(buf, len, msg);
   if (n == -1)
      return kTRUE;
   // The -2 error condition typically only happens when
   // GetHead() failed because not implemented, in the first call to
   // ReadBuffer() in Init(), it is not checked in ReadBuffers10().
   if (n == -2) {
      Error("ReadBuffer10", "%s does not exist", fBasicUrl.Data());
      MakeZombie();
      gDirectory = gROOT;
      return kTRUE;
   }

   fOffset += len;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
{
   // Read specified byte ranges from remote file via HTTP daemon.
   // Reads the nbuf blocks described in arrays pos and len,
   // where pos[i] is the seek position of block i of length len[i].
   // Note that for nbuf=1, this call is equivalent to TFile::ReafBuffer
   // This function is overloaded by TNetFile, TWebFile, etc.
   // Returns kTRUE in case of failure.

   if (!fHasModRoot)
      return ReadBuffers10(buf, pos, len, nbuf);

   // Give full URL so Apache's virtual hosts solution works.
   // Use protocol 0.9 for efficiency, we are not interested in the 1.0 headers.
   if (fMsgReadBuffer == "") {
      fMsgReadBuffer = "GET ";
      fMsgReadBuffer += fBasicUrl;
      fMsgReadBuffer += "?";
   }
   TString msg = fMsgReadBuffer;

   Int_t k = 0, n = 0;
   for (Int_t i = 0; i < nbuf; i++) {
      if (n) msg += ",";
      msg += pos[i] + fArchiveOffset;
      msg += ":";
      msg += len[i];
      n   += len[i];
      if (msg.Length() > 8000) {
         msg += "\r\n";
         if (GetFromWeb(&buf[k], n, msg) == -1)
            return kTRUE;
         msg = fMsgReadBuffer;
         k += n;
         n = 0;
      }
   }

   msg += "\r\n";

   if (GetFromWeb(&buf[k], n, msg) == -1)
      return kTRUE;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffers10(char *buf,  Long64_t *pos, Int_t *len, Int_t nbuf)
{
   // Read specified byte ranges from remote file via HTTP 1.0 daemon (without
   // mod-root installed). Read the nbuf blocks described in arrays pos and len,
   // where pos[i] is the seek position of block i of length len[i].
   // Note that for nbuf=1, this call is equivalent to TFile::ReafBuffer
   // This function is overloaded by TNetFile, TWebFile, etc.
   // Returns kTRUE in case of failure.

   SetMsgReadBuffer10();

   TString msg = fMsgReadBuffer10;

   Int_t k = 0, n = 0, r;
   for (Int_t i = 0; i < nbuf; i++) {
      if (n) msg += ",";
      msg += pos[i] + fArchiveOffset;
      msg += "-";
      msg += pos[i] + fArchiveOffset + len[i] - 1;
      n   += len[i];
      if (msg.Length() > 8000) {
         msg += "\r\n\r\n";
         r = GetFromWeb10(&buf[k], n, msg);
         if (r == -1)
            return kTRUE;
         msg = fMsgReadBuffer10;
         k += n;
         n = 0;
      }
   }

   msg += "\r\n\r\n";

   r = GetFromWeb10(&buf[k], n, msg);
   if (r == -1)
      return kTRUE;

   return kFALSE;
}

//______________________________________________________________________________
Int_t TWebFile::GetFromWeb(char *buf, Int_t len, const TString &msg)
{
   // Read request from web server. Returns -1 in case of error,
   // 0 in case of success.

   if (!len) return 0;

   Double_t start = 0;
   if (gPerfStats) start = TTimeStamp();

   TUrl connurl;
   if (fProxy.IsValid())
      connurl = fProxy;
   else
      connurl = fUrl;

   TSocket s(connurl.GetHost(), connurl.GetPort());
   if (!s.IsValid()) {
      Error("GetFromWeb", "cannot connect to host %s", fUrl.GetHost());
      return -1;
   }

   if (s.SendRaw(msg.Data(), msg.Length()) == -1) {
      Error("GetFromWeb", "error sending command to host %s", fUrl.GetHost());
      return -1;
   }

   if (s.RecvRaw(buf, len) == -1) {
      Error("GetFromWeb", "error receiving data from host %s", fUrl.GetHost());
      return -1;
   }

   // collect statistics
   fBytesRead += len;
   fReadCalls++;
#ifdef R__WIN32
   SetFileBytesRead(GetFileBytesRead() + len);
   SetFileReadCalls(GetFileReadCalls() + 1);
#else
   fgBytesRead += len;
   fgReadCalls++;
#endif

   if (gPerfStats)
      gPerfStats->FileReadEvent(this, len, start);

   return 0;
}

//______________________________________________________________________________
Int_t TWebFile::GetFromWeb10(char *buf, Int_t len, const TString &msg)
{
   // Read multiple byte range request from web server.
   // Uses HTTP 1.0 daemon wihtout mod-root.
   // Returns -2 in case file does not exist, -1 in case
   // of error and 0 in case of success.

   if (!len) return 0;

   Double_t start = 0;
   if (gPerfStats) start = TTimeStamp();

   // open fSocket and close it when going out of scope
   TWebSocket ws(this);

   if (!fSocket || !fSocket->IsValid()) {
      Error("GetFromWeb10", "cannot connect to host %s", fUrl.GetHost());
      return -1;
   }

   if (fSocket->SendRaw(msg.Data(), msg.Length()) == -1) {
      Error("GetFromWeb10", "error sending command to host %s", fUrl.GetHost());
      return -1;
   }

   char line[8192];
   Int_t n, ret = 0, nranges = 0, ltot = 0, redirect = 0;
   TString boundary, boundaryEnd;
   Long64_t first = -1, last = -1, tot;

   while ((n = GetLine(fSocket, line, sizeof(line))) >= 0) {
      if (n == 0) {
         if (ret < 0)
            return ret;
         if (redirect) {
            ws.ReOpen();
            return GetFromWeb10(buf, len, msg);
         }

         if (first >= 0) {
            Int_t ll = Int_t(last - first) + 1;
            if (fSocket->RecvRaw(&buf[ltot], ll) == -1) {
               Error("GetFromWeb10", "error receiving data from host %s", fUrl.GetHost());
               return -1;
            }
            ltot += ll;

            first = -1;

            if (boundary == "")
               break;  // not a multipart response
         }

         continue;
      }

      if (gDebug > 0)
         Info("GetFromWeb10", "header: %s", line);

      if (boundaryEnd == line) {
         if (gDebug > 0)
            Info("GetFromWeb10", "got all headers");
         break;
      }
      if (boundary == line) {
         nranges++;
         if (gDebug > 0)
            Info("GetFromWeb10", "get new multipart byte range (%d)", nranges);
      }

      TString res = line;

      if (res.BeginsWith("HTTP/1.")) {
         if (res.BeginsWith("HTTP/1.1")) {
            if (!fHTTP11)
               fMsgReadBuffer10  = "";
            fHTTP11 = kTRUE;
         }
         TString scode = res(9, 3);
         Int_t code = scode.Atoi();
         if (code >= 500) {
            ret = -1;
            TString mess = res(13, 1000);
            Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
         } else if (code >= 400) {
            if (code == 404)
               ret = -2;   // file does not exist
            else {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         } else if (code >= 300) {
            if (code == 301 || code == 303)
               redirect = 1;   // permanent redirect
            else if (code == 302 || code == 307)
               redirect = 2;   // temp redirect
            else {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         } else if (code > 200) {
            if (code != 206) {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         }
      } else if (res.BeginsWith("Content-Type: multipart")) {
         boundary = "--" + res(res.Index("boundary=")+9, 1000);
         boundaryEnd = boundary + "--";
      } else if (res.BeginsWith("Content-range:")) {
#ifdef R__WIN32<commit_msg>When running on Windows (on possibly on other platforms), the boundary of the multipart web message is surrounded by quotes. So instead of:    Content-Type: multipart/byteranges; boundary=chgs4uxnRx1Nn=x:Rl=E we see:    Content-Type: multipart/byteranges; boundary="chgs4uxnRx1Nn=x:Rl=E" However the calculation of the string to compare the boundary line to was incorrect (contained the spurrious quotes) and was leading to an infinite recursion/infinite loop.<commit_after>// @(#)root/net:$Id$
// Author: <NAME>   17/01/97

/*************************************************************************
 * Copyright (C) 1995-2000, <NAME> and <NAME>.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TWebFile                                                             //
//                                                                      //
// A TWebFile is like a normal TFile except that it reads its data      //
// via a standard apache web server. A TWebFile is a read-only file.    //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TWebFile.h"
#include "TROOT.h"
#include "TSocket.h"
#include "Bytes.h"
#include "TError.h"
#include "TSystem.h"
#include "TBase64.h"
#include "TVirtualPerfStats.h"

#include <errno.h>
#include <stdlib.h>

#ifdef WIN32
#define EADDRINUSE  10048
#define EISCONN     10056
#endif

static const char *gUserAgent = "User-Agent: ROOT-TWebFile/1.1";

TUrl TWebFile::fgProxy;


// Internal class used to manage the socket that may stay open between
// calls when HTTP/1.1 protocol is used
class TWebSocket {
private:
   TWebFile *fWebFile;           // associated web file
public:
   TWebSocket(TWebFile *f);
   ~TWebSocket();
   void ReOpen();
};

//______________________________________________________________________________
TWebSocket::TWebSocket(TWebFile *f)
{
   // Open web file socket.

   fWebFile = f;
   if (!f->fSocket)
      ReOpen();
}

//______________________________________________________________________________
TWebSocket::~TWebSocket()
{
   // Close socket in case not HTTP/1.1 protocol or when explicitly requested.

   if (!fWebFile->fHTTP11) {
      delete fWebFile->fSocket;
      fWebFile->fSocket = 0;
   }
}

//______________________________________________________________________________
void TWebSocket::ReOpen()
{
   // Re-open web file socket.

   if (fWebFile->fSocket)
      delete fWebFile->fSocket;

   TUrl connurl;
   if (fWebFile->fProxy.IsValid())
      connurl = fWebFile->fProxy;
   else
      connurl = fWebFile->fUrl;

   for (Int_t i = 0; i < 5; i++) {
      fWebFile->fSocket = new TSocket(connurl.GetHost(), connurl.GetPort());
      if (!fWebFile->fSocket->IsValid()) {
         delete fWebFile->fSocket;
         fWebFile->fSocket = 0;
         if (gSystem->GetErrno() == EADDRINUSE || gSystem->GetErrno() == EISCONN) {
            gSystem->Sleep(i*10);
         } else {
            ::Error("TWebSocket::ReOpen", "cannot connect to host %s (errno=%d)",
                    fWebFile->fUrl.GetHost(), gSystem->GetErrno());
            return;
         }
      } else
         return;
   }
}


ClassImp(TWebFile)

//______________________________________________________________________________
TWebFile::TWebFile(const char *url, Option_t *opt) : TFile(url, "WEB")
{
   // Create a Web file object. A web file is the same as a read-only
   // TFile except that it is being read via a HTTP server. The url
   // argument must be of the form: http://host.dom.ain/file.root.
   // The opt can be "NOPROXY", to bypass any set "http_proxy" shell
   // variable. The proxy can be specified as (in sh, or equivalent csh):
   //   export http_proxy=http://pcsalo.cern.ch:3128
   // The proxy can also be specified via the static method TWebFile::SetProxy().
   // Basic authentication (AuthType Basic) is supported. The user name and
   // passwd can be specified in the url like this:
   //   http://username:mypasswd@pcsalo.cern.ch/files/aap.root
   // If the file specified in the URL does not exist or is not accessible
   // the kZombie bit will be set in the TWebFile object. Use IsZombie()
   // to see if the file is accessible. The preferred interface to this
   // constructor is via TFile::Open().

   TString option = opt;
   fNoProxy = kFALSE;
   if (option.Contains("NOPROXY", TString::kIgnoreCase))
      fNoProxy = kTRUE;
   CheckProxy();

   Bool_t headOnly = kFALSE;
   if (option.Contains("HEADONLY", TString::kIgnoreCase))
      headOnly = kTRUE;

   Init(headOnly);
}

//______________________________________________________________________________
TWebFile::TWebFile(TUrl url, Option_t *opt) : TFile(url.GetUrl(), "WEB")
{
   // Create a Web file object. A web file is the same as a read-only
   // TFile except that it is being read via a HTTP server. Make sure url
   // is a valid TUrl object.
   // The opt can be "NOPROXY", to bypass any set "http_proxy" shell
   // variable. The proxy can be specified as (in sh, or equivalent csh):
   //   export http_proxy=http://pcsalo.cern.ch:3128
   // The proxy can also be specified via the static method TWebFile::SetProxy().
   // Basic authentication (AuthType Basic) is supported. The user name and
   // passwd can be specified in the url like this:
   //   http://username:mypasswd@pcsalo.cern.ch/files/aap.root
   // If the file specified in the URL does not exist or is not accessible
   // the kZombie bit will be set in the TWebFile object. Use IsZombie()
   // to see if the file is accessible.

   TString option = opt;
   fNoProxy = kFALSE;
   if (option.Contains("NOPROXY", TString::kIgnoreCase))
      fNoProxy = kTRUE;
   CheckProxy();

   Bool_t headOnly = kFALSE;
   if (option.Contains("HEADONLY", TString::kIgnoreCase))
      headOnly = kTRUE;

   Init(headOnly);
}

//______________________________________________________________________________
TWebFile::~TWebFile()
{
   // Cleanup.

   delete fSocket;
}

//______________________________________________________________________________
void TWebFile::Init(Bool_t readHeadOnly)
{
   // Initialize a TWebFile object.

   char buf[4];
   int  err;

   fSocket     = 0;
   fSize       = -1;
   fHasModRoot = kFALSE;
   fHTTP11     = kFALSE;

   SetMsgReadBuffer10();

   if ((err = GetHead()) < 0) {
      if (readHeadOnly) {
         fD = -1;
         fWritten = err;
         return;
      }
      if (err == -2) {
         Error("TWebFile", "%s does not exist", fBasicUrl.Data());
         MakeZombie();
         gDirectory = gROOT;
         return;
      }
      // err == -3 HEAD not supported, fall through and try ReadBuffer()
   }
   if (readHeadOnly) {
      fD = -1;
      return;
   }

   if (fIsRootFile) {
      Seek(0);
      if (ReadBuffer(buf, 4)) {
         MakeZombie();
         gDirectory = gROOT;
         return;
      }

      if (strncmp(buf, "root", 4) && strncmp(buf, "PK", 2)) {  // PK is zip file
         Error("TWebFile", "%s is not a ROOT file", fBasicUrl.Data());
         MakeZombie();
         gDirectory = gROOT;
         return;
      }
   }

   TFile::Init(kFALSE);
   fD = -2;   // so TFile::IsOpen() will return true when in TFile::~TFile
}

//______________________________________________________________________________
void TWebFile::SetMsgReadBuffer10(const char *redirectLocation, Bool_t tempRedirect)
{
   // Set GET command for use by ReadBuffer(s)10(), handle redirection if
   // needed. Give full URL so Apache's virtual hosts solution works.

   TUrl oldUrl;
   TString oldBasicUrl;

   if (redirectLocation) {
      if (tempRedirect) { // temp redirect
         fUrlOrg      = fUrl;
         fBasicUrlOrg = fBasicUrl;
      } else {             // permanent redirect
         fUrlOrg      = "";
         fBasicUrlOrg = "";
      }

      oldUrl = fUrl;
      oldBasicUrl = fBasicUrl;

      fUrl.SetUrl(redirectLocation);
      fBasicUrl = fUrl.GetProtocol();
      fBasicUrl += "://";
      fBasicUrl += fUrl.GetHost();
      fBasicUrl += ":";
      fBasicUrl += fUrl.GetPort();
      fBasicUrl += "/";
      fBasicUrl += fUrl.GetFile();
   }

   if (fMsgReadBuffer10 != "") {
      // patch up existing command
      if (oldBasicUrl != "") {
         // change to redirection location
         fMsgReadBuffer10.ReplaceAll(oldBasicUrl, fBasicUrl);
         fMsgReadBuffer10.ReplaceAll(TString("Host: ")+oldUrl.GetHost(), TString("Host: ")+fUrl.GetHost());
      } else if (fBasicUrlOrg != "") {
         // change back from temp redirection location
         fMsgReadBuffer10.ReplaceAll(fBasicUrl, fBasicUrlOrg);
         fMsgReadBuffer10.ReplaceAll(TString("Host: ")+fUrl.GetHost(), TString("Host: ")+fUrlOrg.GetHost());
         fUrl         = fUrlOrg;
         fBasicUrl    = fBasicUrlOrg;
         fUrlOrg      = "";
         fBasicUrlOrg = "";
      }
   }

   if (fBasicUrl == "") {
      fBasicUrl += fUrl.GetProtocol();
      fBasicUrl += "://";
      fBasicUrl += fUrl.GetHost();
      fBasicUrl += ":";
      fBasicUrl += fUrl.GetPort();
      fBasicUrl += "/";
      fBasicUrl += fUrl.GetFile();
   }

   if (fMsgReadBuffer10 == "") {
      fMsgReadBuffer10 = "GET ";
      fMsgReadBuffer10 += fBasicUrl;
      if (fHTTP11)
         fMsgReadBuffer10 += " HTTP/1.1";
      else
         fMsgReadBuffer10 += " HTTP/1.0";
      fMsgReadBuffer10 += "\r\n";
      if (fHTTP11) {
         fMsgReadBuffer10 += "Host: ";
         fMsgReadBuffer10 += fUrl.GetHost();
         fMsgReadBuffer10 += "\r\n";
      }
      fMsgReadBuffer10 += BasicAuthentication();
      fMsgReadBuffer10 += gUserAgent;
      fMsgReadBuffer10 += "\r\n";
      fMsgReadBuffer10 += "Range: bytes=";
   }
}

//______________________________________________________________________________
void TWebFile::CheckProxy()
{
   // Check if shell var "http_proxy" has been set and should be used.

   if (fNoProxy)
      return;

   if (fgProxy.IsValid()) {
      fProxy = fgProxy;
      return;
   }

   TString proxy = gSystem->Getenv("http_proxy");
   if (proxy != "") {
      TUrl p(proxy);
      if (strcmp(p.GetProtocol(), "http")) {
         Error("CheckProxy", "protocol must be HTTP in proxy URL %s",
               proxy.Data());
         return;
      }
      fProxy = p;
   }
}

//______________________________________________________________________________
Bool_t TWebFile::IsOpen() const
{
   // A TWebFile that has been correctly constructed is always considered open.

   return IsZombie() ? kFALSE : kTRUE;
}

//______________________________________________________________________________
Int_t TWebFile::ReOpen(Option_t *mode)
{
   // Reopen a file with a different access mode, like from READ to
   // UPDATE or from NEW, CREATE, RECREATE, UPDATE to READ. Thus the
   // mode argument can be either "READ" or "UPDATE". The method returns
   // 0 in case the mode was successfully modified, 1 in case the mode
   // did not change (was already as requested or wrong input arguments)
   // and -1 in case of failure, in which case the file cannot be used
   // anymore. A TWebFile cannot be reopened in update mode.

   TString opt = mode;
   opt.ToUpper();

   if (opt != "READ" && opt != "UPDATE")
      Error("ReOpen", "mode must be either READ or UPDATE, not %s", opt.Data());

   if (opt == "UPDATE")
      Error("ReOpen", "update mode not allowed for a TWebFile");

   return 1;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer(char *buf, Int_t len)
{
   // Read specified byte range from remote file via HTTP daemon. This
   // routine connects to the remote host, sends the request and returns
   // the buffer. Returns kTRUE in case of error.

   Int_t st;
   if ((st = ReadBufferViaCache(buf, len))) {
      if (st == 2)
         return kTRUE;
      return kFALSE;
   }

   if (!fHasModRoot)
      return ReadBuffer10(buf, len);

   // Give full URL so Apache's virtual hosts solution works.
   // Use protocol 0.9 for efficiency, we are not interested in the 1.0 headers.
   if (fMsgReadBuffer == "") {
      fMsgReadBuffer = "GET ";
      fMsgReadBuffer += fBasicUrl;
      fMsgReadBuffer += "?";
   }
   TString msg = fMsgReadBuffer;
   msg += fOffset;
   msg += ":";
   msg += len;
   msg += "\r\n";

   if (GetFromWeb(buf, len, msg) == -1)
      return kTRUE;

   fOffset += len;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer(char *buf, Long64_t pos, Int_t len)
{
   // Read specified byte range from remote file via HTTP daemon. This
   // routine connects to the remote host, sends the request and returns
   // the buffer. Returns kTRUE in case of error.

   SetOffset(pos);
   return ReadBuffer(buf, len);
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffer10(char *buf, Int_t len)
{
   // Read specified byte range from remote file via HTTP 1.0 daemon (without
   // mod-root installed). This routine connects to the remote host, sends the
   // request and returns the buffer. Returns kTRUE in case of error.

   SetMsgReadBuffer10();

   TString msg = fMsgReadBuffer10;
   msg += fOffset;
   msg += "-";
   msg += fOffset+len-1;
   msg += "\r\n\r\n";

   Int_t n = GetFromWeb10(buf, len, msg);
   if (n == -1)
      return kTRUE;
   // The -2 error condition typically only happens when
   // GetHead() failed because not implemented, in the first call to
   // ReadBuffer() in Init(), it is not checked in ReadBuffers10().
   if (n == -2) {
      Error("ReadBuffer10", "%s does not exist", fBasicUrl.Data());
      MakeZombie();
      gDirectory = gROOT;
      return kTRUE;
   }

   fOffset += len;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
{
   // Read specified byte ranges from remote file via HTTP daemon.
   // Reads the nbuf blocks described in arrays pos and len,
   // where pos[i] is the seek position of block i of length len[i].
   // Note that for nbuf=1, this call is equivalent to TFile::ReafBuffer
   // This function is overloaded by TNetFile, TWebFile, etc.
   // Returns kTRUE in case of failure.

   if (!fHasModRoot)
      return ReadBuffers10(buf, pos, len, nbuf);

   // Give full URL so Apache's virtual hosts solution works.
   // Use protocol 0.9 for efficiency, we are not interested in the 1.0 headers.
   if (fMsgReadBuffer == "") {
      fMsgReadBuffer = "GET ";
      fMsgReadBuffer += fBasicUrl;
      fMsgReadBuffer += "?";
   }
   TString msg = fMsgReadBuffer;

   Int_t k = 0, n = 0;
   for (Int_t i = 0; i < nbuf; i++) {
      if (n) msg += ",";
      msg += pos[i] + fArchiveOffset;
      msg += ":";
      msg += len[i];
      n   += len[i];
      if (msg.Length() > 8000) {
         msg += "\r\n";
         if (GetFromWeb(&buf[k], n, msg) == -1)
            return kTRUE;
         msg = fMsgReadBuffer;
         k += n;
         n = 0;
      }
   }

   msg += "\r\n";

   if (GetFromWeb(&buf[k], n, msg) == -1)
      return kTRUE;

   return kFALSE;
}

//______________________________________________________________________________
Bool_t TWebFile::ReadBuffers10(char *buf,  Long64_t *pos, Int_t *len, Int_t nbuf)
{
   // Read specified byte ranges from remote file via HTTP 1.0 daemon (without
   // mod-root installed). Read the nbuf blocks described in arrays pos and len,
   // where pos[i] is the seek position of block i of length len[i].
   // Note that for nbuf=1, this call is equivalent to TFile::ReafBuffer
   // This function is overloaded by TNetFile, TWebFile, etc.
   // Returns kTRUE in case of failure.

   SetMsgReadBuffer10();

   TString msg = fMsgReadBuffer10;

   Int_t k = 0, n = 0, r;
   for (Int_t i = 0; i < nbuf; i++) {
      if (n) msg += ",";
      msg += pos[i] + fArchiveOffset;
      msg += "-";
      msg += pos[i] + fArchiveOffset + len[i] - 1;
      n   += len[i];
      if (msg.Length() > 8000) {
         msg += "\r\n\r\n";
         r = GetFromWeb10(&buf[k], n, msg);
         if (r == -1)
            return kTRUE;
         msg = fMsgReadBuffer10;
         k += n;
         n = 0;
      }
   }

   msg += "\r\n\r\n";

   r = GetFromWeb10(&buf[k], n, msg);
   if (r == -1)
      return kTRUE;

   return kFALSE;
}

//______________________________________________________________________________
Int_t TWebFile::GetFromWeb(char *buf, Int_t len, const TString &msg)
{
   // Read request from web server. Returns -1 in case of error,
   // 0 in case of success.

   if (!len) return 0;

   Double_t start = 0;
   if (gPerfStats) start = TTimeStamp();

   TUrl connurl;
   if (fProxy.IsValid())
      connurl = fProxy;
   else
      connurl = fUrl;

   TSocket s(connurl.GetHost(), connurl.GetPort());
   if (!s.IsValid()) {
      Error("GetFromWeb", "cannot connect to host %s", fUrl.GetHost());
      return -1;
   }

   if (s.SendRaw(msg.Data(), msg.Length()) == -1) {
      Error("GetFromWeb", "error sending command to host %s", fUrl.GetHost());
      return -1;
   }

   if (s.RecvRaw(buf, len) == -1) {
      Error("GetFromWeb", "error receiving data from host %s", fUrl.GetHost());
      return -1;
   }

   // collect statistics
   fBytesRead += len;
   fReadCalls++;
#ifdef R__WIN32
   SetFileBytesRead(GetFileBytesRead() + len);
   SetFileReadCalls(GetFileReadCalls() + 1);
#else
   fgBytesRead += len;
   fgReadCalls++;
#endif

   if (gPerfStats)
      gPerfStats->FileReadEvent(this, len, start);

   return 0;
}

//______________________________________________________________________________
Int_t TWebFile::GetFromWeb10(char *buf, Int_t len, const TString &msg)
{
   // Read multiple byte range request from web server.
   // Uses HTTP 1.0 daemon wihtout mod-root.
   // Returns -2 in case file does not exist, -1 in case
   // of error and 0 in case of success.

   if (!len) return 0;

   Double_t start = 0;
   if (gPerfStats) start = TTimeStamp();

   // open fSocket and close it when going out of scope
   TWebSocket ws(this);

   if (!fSocket || !fSocket->IsValid()) {
      Error("GetFromWeb10", "cannot connect to host %s", fUrl.GetHost());
      return -1;
   }

   if (fSocket->SendRaw(msg.Data(), msg.Length()) == -1) {
      Error("GetFromWeb10", "error sending command to host %s", fUrl.GetHost());
      return -1;
   }

   char line[8192];
   Int_t n, ret = 0, nranges = 0, ltot = 0, redirect = 0;
   TString boundary, boundaryEnd;
   Long64_t first = -1, last = -1, tot;

   while ((n = GetLine(fSocket, line, sizeof(line))) >= 0) {
      if (n == 0) {
         if (ret < 0)
            return ret;
         if (redirect) {
            ws.ReOpen();
            return GetFromWeb10(buf, len, msg);
         }

         if (first >= 0) {
            Int_t ll = Int_t(last - first) + 1;
            if (fSocket->RecvRaw(&buf[ltot], ll) == -1) {
               Error("GetFromWeb10", "error receiving data from host %s", fUrl.GetHost());
               return -1;
            }
            ltot += ll;

            first = -1;

            if (boundary == "")
               break;  // not a multipart response
         }

         continue;
      }

      if (gDebug > 0)
         Info("GetFromWeb10", "header: %s", line);

      if (boundaryEnd == line) {
         if (gDebug > 0)
            Info("GetFromWeb10", "got all headers");
         break;
      }
      if (boundary == line) {
         nranges++;
         if (gDebug > 0)
            Info("GetFromWeb10", "get new multipart byte range (%d)", nranges);
      }

      TString res = line;

      if (res.BeginsWith("HTTP/1.")) {
         if (res.BeginsWith("HTTP/1.1")) {
            if (!fHTTP11)
               fMsgReadBuffer10  = "";
            fHTTP11 = kTRUE;
         }
         TString scode = res(9, 3);
         Int_t code = scode.Atoi();
         if (code >= 500) {
            ret = -1;
            TString mess = res(13, 1000);
            Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
         } else if (code >= 400) {
            if (code == 404)
               ret = -2;   // file does not exist
            else {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         } else if (code >= 300) {
            if (code == 301 || code == 303)
               redirect = 1;   // permanent redirect
            else if (code == 302 || code == 307)
               redirect = 2;   // temp redirect
            else {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         } else if (code > 200) {
            if (code != 206) {
               ret = -1;
               TString mess = res(13, 1000);
               Error("GetFromWeb10", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
            }
         }
      } else if (res.BeginsWith("Content-Type: multipart")) {
         boundary = res(res.Index("boundary=")+9, 1000);
         if (boundary[0]=='"' && boundary[boundary.Length()-1]=='"') {
            boundaryEnd = "--" + boundary(1,boundary.Length()-2) + "--";            
         } else {
            boundaryEnd = "--" + boundary + "--";
         }
      } else if (res.BeginsWith("Content-range:")) {
#ifdef R__WIN32
{'fraction_non_alphanumeric': 0.13636889207501793, 'fraction_numerical': 0.016048840274726545, 'mean_word_length': 2.2509397083145393, 'pattern_counts': {'":': 12, '<': 23, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4592104', 'n_tokens_mistral': 13529, 'n_tokens_neox': 12827, 'n_words': 3921}