TS3WebFile.cxx

Go to the documentation of this file.

// @(#)root/net:$Id$
// Author: Fabio Hernandez   22/01/2013
//         extending an initial version by Marcelo Sousa (class TAS3File)

/*************************************************************************
 * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TS3WebFile                                                           //
//                                                                      //
// A TS3WebFile is a TWebFile which retrieves the file contents from a  //
// web server implementing the REST API of the Amazon S3 protocol. This //
// class is meant to be as generic as possible to be used with files    //
// hosted not only by Amazon S3 servers but also by other providers     //
// implementing the core of the S3 protocol.                            //
//                                                                      //
// The S3 protocol works on top of HTTPS (and HTTP) and imposes that    //
// each HTTP request be signed using a specific convention: the request //
// must include an 'Authorization' header which contains the signature  //
// of a concatenation of selected request fields. For signing the       //
// request, an 'Access Key Id' and a 'Secret Access Key' need to be     //
// known. These keys are used by the S3 servers to identify the client  //
// and to authenticate the request as genuine.                          //
//                                                                      //
// As an end user, you must know the Access Key and Secret Access Key   //
// in order to access each S3 file. They are provided to you by your S3 //
// service provider. Those two keys can be provided to ROOT when        //
// initializing an object of this class by two means:                   //
// a) by using the environmental variables S3_ACCESS_KEY and            //
//    S3_SECRET_KEY, or                                                 //
// b) by specifying them when opening each file.                        //
//                                                                      //
// You can use AWS temporary security credentials (temporary access key //
// and secret access key), but you must also give the associated        //
// session token. The token may be set in the S3_SESSION_TOKEN          //
// environmental variable, or on open in the TOKEN option.              //
//                                                                      //
// The first method is convenient if all the S3 files you want to       //
// access are hosted by a single provider. The second one is more       //
// flexible as it allows you to specify which credentials to use        //
// on a per-file basis. See the documentation of the constructor of     //
// this class for details on the syntax.                                //
//                                                                      //
// For generating and signing the HTTP request, this class uses         //
// TS3HTTPRequest.                                                      //
//                                                                      //
// For more information on the details of S3 protocol please refer to:  //
// "Amazon Simple Storage Service Developer Guide":                     //
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/Welcome.html   //
//                                                                      //
// "Amazon Simple Storage Service REST API Reference"                   //
//  http://docs.amazonwebservices.com/AmazonS3/latest/API/APIRest.html  //
//////////////////////////////////////////////////////////////////////////

#include "TS3WebFile.h"
#include "TROOT.h"
#include "TError.h"
#include "TSystem.h"
#include "TPRegexp.h"
#include "TEnv.h"


ClassImp(TS3WebFile);

////////////////////////////////////////////////////////////////////////////////
/// Construct a TS3WebFile object. The path argument is a URL of one of the
/// following forms:
///
///         s3://host.example.com/bucket/path/to/my/file
///     s3http://host.example.com/bucket/path/to/my/file
///    s3https://host.example.com/bucket/path/to/my/file
///        as3://host.example.com/bucket/path/to/my/file
///
/// For files hosted by Google Storage, use the following forms:
///
///        gs://storage.googleapis.com/bucket/path/to/my/file
///    gshttp://storage.googleapis.com/bucket/path/to/my/file
///  gsthttps://storage.googleapis.com/bucket/path/to/my/file
///
/// The 'as3' scheme is accepted for backwards compatibility but its usage is
/// deprecated.
///
/// The recommended way to create an instance of this class is through
/// TFile::Open, for instance:
///
/// TFile* f1 = TFile::Open("s3://host.example.com/bucket/path/to/my/file")
/// TFile* f2 = TFile::Open("gs://storage.googleapis.com/bucket/path/to/my/file")
///
/// The specified scheme (i.e. s3, s3http, s3https, ...) determines the underlying
/// transport protocol to use for downloading the file contents, namely HTTP or HTTPS.
/// The 's3', 's3https', 'gs' and 'gshttps' schemes imply using HTTPS as the transport
/// protocol. The 's3http', 'as3' and 'gshttp' schemes imply using HTTP as the transport
/// protocol.
///
/// The 'options' argument can contain 'NOPROXY' if you want to bypass
/// the HTTP proxy when retrieving this file's contents. As for any TWebFile-derived
/// object, the URL of the web proxy can be specified by setting an environmental
/// variable 'http_proxy'. If this variable is set, we ask that proxy to route our
/// requests HTTP(S) requests to the file server.
///
/// In addition, you can also use the 'options' argument to provide the access key
/// and secret key to be used for authentication purposes for this file by using a
/// string of the form "AUTH=myAccessKey:mySecretkey". This may be useful to
/// open several files hosted by different providers in the same program/macro,
/// where the environemntal variables solution is not convenient (see below).
///
/// To use AWS temporary security credentials you need to specify the session
/// token. This can be added to the options argument with a string of the form
/// TOKEN=mySessionToken. The temporary access and secret keys must also be
/// available, either via the AUTH option or by environmental variable.
///
/// If you need to specify more than one option separate them by ' '
/// (blank), for instance:
/// "NOPROXY AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+"
///
/// Examples:
///    TFile* f1 = TFile::Open("s3://host.example.com/bucket/path/to/my/file",
///                            "NOPROXY AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+");
///    TFile* f2 = TFile::Open("s3://host.example.com/bucket/path/to/my/file",
///                            "AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+");
///    TFile* f3 = TFile::Open("s3://host.example.com/bucket/path/to/my/file",
///                            "TOKEN=AQoDYXdzEM///////////wEa8AHEYmCinjD+TsGEjtgKSMAT6wnY");
///
/// If there is no authentication information in the 'options' argument
/// (i.e. not AUTH="....") the values of the environmental variables
/// S3_ACCESS_KEY and S3_SECRET_KEY (if set) are expected to contain
/// the access key id and the secret access key, respectively. You have
/// been provided with these credentials by your S3 service provider.
///
/// If neither the AUTH information is provided in the 'options' argument
/// nor the environmental variables are set, we try to open the file
/// without providing any authentication information to the server. This
/// is useful when the file is set an access control that allows for
/// any unidentified user to read the file.

TS3WebFile::TS3WebFile(const char* path, Option_t* options)
           : TWebFile(path, "IO")
{
   // Make sure this is a valid S3 path. We accept 'as3' as a scheme, for
   // backwards compatibility
   Bool_t doMakeZombie = kFALSE;
   TString errorMsg;
   TString accessKey;
   TString secretKey;
   TString token;
   TPMERegexp rex("^([a]?s3|s3http[s]?|gs|gshttp[s]?){1}://([^/]+)/([^/]+)/([^/].*)", "i");
   if (rex.Match(TString(path)) != 5) {
      errorMsg = TString::Format("invalid S3 path '%s'", path);
      doMakeZombie = kTRUE;
   }
   else if (!ParseOptions(options, accessKey, secretKey, token)) {
      errorMsg = TString::Format("could not parse options '%s'", options);
      doMakeZombie = kTRUE;
   }

   // Should we stop initializing this object?
   if (doMakeZombie) {
      Error("TS3WebFile", "%s", (const char*)errorMsg);
      MakeZombie();
      gDirectory = gROOT;
      return;
   }

   // Set this S3 object's URL, the bucket name this file is located in
   // and the object key
   fS3Request.SetBucket(rex[3]);
   fS3Request.SetObjectKey(TString::Format("/%s", (const char*)rex[4]));

   // Initialize super-classes data members (fUrl is a data member of
   // super-super class TFile)
   TString protocol = "https";
   if (rex[1].EndsWith("http", TString::kIgnoreCase) ||
       rex[1].EqualTo("as3", TString::kIgnoreCase))
      protocol = "http";
   fUrl.SetUrl(TString::Format("%s://%s/%s/%s", (const char*)protocol,
      (const char*)rex[2], (const char*)rex[3], (const char*)rex[4]));

   // Set S3-specific data members. If the access and secret keys are not
   // provided in the 'options' argument we look in the environmental
   // variables.
   const char* kAccessKeyEnv = "S3_ACCESS_KEY";
   const char* kSecretKeyEnv = "S3_SECRET_KEY";
   const char* kSessionToken = "S3_SESSION_TOKEN";
   if (accessKey.IsNull())
      GetCredentialsFromEnv(kAccessKeyEnv, kSecretKeyEnv, kSessionToken,
         accessKey, secretKey, token);

   // Initialize the S3 HTTP request
   fS3Request.SetHost(fUrl.GetHost());
   if (accessKey.IsNull() || secretKey.IsNull()) {
      // We have no authentication information, neither in the options
      // nor in the enviromental variables. So may be this is a
      // world-readable file, so let's continue and see if
      // we can open it.
      fS3Request.SetAuthType(TS3HTTPRequest::kNoAuth);
   } else {
      // Set the authentication information we need to use
      // for this file
      fS3Request.SetAuthKeys(accessKey, secretKey);
      if (!token.IsNull())
         fS3Request.SetSessionToken(token);
      if (rex[1].BeginsWith("gs"))
         fS3Request.SetAuthType(TS3HTTPRequest::kGoogle);
      else
         fS3Request.SetAuthType(TS3HTTPRequest::kAmazon);
   }

   // Assume this server does not serve multi-range HTTP GET requests. We
   // will detect this when the HTTP headers of this files are retrieved
   // later in the initialization process
   fUseMultiRange = kFALSE;

   // Call super-class initializer
   TWebFile::Init(kFALSE);

   // Were there some errors opening this file?
   if (IsZombie() && (accessKey.IsNull() || secretKey.IsNull())) {
      // We could not open the file and we have no authentication information
      // so inform the user so that they can check.
      Error("TS3WebFile", "could not find authentication info in "\
         "'options' argument and at least one of the environment variables '%s' or '%s' is not set",
         kAccessKeyEnv, kSecretKeyEnv);
   }
}


////////////////////////////////////////////////////////////////////////////////
/// Extracts the S3 authentication key pair (access key and secret key)
/// from the options. The authentication credentials can be specified in
/// the options provided to the constructor of this class as a string
/// containing: "AUTH=<access key>:<secret key>" and can include other
/// options, for instance "NOPROXY" for not using the HTTP proxy for
/// accessing this file's contents.
/// For instance:
/// "NOPROXY AUTH=F38XYZABCDeFgHiJkLm:V+frt4re7J1euSNFnmaf8wwmI401234E7kzxZ/TTM+"
/// A security token may be given by the TOKEN option, in order to allow the
/// use of a temporary key pair.

Bool_t TS3WebFile::ParseOptions(Option_t* options, TString& accessKey, TString& secretKey, TString& token)
{
   TString optStr = (const char*)options;
   if (optStr.IsNull())
      return kTRUE;

   fNoProxy = kFALSE;
   if (optStr.Contains("NOPROXY", TString::kIgnoreCase))
      fNoProxy = kTRUE;
   CheckProxy();

   // Look in the options string for the authentication information.
   TPMERegexp rex_token("(^TOKEN=|^.* TOKEN=)([\\S]+)[\\s]*.*$", "i");
   if (rex_token.Match(optStr) == 3) {
      token = rex_token[2];
   }
   TPMERegexp rex("(^AUTH=|^.* AUTH=)([a-z0-9]+):([a-z0-9+/]+)[\\s]*.*$", "i");
   if (rex.Match(optStr) == 4) {
      accessKey = rex[2];
      secretKey = rex[3];
   }
   if (gDebug > 0)
      Info("ParseOptions", "using authentication information from 'options' argument");
   return kTRUE;
}


////////////////////////////////////////////////////////////////////////////////
/// Overwrites TWebFile::GetHead() for retrieving the HTTP headers of this
/// file. Uses TS3HTTPRequest to generate an HTTP HEAD request which includes
/// the authorization header expected by the S3 server.

Int_t TS3WebFile::GetHead()
{
   fMsgGetHead = fS3Request.GetRequest(TS3HTTPRequest::kHEAD);
   return TWebFile::GetHead();
}


////////////////////////////////////////////////////////////////////////////////
/// Overwrites TWebFile::SetMsgReadBuffer10() for setting the HTTP GET
/// request compliant to the authentication mechanism used by the S3
/// protocol. The GET request must contain an "Authorization" header with
/// the signature of the request, generated using the user's secret access
/// key.

void TS3WebFile::SetMsgReadBuffer10(const char* redirectLocation, Bool_t tempRedirect)
{
   TWebFile::SetMsgReadBuffer10(redirectLocation, tempRedirect);
   fMsgReadBuffer10 = fS3Request.GetRequest(TS3HTTPRequest::kGET, kFALSE) + "Range: bytes=";
   return;
}


////////////////////////////////////////////////////////////////////////////////

Bool_t TS3WebFile::ReadBuffers(char* buf, Long64_t* pos, Int_t* len, Int_t nbuf)
{
   // Overwrites TWebFile::ReadBuffers() for reading specified byte ranges.
   // According to the kind of server this file is hosted by, we use a
   // single HTTP request with a muti-range header or we generate multiple
   // requests with a single range each.

   // Does this server support multi-range GET requests?
   if (fUseMultiRange)
      return TWebFile::ReadBuffers(buf, pos, len, nbuf);

   // Send multiple GET requests with a single range of bytes
   // Adapted from original version by Wang Lu
   for (Int_t i=0, offset=0; i < nbuf; i++) {
      TString rangeHeader = TString::Format("Range: bytes=%lld-%lld\r\n\r\n",
         pos[i], pos[i] + len[i] - 1);
      TString s3Request = fS3Request.GetRequest(TS3HTTPRequest::kGET, kFALSE) + rangeHeader;
      if (GetFromWeb10(&buf[offset], len[i], s3Request) == -1)
         return kTRUE;
      offset += len[i];
   }
   return kFALSE;
}


////////////////////////////////////////////////////////////////////////////////
/// This method is called by the super-class TWebFile when a HTTP header
/// for this file is retrieved. We scan the 'Server' header to detect the
/// type of S3 server this file is hosted on and to determine if it is
/// known to support multi-range HTTP GET requests. Some S3 servers (for
/// instance Amazon's) do not support that feature and when they
/// receive a multi-range request they sent back the whole file contents.
/// For this class, if the server do not support multirange requests
/// we issue multiple single-range requests instead.

void TS3WebFile::ProcessHttpHeader(const TString& headerLine)
{
   TPMERegexp rex("^Server: (.+)", "i");
   if (rex.Match(headerLine) != 2)
      return;

   // Extract the identity of this server and compare it to the
   // identify of the servers known to support multi-range requests.
   // The list of server identities is expected to be found in ROOT
   // configuration.
   TString serverId = rex[1].ReplaceAll("\r", "").ReplaceAll("\n", "");
   TString multirangeServers(gEnv->GetValue("TS3WebFile.Root.MultiRangeServer", ""));
   fUseMultiRange = multirangeServers.Contains(serverId, TString::kIgnoreCase) ? kTRUE : kFALSE;
}


////////////////////////////////////////////////////////////////////////////////
/// Sets the access and secret keys from the environmental variables, if
/// they are both set. Sets the security session token if it is given.

Bool_t TS3WebFile::GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv,
                                         const char* tokenEnv, TString& outAccessKey,
                                         TString& outSecretKey, TString& outToken)
{
   // Look first in the recommended environmental variables. Both variables
   // must be set.
   TString accKey = gSystem->Getenv(accessKeyEnv);
   TString secKey = gSystem->Getenv(secretKeyEnv);
   TString token = gSystem->Getenv(tokenEnv);
   if (!token.IsNull()) {
      outToken = token;
   }
   if (!accKey.IsNull() && !secKey.IsNull()) {
      outAccessKey = accKey;
      outSecretKey = secKey;
      if (gDebug > 0)
         Info("GetCredentialsFromEnv", "using authentication information from environmental variables '%s' and '%s'",
            accessKeyEnv, secretKeyEnv);
      return kTRUE;
   }

   // Look now in the legacy environmental variables, for keeping backwards
   // compatibility.
   accKey = gSystem->Getenv("S3_ACCESS_ID"); // Legacy access key
   secKey = gSystem->Getenv("S3_ACCESS_KEY"); // Legacy secret key
   if (!accKey.IsNull() && !secKey.IsNull()) {
      Warning("SetAuthKeys", "usage of S3_ACCESS_ID and S3_ACCESS_KEY environmental variables is deprecated.");
      Warning("SetAuthKeys", "please use S3_ACCESS_KEY and S3_SECRET_KEY environmental variables.");
      outAccessKey = accKey;
      outSecretKey = secKey;
      return kTRUE;
   }

   return kFALSE;
}