<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "webkit/appcache/appcache_update_job.h"

#include "base/compiler_specific.h"
#include "base/message_loop.h"
#include "base/string_util.h"
#include "net/base/io_buffer.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/http/http_request_headers.h"
#include "webkit/appcache/appcache_group.h"
#include "webkit/appcache/appcache_policy.h"

namespace appcache {

static const int kBufferSize = 32768;
static const size_t kMaxConcurrentUrlFetches = 2;
static const int kMax503Retries = 3;

// Extra info associated with requests for use during response processing.
// This info is deleted when the URLRequest is deleted.
class UpdateJobInfo :  public URLRequest::UserData {
 public:
  enum RequestType {
    MANIFEST_FETCH,
    URL_FETCH,
    MASTER_ENTRY_FETCH,
    MANIFEST_REFETCH,
  };

  explicit UpdateJobInfo(RequestType request_type)
      : type_(request_type),
        buffer_(new net::IOBuffer(kBufferSize)),
        retry_503_attempts_(0),
        update_job_(NULL),
        request_(NULL),
        ALLOW_THIS_IN_INITIALIZER_LIST(write_callback_(
            this, &UpdateJobInfo::OnWriteComplete)) {
  }

  void SetUpResponseWriter(AppCacheResponseWriter* writer,
                           AppCacheUpdateJob* update,
                           URLRequest* request) {
    DCHECK(!response_writer_.get());
    response_writer_.reset(writer);
    update_job_ = update;
    request_ = request;
  }

  void OnWriteComplete(int result) {
    // A completed write may delete the URL request and this object.
    update_job_->OnWriteResponseComplete(result, request_, this);
  }

  RequestType type_;
  scoped_refptr<net::IOBuffer> buffer_;
  int retry_503_attempts_;

  // The entry from the newest cache for this url, used for 304 responses.
  AppCacheEntry existing_entry_;

  // Info needed to write responses to storage and process callbacks.
  scoped_ptr<AppCacheResponseWriter> response_writer_;
  AppCacheUpdateJob* update_job_;
  URLRequest* request_;
  net::CompletionCallbackImpl<UpdateJobInfo> write_callback_;
};

// Helper class for collecting hosts per frontend when sending notifications
// so that only one notification is sent for all hosts using the same frontend.
class HostNotifier {
 public:
  typedef std::vector<int> HostIds;
  typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;

  // Caller is responsible for ensuring there will be no duplicate hosts.
  void AddHost(AppCacheHost* host) {
    std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
        NotifyHostMap::value_type(host->frontend(), HostIds()));
    ret.first->second.push_back(host->host_id());
  }

  void AddHosts(const std::set<AppCacheHost*>& hosts) {
    for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
         it != hosts.end(); ++it) {
      AddHost(*it);
    }
  }

  void SendNotifications(EventID event_id) {
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnEventRaised(it->second, event_id);
    }
  }

  void SendProgressNotifications(
      const GURL& url, int num_total, int num_complete) {
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnProgressEventRaised(it->second, url,
                                      num_total, num_complete);
    }
  }

  void SendErrorNotifications(const std::string& error_message) {
    DCHECK(!error_message.empty());
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnErrorEventRaised(it->second, error_message);
    }
  }

 private:
  NotifyHostMap hosts_to_notify;
};

AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
                                     AppCacheGroup* group)
    : ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)),
      service_(service),
      group_(group),
      update_type_(UNKNOWN_TYPE),
      internal_state_(FETCH_MANIFEST),
      master_entries_completed_(0),
      url_fetches_completed_(0),
      manifest_url_request_(NULL),
      stored_state_(UNSTORED),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_info_write_callback_(
          this, &AppCacheUpdateJob::OnManifestInfoWriteComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_data_write_callback_(
          this, &AppCacheUpdateJob::OnManifestDataWriteComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_data_read_callback_(
          this, &AppCacheUpdateJob::OnManifestDataReadComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(policy_callback_(
          new net::CancelableCompletionCallback<AppCacheUpdateJob>(
              this, &AppCacheUpdateJob::OnPolicyCheckComplete))) {
  DCHECK(group_);
  manifest_url_ = group_->manifest_url();
}

AppCacheUpdateJob::~AppCacheUpdateJob() {
  if (internal_state_ != COMPLETED)
    Cancel();

  DCHECK(!manifest_url_request_);
  DCHECK(pending_url_fetches_.empty());
  DCHECK(!inprogress_cache_);
  DCHECK(pending_master_entries_.empty());
  DCHECK(master_entry_fetches_.empty());

  if (group_)
    group_->SetUpdateStatus(AppCacheGroup::IDLE);

  policy_callback_->Cancel();
}

UpdateJobInfo* AppCacheUpdateJob::GetUpdateJobInfo(URLRequest* request) {
  return static_cast<UpdateJobInfo*>(request->GetUserData(this));
}

void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
                                    const GURL& new_master_resource) {
  DCHECK(group_->update_job() == this);
  DCHECK(!group_->is_obsolete());

  bool is_new_pending_master_entry = false;
  if (!new_master_resource.is_empty()) {
    DCHECK(new_master_resource == host->pending_master_entry_url());
    DCHECK(!new_master_resource.has_ref());
    DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());

    // Cannot add more to this update if already terminating.
    if (IsTerminating()) {
      group_->QueueUpdate(host, new_master_resource);
      return;
    }

    std::pair<PendingMasters::iterator, bool> ret =
        pending_master_entries_.insert(
            PendingMasters::value_type(new_master_resource, PendingHosts()));
    is_new_pending_master_entry = ret.second;
    ret.first->second.push_back(host);
    host->AddObserver(this);
  }

  // Notify host (if any) if already checking or downloading.
  AppCacheGroup::UpdateStatus update_status = group_->update_status();
  if (update_status == AppCacheGroup::CHECKING ||
      update_status == AppCacheGroup::DOWNLOADING) {
    if (host) {
      NotifySingleHost(host, CHECKING_EVENT);
      if (update_status == AppCacheGroup::DOWNLOADING)
        NotifySingleHost(host, DOWNLOADING_EVENT);

      // Add to fetch list or an existing entry if already fetched.
      if (!new_master_resource.is_empty()) {
        AddMasterEntryToFetchList(host, new_master_resource,
                                  is_new_pending_master_entry);
      }
    }
    return;
  }

  // Begin update process for the group.
  group_->SetUpdateStatus(AppCacheGroup::CHECKING);
  if (group_->HasCache()) {
    update_type_ = UPGRADE_ATTEMPT;
    NotifyAllAssociatedHosts(CHECKING_EVENT);
  } else {
    update_type_ = CACHE_ATTEMPT;
    DCHECK(host);
    NotifySingleHost(host, CHECKING_EVENT);
  }

  if (!new_master_resource.is_empty()) {
    AddMasterEntryToFetchList(host, new_master_resource,
                              is_new_pending_master_entry);
  }

  if (update_type_ == CACHE_ATTEMPT)
    CheckPolicy();
  else
    FetchManifest(true);
}

void AppCacheUpdateJob::CheckPolicy() {
  int rv = net::OK;
  policy_callback_->AddRef();  // Balanced in OnPolicyCheckComplete.
  AppCachePolicy* policy = service_->appcache_policy();
  if (policy) {
    rv = policy->CanCreateAppCache(manifest_url_, policy_callback_);
    if (rv == net::ERR_IO_PENDING)
      return;
  }
  OnPolicyCheckComplete(rv);
}

void AppCacheUpdateJob::OnPolicyCheckComplete(int rv) {
  policy_callback_->Release();  // Balanced in CheckPolicy.
  if (rv == net::OK) {
    FetchManifest(true);
    return;
  }

  group_->NotifyContentBlocked();

  const char* kErrorMessage =
      "Cache creation was blocked by the content policy";
  MessageLoop::current()->PostTask(FROM_HERE,
      method_factory_.NewRunnableMethod(
          &AppCacheUpdateJob::HandleCacheFailure,
          kErrorMessage));
}

void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
  // 6.9.4 cache failure steps 2-8.
  DCHECK(internal_state_ != CACHE_FAILURE);
  DCHECK(!error_message.empty());
  internal_state_ = CACHE_FAILURE;
  CancelAllUrlFetches();
  CancelAllMasterEntryFetches(error_message);
  NotifyAllError(error_message);
  DiscardInprogressCache();
  internal_state_ = COMPLETED;
  DeleteSoon();  // To unwind the stack prior to deletion.
}

void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
  DCHECK(!manifest_url_request_);
  manifest_url_request_ = new URLRequest(manifest_url_, this);
  UpdateJobInfo::RequestType fetch_type = is_first_fetch ?
      UpdateJobInfo::MANIFEST_FETCH : UpdateJobInfo::MANIFEST_REFETCH;
  manifest_url_request_->SetUserData(this, new UpdateJobInfo(fetch_type));
  manifest_url_request_->set_context(service_->request_context());
  manifest_url_request_->set_load_flags(
      manifest_url_request_->load_flags() | net::LOAD_DISABLE_INTERCEPT);

  // Add any necessary Http headers before sending fetch request.
  if (is_first_fetch) {
    AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
        group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
    if (entry) {
      // Asynchronously load response info for manifest from newest cache.
      service_->storage()->LoadResponseInfo(manifest_url_,
                                            entry->response_id(), this);
    } else {
      manifest_url_request_->Start();
    }
  } else {
    DCHECK(internal_state_ == REFETCH_MANIFEST);
    DCHECK(manifest_response_info_.get());
    AddConditionalHeaders(manifest_url_request_,
                          manifest_response_info_.get());
    manifest_url_request_->Start();
  }
}

void AppCacheUpdateJob::AddConditionalHeaders(
    URLRequest* request, const net::HttpResponseInfo* info) {
  DCHECK(request && info);
  net::HttpRequestHeaders extra_headers;

  // Add If-Modified-Since header if response info has Last-Modified header.
  const std::string last_modified = "Last-Modified";
  std::string last_modified_value;
  info->headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
  if (!last_modified_value.empty()) {
    extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
                            last_modified_value);
  }

  // Add If-None-Match header if resposne info has ETag header.
  const std::string etag = "ETag";
  std::string etag_value;
  info->headers->EnumerateHeader(NULL, etag, &etag_value);
  if (!etag_value.empty()) {
    extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
                            etag_value);
  }
  if (!extra_headers.IsEmpty())
    request->SetExtraRequestHeaders(extra_headers);
}

void AppCacheUpdateJob::OnResponseStarted(URLRequest *request) {
  if (request->status().is_success() &&
      (request->GetResponseCode() / 100) == 2) {
    // Write response info to storage for URL fetches. Wait for async write
    // completion before reading any response data.
    UpdateJobInfo* info = GetUpdateJobInfo(request);
    if (info->type_ == UpdateJobInfo::URL_FETCH ||
        info->type_ == UpdateJobInfo::MASTER_ENTRY_FETCH) {
      info->SetUpResponseWriter(
          service_->storage()->CreateResponseWriter(manifest_url_),
          this, request);
      stored_response_ids_.push_back(info->response_writer_->response_id());
      scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
          new HttpResponseInfoIOBuffer(
              new net::HttpResponseInfo(request->response_info()));
      info->response_writer_->WriteInfo(io_buffer, &info->write_callback_);
    } else {
      ReadResponseData(request);
    }
  } else {
    OnResponseCompleted(request);
  }
}

void AppCacheUpdateJob::ReadResponseData(URLRequest* request) {
  if (internal_state_ == CACHE_FAILURE || internal_state_ == CANCELLED ||
      internal_state_ == COMPLETED) {
    return;
  }

  int bytes_read = 0;
  UpdateJobInfo* info = GetUpdateJobInfo(request);
  request->Read(info->buffer_, kBufferSize, &bytes_read);
  OnReadCompleted(request, bytes_read);
}

void AppCacheUpdateJob::OnReadCompleted(URLRequest* request, int bytes_read) {
  bool data_consumed = true;
  if (request->status().is_success() && bytes_read > 0) {
    UpdateJobInfo* info = GetUpdateJobInfo(request);
    data_consumed = ConsumeResponseData(request, info, bytes_read);
    if (data_consumed) {
      bytes_read = 0;
      while (request->Read(info->buffer_, kBufferSize, &bytes_read)) {
        if (bytes_read > 0) {
          data_consumed = ConsumeResponseData(request, info, bytes_read);
          if (!data_consumed)
            break;  // wait for async data processing, then read more
        } else {
          break;
        }
      }
    }
  }

  if (data_consumed && !request->status().is_io_pending())
    OnResponseCompleted(request);
}

bool AppCacheUpdateJob::ConsumeResponseData(URLRequest* request,
                                            UpdateJobInfo* info,
                                            int bytes_read) {
  DCHECK_GT(bytes_read, 0);
  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
      manifest_data_.append(info->buffer_->data(), bytes_read);
      break;
    case UpdateJobInfo::URL_FETCH:
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      DCHECK(info->response_writer_.get());
      info->response_writer_->WriteData(info->buffer_, bytes_read,
                                        &info->write_callback_);
      return false;  // wait for async write completion to continue reading
    case UpdateJobInfo::MANIFEST_REFETCH:
      manifest_refetch_data_.append(info->buffer_->data(), bytes_read);
      break;
    default:
      NOTREACHED();
  }
  return true;
}

void AppCacheUpdateJob::OnWriteResponseComplete(int result,
                                                URLRequest* request,
                                                UpdateJobInfo* info) {
  if (result < 0) {
    request->Cancel();
    OnResponseCompleted(request);
    return;
  }

  ReadResponseData(request);
}

void AppCacheUpdateJob::OnReceivedRedirect(URLRequest* request,
                                           const GURL& new_url,
                                           bool* defer_redirect) {
  // Redirect is not allowed by the update process.
  request->Cancel();
  OnResponseCompleted(request);
}

void AppCacheUpdateJob::OnResponseCompleted(URLRequest* request) {
  // Retry for 503s where retry-after is 0.
  if (request->status().is_success() &&
      request->GetResponseCode() == 503 &&
      RetryRequest(request)) {
    return;
  }

  UpdateJobInfo* info = GetUpdateJobInfo(request);
  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
      HandleManifestFetchCompleted(request);
      break;
    case UpdateJobInfo::URL_FETCH:
      HandleUrlFetchCompleted(request);
      break;
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      HandleMasterEntryFetchCompleted(request);
      break;
    case UpdateJobInfo::MANIFEST_REFETCH:
      HandleManifestRefetchCompleted(request);
      break;
    default:
      NOTREACHED();
  }

  delete request;
}

bool AppCacheUpdateJob::RetryRequest(URLRequest* request) {
  UpdateJobInfo* info = GetUpdateJobInfo(request);
  if (info->retry_503_attempts_ >= kMax503Retries) {
    return false;
  }

  if (!request->response_headers()->HasHeaderValue("retry-after", "0"))
    return false;

  const GURL& url = request->original_url();
  URLRequest* retry = new URLRequest(url, this);
  UpdateJobInfo* retry_info = new UpdateJobInfo(info->type_);
  retry_info->retry_503_attempts_ = info->retry_503_attempts_ + 1;
  retry_info->existing_entry_ = info->existing_entry_;
  retry->SetUserData(this, retry_info);
  retry->set_context(request->context());
  retry->set_load_flags(request->load_flags());

  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
    case UpdateJobInfo::MANIFEST_REFETCH:
      manifest_url_request_ = retry;
      manifest_data_.clear();
      break;
    case UpdateJobInfo::URL_FETCH:
      pending_url_fetches_.erase(url);
      pending_url_fetches_.insert(PendingUrlFetches::value_type(url, retry));
      break;
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      master_entry_fetches_.erase(url);
      master_entry_fetches_.insert(PendingUrlFetches::value_type(url, retry));
      break;
    default:
      NOTREACHED();
  }

  retry->Start();

  delete request;
  return true;
}

void AppCacheUpdateJob::HandleManifestFetchCompleted(URLRequest* request) {
  DCHECK(internal_state_ == FETCH_MANIFEST);
  manifest_url_request_ = NULL;

  int response_code = -1;
  std::string mime_type;
  if (request->status().is_success()) {
    response_code = request->GetResponseCode();
    request->GetMimeType(&mime_type);
  }

  if ((response_code / 100 == 2) && mime_type == kManifestMimeType) {
    manifest_response_info_.reset(
        new net::HttpResponseInfo(request->response_info()));
    if (update_type_ == UPGRADE_ATTEMPT)
      CheckIfManifestChanged();  // continues asynchronously
    else
      ContinueHandleManifestFetchCompleted(true);
  } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
    ContinueHandleManifestFetchCompleted(false);
  } else if (response_code == 404 || response_code == 410) {
    service_->storage()->MakeGroupObsolete(group_, this);  // async
  } else {
    const char* kFormatString = "Manifest fetch failed (%d) %s";
    const std::string message = StringPrintf(kFormatString, response_code,
                                             manifest_url_.spec().c_str());
    HandleCacheFailure(message);
  }
}

void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
                                            bool success) {
  DCHECK(master_entry_fetches_.empty());
  CancelAllMasterEntryFetches("The group has been made obsolete");
  if (success) {
    DCHECK(group->is_obsolete());
    NotifyAllAssociatedHosts(OBSOLETE_EVENT);
    internal_state_ = COMPLETED;
    MaybeCompleteUpdate();
  } else {
    // Treat failure to mark group obsolete as a cache failure.
    HandleCacheFailure("Failed to make the group as obsolete");
  }
}

void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
  DCHECK(internal_state_ == FETCH_MANIFEST);

  if (!changed) {
    DCHECK(update_type_ == UPGRADE_ATTEMPT);
    internal_state_ = NO_UPDATE;<commit_msg>A better error message for the mime type of a manifest file being off.<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "webkit/appcache/appcache_update_job.h"

#include "base/compiler_specific.h"
#include "base/message_loop.h"
#include "base/string_util.h"
#include "net/base/io_buffer.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/http/http_request_headers.h"
#include "webkit/appcache/appcache_group.h"
#include "webkit/appcache/appcache_policy.h"

namespace appcache {

static const int kBufferSize = 32768;
static const size_t kMaxConcurrentUrlFetches = 2;
static const int kMax503Retries = 3;

// Extra info associated with requests for use during response processing.
// This info is deleted when the URLRequest is deleted.
class UpdateJobInfo :  public URLRequest::UserData {
 public:
  enum RequestType {
    MANIFEST_FETCH,
    URL_FETCH,
    MASTER_ENTRY_FETCH,
    MANIFEST_REFETCH,
  };

  explicit UpdateJobInfo(RequestType request_type)
      : type_(request_type),
        buffer_(new net::IOBuffer(kBufferSize)),
        retry_503_attempts_(0),
        update_job_(NULL),
        request_(NULL),
        ALLOW_THIS_IN_INITIALIZER_LIST(write_callback_(
            this, &UpdateJobInfo::OnWriteComplete)) {
  }

  void SetUpResponseWriter(AppCacheResponseWriter* writer,
                           AppCacheUpdateJob* update,
                           URLRequest* request) {
    DCHECK(!response_writer_.get());
    response_writer_.reset(writer);
    update_job_ = update;
    request_ = request;
  }

  void OnWriteComplete(int result) {
    // A completed write may delete the URL request and this object.
    update_job_->OnWriteResponseComplete(result, request_, this);
  }

  RequestType type_;
  scoped_refptr<net::IOBuffer> buffer_;
  int retry_503_attempts_;

  // The entry from the newest cache for this url, used for 304 responses.
  AppCacheEntry existing_entry_;

  // Info needed to write responses to storage and process callbacks.
  scoped_ptr<AppCacheResponseWriter> response_writer_;
  AppCacheUpdateJob* update_job_;
  URLRequest* request_;
  net::CompletionCallbackImpl<UpdateJobInfo> write_callback_;
};

// Helper class for collecting hosts per frontend when sending notifications
// so that only one notification is sent for all hosts using the same frontend.
class HostNotifier {
 public:
  typedef std::vector<int> HostIds;
  typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;

  // Caller is responsible for ensuring there will be no duplicate hosts.
  void AddHost(AppCacheHost* host) {
    std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
        NotifyHostMap::value_type(host->frontend(), HostIds()));
    ret.first->second.push_back(host->host_id());
  }

  void AddHosts(const std::set<AppCacheHost*>& hosts) {
    for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
         it != hosts.end(); ++it) {
      AddHost(*it);
    }
  }

  void SendNotifications(EventID event_id) {
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnEventRaised(it->second, event_id);
    }
  }

  void SendProgressNotifications(
      const GURL& url, int num_total, int num_complete) {
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnProgressEventRaised(it->second, url,
                                      num_total, num_complete);
    }
  }

  void SendErrorNotifications(const std::string& error_message) {
    DCHECK(!error_message.empty());
    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
         it != hosts_to_notify.end(); ++it) {
      AppCacheFrontend* frontend = it->first;
      frontend->OnErrorEventRaised(it->second, error_message);
    }
  }

 private:
  NotifyHostMap hosts_to_notify;
};

AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
                                     AppCacheGroup* group)
    : ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)),
      service_(service),
      group_(group),
      update_type_(UNKNOWN_TYPE),
      internal_state_(FETCH_MANIFEST),
      master_entries_completed_(0),
      url_fetches_completed_(0),
      manifest_url_request_(NULL),
      stored_state_(UNSTORED),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_info_write_callback_(
          this, &AppCacheUpdateJob::OnManifestInfoWriteComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_data_write_callback_(
          this, &AppCacheUpdateJob::OnManifestDataWriteComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(manifest_data_read_callback_(
          this, &AppCacheUpdateJob::OnManifestDataReadComplete)),
      ALLOW_THIS_IN_INITIALIZER_LIST(policy_callback_(
          new net::CancelableCompletionCallback<AppCacheUpdateJob>(
              this, &AppCacheUpdateJob::OnPolicyCheckComplete))) {
  DCHECK(group_);
  manifest_url_ = group_->manifest_url();
}

AppCacheUpdateJob::~AppCacheUpdateJob() {
  if (internal_state_ != COMPLETED)
    Cancel();

  DCHECK(!manifest_url_request_);
  DCHECK(pending_url_fetches_.empty());
  DCHECK(!inprogress_cache_);
  DCHECK(pending_master_entries_.empty());
  DCHECK(master_entry_fetches_.empty());

  if (group_)
    group_->SetUpdateStatus(AppCacheGroup::IDLE);

  policy_callback_->Cancel();
}

UpdateJobInfo* AppCacheUpdateJob::GetUpdateJobInfo(URLRequest* request) {
  return static_cast<UpdateJobInfo*>(request->GetUserData(this));
}

void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
                                    const GURL& new_master_resource) {
  DCHECK(group_->update_job() == this);
  DCHECK(!group_->is_obsolete());

  bool is_new_pending_master_entry = false;
  if (!new_master_resource.is_empty()) {
    DCHECK(new_master_resource == host->pending_master_entry_url());
    DCHECK(!new_master_resource.has_ref());
    DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());

    // Cannot add more to this update if already terminating.
    if (IsTerminating()) {
      group_->QueueUpdate(host, new_master_resource);
      return;
    }

    std::pair<PendingMasters::iterator, bool> ret =
        pending_master_entries_.insert(
            PendingMasters::value_type(new_master_resource, PendingHosts()));
    is_new_pending_master_entry = ret.second;
    ret.first->second.push_back(host);
    host->AddObserver(this);
  }

  // Notify host (if any) if already checking or downloading.
  AppCacheGroup::UpdateStatus update_status = group_->update_status();
  if (update_status == AppCacheGroup::CHECKING ||
      update_status == AppCacheGroup::DOWNLOADING) {
    if (host) {
      NotifySingleHost(host, CHECKING_EVENT);
      if (update_status == AppCacheGroup::DOWNLOADING)
        NotifySingleHost(host, DOWNLOADING_EVENT);

      // Add to fetch list or an existing entry if already fetched.
      if (!new_master_resource.is_empty()) {
        AddMasterEntryToFetchList(host, new_master_resource,
                                  is_new_pending_master_entry);
      }
    }
    return;
  }

  // Begin update process for the group.
  group_->SetUpdateStatus(AppCacheGroup::CHECKING);
  if (group_->HasCache()) {
    update_type_ = UPGRADE_ATTEMPT;
    NotifyAllAssociatedHosts(CHECKING_EVENT);
  } else {
    update_type_ = CACHE_ATTEMPT;
    DCHECK(host);
    NotifySingleHost(host, CHECKING_EVENT);
  }

  if (!new_master_resource.is_empty()) {
    AddMasterEntryToFetchList(host, new_master_resource,
                              is_new_pending_master_entry);
  }

  if (update_type_ == CACHE_ATTEMPT)
    CheckPolicy();
  else
    FetchManifest(true);
}

void AppCacheUpdateJob::CheckPolicy() {
  int rv = net::OK;
  policy_callback_->AddRef();  // Balanced in OnPolicyCheckComplete.
  AppCachePolicy* policy = service_->appcache_policy();
  if (policy) {
    rv = policy->CanCreateAppCache(manifest_url_, policy_callback_);
    if (rv == net::ERR_IO_PENDING)
      return;
  }
  OnPolicyCheckComplete(rv);
}

void AppCacheUpdateJob::OnPolicyCheckComplete(int rv) {
  policy_callback_->Release();  // Balanced in CheckPolicy.
  if (rv == net::OK) {
    FetchManifest(true);
    return;
  }

  group_->NotifyContentBlocked();

  const char* kErrorMessage =
      "Cache creation was blocked by the content policy";
  MessageLoop::current()->PostTask(FROM_HERE,
      method_factory_.NewRunnableMethod(
          &AppCacheUpdateJob::HandleCacheFailure,
          kErrorMessage));
}

void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
  // 6.9.4 cache failure steps 2-8.
  DCHECK(internal_state_ != CACHE_FAILURE);
  DCHECK(!error_message.empty());
  internal_state_ = CACHE_FAILURE;
  CancelAllUrlFetches();
  CancelAllMasterEntryFetches(error_message);
  NotifyAllError(error_message);
  DiscardInprogressCache();
  internal_state_ = COMPLETED;
  DeleteSoon();  // To unwind the stack prior to deletion.
}

void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
  DCHECK(!manifest_url_request_);
  manifest_url_request_ = new URLRequest(manifest_url_, this);
  UpdateJobInfo::RequestType fetch_type = is_first_fetch ?
      UpdateJobInfo::MANIFEST_FETCH : UpdateJobInfo::MANIFEST_REFETCH;
  manifest_url_request_->SetUserData(this, new UpdateJobInfo(fetch_type));
  manifest_url_request_->set_context(service_->request_context());
  manifest_url_request_->set_load_flags(
      manifest_url_request_->load_flags() | net::LOAD_DISABLE_INTERCEPT);

  // Add any necessary Http headers before sending fetch request.
  if (is_first_fetch) {
    AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
        group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
    if (entry) {
      // Asynchronously load response info for manifest from newest cache.
      service_->storage()->LoadResponseInfo(manifest_url_,
                                            entry->response_id(), this);
    } else {
      manifest_url_request_->Start();
    }
  } else {
    DCHECK(internal_state_ == REFETCH_MANIFEST);
    DCHECK(manifest_response_info_.get());
    AddConditionalHeaders(manifest_url_request_,
                          manifest_response_info_.get());
    manifest_url_request_->Start();
  }
}

void AppCacheUpdateJob::AddConditionalHeaders(
    URLRequest* request, const net::HttpResponseInfo* info) {
  DCHECK(request && info);
  net::HttpRequestHeaders extra_headers;

  // Add If-Modified-Since header if response info has Last-Modified header.
  const std::string last_modified = "Last-Modified";
  std::string last_modified_value;
  info->headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
  if (!last_modified_value.empty()) {
    extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
                            last_modified_value);
  }

  // Add If-None-Match header if resposne info has ETag header.
  const std::string etag = "ETag";
  std::string etag_value;
  info->headers->EnumerateHeader(NULL, etag, &etag_value);
  if (!etag_value.empty()) {
    extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
                            etag_value);
  }
  if (!extra_headers.IsEmpty())
    request->SetExtraRequestHeaders(extra_headers);
}

void AppCacheUpdateJob::OnResponseStarted(URLRequest *request) {
  if (request->status().is_success() &&
      (request->GetResponseCode() / 100) == 2) {
    // Write response info to storage for URL fetches. Wait for async write
    // completion before reading any response data.
    UpdateJobInfo* info = GetUpdateJobInfo(request);
    if (info->type_ == UpdateJobInfo::URL_FETCH ||
        info->type_ == UpdateJobInfo::MASTER_ENTRY_FETCH) {
      info->SetUpResponseWriter(
          service_->storage()->CreateResponseWriter(manifest_url_),
          this, request);
      stored_response_ids_.push_back(info->response_writer_->response_id());
      scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
          new HttpResponseInfoIOBuffer(
              new net::HttpResponseInfo(request->response_info()));
      info->response_writer_->WriteInfo(io_buffer, &info->write_callback_);
    } else {
      ReadResponseData(request);
    }
  } else {
    OnResponseCompleted(request);
  }
}

void AppCacheUpdateJob::ReadResponseData(URLRequest* request) {
  if (internal_state_ == CACHE_FAILURE || internal_state_ == CANCELLED ||
      internal_state_ == COMPLETED) {
    return;
  }

  int bytes_read = 0;
  UpdateJobInfo* info = GetUpdateJobInfo(request);
  request->Read(info->buffer_, kBufferSize, &bytes_read);
  OnReadCompleted(request, bytes_read);
}

void AppCacheUpdateJob::OnReadCompleted(URLRequest* request, int bytes_read) {
  bool data_consumed = true;
  if (request->status().is_success() && bytes_read > 0) {
    UpdateJobInfo* info = GetUpdateJobInfo(request);
    data_consumed = ConsumeResponseData(request, info, bytes_read);
    if (data_consumed) {
      bytes_read = 0;
      while (request->Read(info->buffer_, kBufferSize, &bytes_read)) {
        if (bytes_read > 0) {
          data_consumed = ConsumeResponseData(request, info, bytes_read);
          if (!data_consumed)
            break;  // wait for async data processing, then read more
        } else {
          break;
        }
      }
    }
  }

  if (data_consumed && !request->status().is_io_pending())
    OnResponseCompleted(request);
}

bool AppCacheUpdateJob::ConsumeResponseData(URLRequest* request,
                                            UpdateJobInfo* info,
                                            int bytes_read) {
  DCHECK_GT(bytes_read, 0);
  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
      manifest_data_.append(info->buffer_->data(), bytes_read);
      break;
    case UpdateJobInfo::URL_FETCH:
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      DCHECK(info->response_writer_.get());
      info->response_writer_->WriteData(info->buffer_, bytes_read,
                                        &info->write_callback_);
      return false;  // wait for async write completion to continue reading
    case UpdateJobInfo::MANIFEST_REFETCH:
      manifest_refetch_data_.append(info->buffer_->data(), bytes_read);
      break;
    default:
      NOTREACHED();
  }
  return true;
}

void AppCacheUpdateJob::OnWriteResponseComplete(int result,
                                                URLRequest* request,
                                                UpdateJobInfo* info) {
  if (result < 0) {
    request->Cancel();
    OnResponseCompleted(request);
    return;
  }

  ReadResponseData(request);
}

void AppCacheUpdateJob::OnReceivedRedirect(URLRequest* request,
                                           const GURL& new_url,
                                           bool* defer_redirect) {
  // Redirect is not allowed by the update process.
  request->Cancel();
  OnResponseCompleted(request);
}

void AppCacheUpdateJob::OnResponseCompleted(URLRequest* request) {
  // Retry for 503s where retry-after is 0.
  if (request->status().is_success() &&
      request->GetResponseCode() == 503 &&
      RetryRequest(request)) {
    return;
  }

  UpdateJobInfo* info = GetUpdateJobInfo(request);
  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
      HandleManifestFetchCompleted(request);
      break;
    case UpdateJobInfo::URL_FETCH:
      HandleUrlFetchCompleted(request);
      break;
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      HandleMasterEntryFetchCompleted(request);
      break;
    case UpdateJobInfo::MANIFEST_REFETCH:
      HandleManifestRefetchCompleted(request);
      break;
    default:
      NOTREACHED();
  }

  delete request;
}

bool AppCacheUpdateJob::RetryRequest(URLRequest* request) {
  UpdateJobInfo* info = GetUpdateJobInfo(request);
  if (info->retry_503_attempts_ >= kMax503Retries) {
    return false;
  }

  if (!request->response_headers()->HasHeaderValue("retry-after", "0"))
    return false;

  const GURL& url = request->original_url();
  URLRequest* retry = new URLRequest(url, this);
  UpdateJobInfo* retry_info = new UpdateJobInfo(info->type_);
  retry_info->retry_503_attempts_ = info->retry_503_attempts_ + 1;
  retry_info->existing_entry_ = info->existing_entry_;
  retry->SetUserData(this, retry_info);
  retry->set_context(request->context());
  retry->set_load_flags(request->load_flags());

  switch (info->type_) {
    case UpdateJobInfo::MANIFEST_FETCH:
    case UpdateJobInfo::MANIFEST_REFETCH:
      manifest_url_request_ = retry;
      manifest_data_.clear();
      break;
    case UpdateJobInfo::URL_FETCH:
      pending_url_fetches_.erase(url);
      pending_url_fetches_.insert(PendingUrlFetches::value_type(url, retry));
      break;
    case UpdateJobInfo::MASTER_ENTRY_FETCH:
      master_entry_fetches_.erase(url);
      master_entry_fetches_.insert(PendingUrlFetches::value_type(url, retry));
      break;
    default:
      NOTREACHED();
  }

  retry->Start();

  delete request;
  return true;
}

void AppCacheUpdateJob::HandleManifestFetchCompleted(URLRequest* request) {
  DCHECK(internal_state_ == FETCH_MANIFEST);
  manifest_url_request_ = NULL;

  int response_code = -1;
  std::string mime_type;
  bool is_valid_response_code = false;
  bool is_valid_mime_type = false;
  if (request->status().is_success()) {
    response_code = request->GetResponseCode();
    is_valid_response_code = (response_code / 100 == 2);
    request->GetMimeType(&mime_type);
    is_valid_mime_type = (mime_type == kManifestMimeType);
  }

  if (is_valid_response_code && is_valid_mime_type) {
    manifest_response_info_.reset(
        new net::HttpResponseInfo(request->response_info()));
    if (update_type_ == UPGRADE_ATTEMPT)
      CheckIfManifestChanged();  // continues asynchronously
    else
      ContinueHandleManifestFetchCompleted(true);
  } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
    ContinueHandleManifestFetchCompleted(false);
  } else if (response_code == 404 || response_code == 410) {
    service_->storage()->MakeGroupObsolete(group_, this);  // async
  } else {
    std::string message;
    if (!is_valid_response_code) {
      const char* kFormatString = "Manifest fetch failed (%d) %s";
      message = StringPrintf(kFormatString, response_code,
                             manifest_url_.spec().c_str());
    } else {
      DCHECK(!is_valid_mime_type);
      const char* kFormatString = "Invalid manifest mime type (%s) %s";
      message = StringPrintf(kFormatString, mime_type.c_str(),
                             manifest_url_.spec().c_str());
    }
    HandleCacheFailure(message);
  }
}

void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
                                            bool success) {
  DCHECK(master_entry_fetches_.empty());
  CancelAllMasterEntryFetches("The group has been made obsolete");
  if (success) {
    DCHECK(group->is_obsolete());
    NotifyAllAssociatedHosts(OBSOLETE_EVENT);
    internal_state_ = COMPLETED;
    MaybeCompleteUpdate();
  } else {
    // Treat failure to mark group obsolete as a cache failure.
    HandleCacheFailure("Failed to make the group as obsolete");
  }
}

void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
  DCHECK(internal_state_ == FETCH_MANIFEST);

  if (!changed) {
    DCHECK(update_type_ == UPGRADE_ATTEMPT);
    internal_state_ = NO_UPDATE;
{'fraction_non_alphanumeric': 0.10699514010960604, 'fraction_numerical': 0.003929273084479371, 'mean_word_length': 3.6384892086330933, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 245, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5775030', 'n_tokens_mistral': 11786, 'n_tokens_neox': 11517, 'n_words': 2207}