<commit_before>// Copyright 2010 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: <EMAIL> (<NAME>)
//         <EMAIL> (<NAME>)

#include "net/instaweb/apache/instaweb_handler.h"

#include <cstddef>
#include <set>
#include <vector>

#include "net/instaweb/apache/apache_config.h"
#include "net/instaweb/apache/apache_message_handler.h"
#include "net/instaweb/apache/apache_request_context.h"
#include "net/instaweb/apache/apache_rewrite_driver_factory.h"
#include "net/instaweb/apache/apache_server_context.h"
#include "net/instaweb/apache/apache_slurp.h"
#include "net/instaweb/apache/apache_writer.h"
#include "net/instaweb/apache/apr_timer.h"
#include "net/instaweb/apache/header_util.h"
#include "net/instaweb/apache/in_place_resource_recorder.h"
#include "net/instaweb/apache/instaweb_context.h"
#include "net/instaweb/apache/mod_instaweb.h"
#include "net/instaweb/automatic/public/proxy_fetch.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/meta_data.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/http/public/request_headers.h"
#include "net/instaweb/http/public/response_headers.h"
#include "net/instaweb/http/public/sync_fetcher_adapter_callback.h"
#include "net/instaweb/public/global_constants.h"
#include "net/instaweb/rewriter/public/domain_lawyer.h"
#include "net/instaweb/rewriter/public/resource_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/util/public/abstract_mutex.h"
#include "net/instaweb/util/public/basictypes.h"
#include "net/instaweb/util/public/condvar.h"
#include "net/instaweb/util/public/escaping.h"
#include "net/instaweb/util/public/google_url.h"
#include "net/instaweb/util/public/message_handler.h"
#include "net/instaweb/util/public/query_params.h"
#include "net/instaweb/util/public/ref_counted_ptr.h"
#include "net/instaweb/util/public/scoped_ptr.h"
#include "net/instaweb/util/public/statistics.h"
#include "net/instaweb/util/public/string.h"
#include "net/instaweb/util/public/string_util.h"
#include "net/instaweb/util/public/string_writer.h"
#include "net/instaweb/util/public/thread_system.h"
#include "net/instaweb/util/public/timer.h"
#include "net/instaweb/util/public/writer.h"

#include "http_config.h"
#include "http_core.h"
#include "http_protocol.h"
#include "http_request.h"
#include "net/instaweb/apache/apache_logging_includes.h"

namespace net_instaweb {

extern const char* JS_mod_pagespeed_console_js;
extern const char* CSS_mod_pagespeed_console_css;
extern const char* HTML_mod_pagespeed_console_body;

namespace {

const char kStatisticsHandler[] = "mod_pagespeed_statistics";
const char kConsoleHandler[] = "mod_pagespeed_console";
const char kGlobalStatisticsHandler[] = "mod_pagespeed_global_statistics";
const char kMessageHandler[] = "mod_pagespeed_message";
const char kBeaconHandler[] = "mod_pagespeed_beacon";
const char kLogRequestHeadersHandler[] = "mod_pagespeed_log_request_headers";
const char kGenerateResponseWithOptionsHandler[] =
    "mod_pagespeed_response_options_handler";
const char kResourceUrlNote[] = "mod_pagespeed_resource";
const char kResourceUrlNo[] = "<NO>";
const char kResourceUrlYes[] = "<YES>";

// Set the maximum size we allow for processing a POST body. The limit of 128k
// is based on a best guess for the maximum size of beacons required for
// critical CSS.
// TODO(jud): Factor this out, potentially into an option, and pass the value to
// any filters using beacons with POST requests (CriticalImagesBeaconFilter for
// instance).
const size_t kMaxPostSizeBytes = 131072;

// Links an apache request_rec* to an AsyncFetch, adding the ability to
// block based on a condition variable.
//
// TODO(jmarantz): consider refactoring to share code with ProxyFetch, though
// this implementation does not imply any rewriting; it's just a caching
// proxy.
class ApacheProxyFetch : public AsyncFetchUsingWriter {
 public:
  ApacheProxyFetch(const GoogleString& mapped_url, ThreadSystem* thread_system,
                   RewriteDriver* driver, request_rec* request)
      : AsyncFetchUsingWriter(driver->request_context(), &apache_writer_),
        mapped_url_(mapped_url),
        apache_writer_(request),
        driver_(driver),
        mutex_(thread_system->NewMutex()),
        condvar_(mutex_->NewCondvar()),
        done_(false),
        handle_error_(true),
        status_ok_(false) {
    // We are proxying content, and the caching in the http configuration
    // should not apply; we want to use the caching from the proxy.
    apache_writer_.set_disable_downstream_header_filters(true);
    apache_writer_.set_strip_cookies(true);
    ApacheRequestToRequestHeaders(*request, request_headers());
    request_headers()->RemoveAll(HttpAttributes::kCookie);
    request_headers()->RemoveAll(HttpAttributes::kCookie2);
  }

  virtual ~ApacheProxyFetch() {
  }

  // When used for in-place resource optimization in mod_pagespeed, we have
  // disabled fetching resources that are not in cache, otherwise we may wind
  // up doing a loopback fetch to the same Apache server.  So the
  // CacheUrlAsyncFetcher will return a 501 or 404 but we do not want to
  // send that to the client.  So for ipro we suppress resporting errors
  // in this flow.
  //
  // TODO(jmarantz): consider allowing serf fetches in ipro when running as
  // a reverse-proxy.
  void set_handle_error(bool x) { handle_error_ = x; }

  virtual void HandleHeadersComplete() {
    int status_code = response_headers()->status_code();
    status_ok_ = (status_code != 0) && (status_code < 400);
    if (handle_error_ || status_ok_) {
      // TODO(sligocki): Add X-Mod-Pagespeed header.
      if (content_length_known()) {
        apache_writer_.set_content_length(content_length());
      }
      apache_writer_.OutputHeaders(response_headers());
    }
  }

  virtual void HandleDone(bool success) {
    ScopedMutex lock(mutex_.get());
    done_ = true;
    if (status_ok_ && !success) {
      driver_->message_handler()->Message(
          kWarning,
          "Response for url %s issued with status %d %s but "
          "failed to complete.",
          mapped_url_.c_str(), response_headers()->status_code(),
          response_headers()->reason_phrase());
    }
    condvar_->Signal();
  }

  // Blocks indefinitely waiting for the proxy fetch to complete.
  // Every 'blocking_fetch_timeout_ms', log a message so that if
  // we get stuck there's noise in the logs, but we don't expect this
  // to happen because underlying fetch/cache timeouts should fire.
  //
  // Note that enforcing a timeout in this function makes debugging
  // difficult.
  void Wait() {
    int64 timeout_ms = driver_->options()->blocking_fetch_timeout_ms();
    ServerContext* server_context = driver_->server_context();
    MessageHandler* handler = server_context->message_handler();
    Timer* timer = server_context->timer();
    int64 start_ms = timer->NowMs();
    {
      ScopedMutex lock(mutex_.get());
      while (!done_) {
        condvar_->TimedWait(timeout_ms);
        if (!done_) {
          int64 elapsed_ms = timer->NowMs() - start_ms;
          handler->Message(
              kWarning, "Waiting for in-place ProxyFetch on URL %s for %g sec",
              mapped_url_.c_str(), elapsed_ms / 1000.0);
        }
      }
    }
  }

  // Returning true here tells Serf to use the threaded fetcher.  We can
  // also consider returning false here and polling the fetcher.
  //
  // TODO(jmarantz): delete the non-threaded fetcher support in Serf.
  virtual bool EnableThreaded() const { return true; }

  bool status_ok() const { return status_ok_; }

 private:
  GoogleString mapped_url_;
  ApacheWriter apache_writer_;
  RewriteDriver* driver_;
  scoped_ptr<ThreadSystem::CondvarCapableMutex> mutex_;
  scoped_ptr<ThreadSystem::Condvar> condvar_;
  bool done_;
  bool handle_error_;
  bool status_ok_;

  DISALLOW_COPY_AND_ASSIGN(ApacheProxyFetch);
};

bool IsCompressibleContentType(const char* content_type) {
  if (content_type == NULL) {
    return false;
  }
  GoogleString type = content_type;
  size_t separator_idx = type.find(";");
  if (separator_idx != GoogleString::npos) {
    type.erase(separator_idx);
  }

  bool res = false;
  if (type.find("text/") == 0) {
    res = true;
  } else if (type.find("application/") == 0) {
    if (type.find("javascript") != type.npos ||
        type.find("json") != type.npos ||
        type.find("ecmascript") != type.npos ||
        type == "application/livescript" ||
        type == "application/js" ||
        type == "application/jscript" ||
        type == "application/x-js" ||
        type == "application/xhtml+xml" ||
        type == "application/xml") {
      res = true;
    }
  }

  return res;
}

// Default handler when the file is not found
void instaweb_404_handler(const GoogleString& url, request_rec* request) {
  request->status = HTTP_NOT_FOUND;
  ap_set_content_type(request, "text/html; charset=utf-8");
  ap_rputs("<html><head><title>Not Found</title></head>", request);
  ap_rputs("<body><h1>Apache server with mod_pagespeed</h1>OK", request);
  ap_rputs("<hr>NOT FOUND:", request);
  ap_rputs(url.c_str(), request);
  ap_rputs("</body></html>", request);
}

void send_out_headers_and_body(request_rec* request,
                               const ResponseHeaders& response_headers,
                               const GoogleString& output) {
  // We always disable downstream header filters when sending out
  // pagespeed resources, since we've captured them in the origin fetch.
  ResponseHeadersToApacheRequest(response_headers,
                                 true,  // Disable downstream header filters.
                                 request);
  if (response_headers.status_code() == HttpStatus::kOK &&
      IsCompressibleContentType(request->content_type)) {
    // Make sure compression is enabled for this response.
    ap_add_output_filter("DEFLATE", NULL, request, request->connection);
  }

  // Recompute the content-length, because the content may have changed.
  ap_set_content_length(request, output.size());
  // Send the body
  ap_rwrite(output.c_str(), output.size(), request);
}

// Evaluate custom_options based upon global_options, directory-specific
// options and query-param/request-header options. Returns computed
// custom options (or NULL if global_options should be used).
//
// Caller takes ownership of options.
RewriteOptions* get_custom_options(ApacheServerContext* server_context,
                                   request_rec* request,
                                   GoogleUrl* gurl,
                                   RequestHeaders* request_headers,
                                   RewriteOptions* global_options) {
  // Set directory specific options.  These will be the options for the
  // directory the resource is in, which under some configurations will be
  // different from the options for the directory that the referencing html is
  // in.  This can lead to us using different options here when regenerating
  // the resource than would be used if the resource were generated as part of
  // a rewrite kicked off by a request for the referencing html file.  This is
  // hard to fix, so instead we're documenting that you must make sure the
  // configuration for your resources matches the configuration for your html
  // files.
  RewriteOptions* custom_options = NULL;
  ApacheConfig* directory_options = static_cast<ApacheConfig*>
      ap_get_module_config(request->per_dir_config, &pagespeed_module);
  if ((directory_options != NULL) && directory_options->modified()) {
    custom_options = server_context->apache_factory()->NewRewriteOptions();
    custom_options->Merge(*global_options);
    custom_options->Merge(*directory_options);
  }

  // TODO(sligocki): Move inside PSOL.
  // Merge in query-param or header-based options.
  // Note: We do not generally get response headers in the resource flow,
  // so NULL is passed in instead.
  ServerContext::OptionsBoolPair query_options_success =
      server_context->GetQueryOptions(gurl, request_headers, NULL);
  if (!query_options_success.second) {
    server_context->message_handler()->Message(
        kWarning, "Invalid ModPagespeed query params or headers for "
        "request %s. Serving with default options.", gurl->spec_c_str());
  }
  if (query_options_success.first != NULL) {
    if (custom_options == NULL) {
      custom_options = server_context->apache_factory()->NewRewriteOptions();
      custom_options->Merge(*global_options);
    }
    custom_options->Merge(*query_options_success.first);
    delete query_options_success.first;
    // Don't run any experiments if we're handling a customized request.
    custom_options->set_running_furious_experiment(false);
  }

  return custom_options;
}

// Handle url as .pagespeed. rewritten resource.
void handle_as_pagespeed_resource(const RequestContextPtr& request_context,
                                  GoogleUrl* gurl,
                                  const GoogleString& url,
                                  RewriteOptions* custom_options,
                                  ApacheServerContext* server_context,
                                  RequestHeaders* request_headers,
                                  request_rec* request) {
  RewriteDriver* driver = ResourceFetch::GetDriver(
      *gurl, custom_options, server_context, request_context);

  MessageHandler* message_handler = server_context->message_handler();
  message_handler->Message(kInfo, "Fetching resource %s...", url.c_str());

  GoogleString output;  // TODO(jmarantz): Quit buffering resource output.
  StringWriter writer(&output);

  SyncFetcherAdapterCallback* callback = new SyncFetcherAdapterCallback(
      server_context->thread_system(), &writer, request_context);
  callback->SetRequestHeadersTakingOwnership(request_headers);

  if (ResourceFetch::BlockingFetch(*gurl, server_context, driver, callback)) {
    ResponseHeaders* response_headers = callback->response_headers();
    // TODO(sligocki): Check that this is already done in ResourceFetch
    // and remove redundant setting here.
    response_headers->SetDate(server_context->timer()->NowMs());
    // ResourceFetch adds X-Page-Speed header, old mod_pagespeed code
    // did not. For now, we remove that header for consistency.
    // TODO(sligocki): Consistently use X- headers in MPS and PSOL.
    // I think it would be good to change X-Mod-Pagespeed -> X-Page-Speed
    // and use that for all HTML and resource requests.
    response_headers->RemoveAll(kPageSpeedHeader);
    message_handler->Message(kInfo, "Fetch succeeded for %s, status=%d",
                             url.c_str(), response_headers->status_code());
    send_out_headers_and_body(request, *response_headers, output);
  } else {
    RewriteStats* stats = server_context->rewrite_stats();
    stats->resource_404_count()->Add(1);
    instaweb_404_handler(url, request);
  }

  callback->Release();
}

// Handle url with In Place Resource Optimization (IPRO) flow.
bool handle_as_in_place(const RequestContextPtr& request_context,
                        GoogleUrl* gurl,
                        const GoogleString& url,
                        RewriteOptions* custom_options,
                        ApacheServerContext* server_context,
                        RequestHeaders* owned_headers,
                        request_rec* request) {
  scoped_ptr<RequestHeaders> request_headers(owned_headers);
  bool handled = false;

  RewriteDriver* driver = ResourceFetch::GetDriver(
      *gurl, custom_options, server_context, request_context);

  MessageHandler* message_handler = server_context->message_handler();
  message_handler->Message(kInfo, "Trying to serve rewritten resource "
                           "in-place: %s", url.c_str());

  ApacheProxyFetch fetch(
      url, server_context->thread_system(), driver, request);
  fetch.set_handle_error(false);
  driver->FetchInPlaceResource(*gurl, false /* proxy_mode */, &fetch);

  fetch.Wait();
  if (fetch.status_ok()) {
    server_context->rewrite_stats()->ipro_served()->Add(1);
    message_handler->Message(kInfo, "Serving rewritten resource in-place: %s",
                             url.c_str());
    handled = true;
  } else if (fetch.response_headers()->status_code() ==
             CacheUrlAsyncFetcher::kNotInCacheStatus) {
    server_context->rewrite_stats()->ipro_not_in_cache()->Add(1);
    message_handler->Message(kInfo, "Could not rewrite resource in-place "
                             "because URL is not in cache: %s",
                             url.c_str());
    // This URL was not found in cache (neither the input resource nor
    // a ResourceNotCacheable entry) so we need to get it into cache
    // (or at least a note that it cannot be cached stored there).
    // We do that using an Apache output filter.
    InPlaceResourceRecorder* recorder = new InPlaceResourceRecorder(
        url, request_headers.release(), driver->options()->respect_vary(),
        server_context->http_cache(), server_context->statistics(),
        message_handler);
    ap_add_output_filter(kModPagespeedInPlaceFilterName, recorder,
                         request, request->connection);
    ap_add_output_filter(kModPagespeedInPlaceCheckHeadersName, recorder,
                         request, request->connection);
  } else {
    server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
    message_handler->Message(kInfo, "Could not rewrite resource in-place: %s",
                             url.c_str());
  }
  driver->Cleanup();

  return handled;
}

bool handle_as_proxy(ApacheServerContext* server_context,
                     request_rec* request,
                     const RequestContextPtr& request_context,
                     GoogleUrl* gurl,
                     RewriteOptions* options,
                     scoped_ptr<RewriteOptions>* custom_options) {
  bool handled = false;
  // Consider Issue 609: proxying an external CSS file via
  // ModPagespeedMapProxyDomain, and the CSS file makes reference to
  // a font file, which mod_pagespeed does not know anything about, and
  // does not know how to absolutify.  We need to handle the request for
  // the external font file here, even if IPRO (in place resource
  // optimization) is off.
  bool is_proxy = false;
  GoogleString mapped_url;
  if (options->domain_lawyer()->MapOriginUrl(*gurl, &mapped_url, &is_proxy) &&
      is_proxy) {
    RewriteDriver* driver = ResourceFetch::GetDriver(
        *gurl, custom_options->release(), server_context, request_context);
    ApacheProxyFetch apache_proxy_fetch(
        mapped_url, server_context->thread_system(), driver, request);
    server_context->proxy_fetch_factory()->StartNewProxyFetch(
        mapped_url, &apache_proxy_fetch, driver, NULL, NULL);
    apache_proxy_fetch.Wait();
    handled = true;
  }

  return handled;
}

// Determines whether the url can be handled as a mod_pagespeed or in-place
// optimized resource, and handles it, returning true.  Success status is
// written to the status code in the response headers.
bool handle_as_resource(ApacheServerContext* server_context,
                        request_rec* request,
                        GoogleUrl* gurl,
                        const GoogleString& url) {
  if (!gurl->is_valid()) {
    return false;
  }

  // Flushing the cache mutates global_options, so this has to happen before we
  // construct the options that we use to decide whether IPRO is enabled.
  server_context->FlushCacheIfNecessary();

  ApacheRequestContext* apache_request_context = new ApacheRequestContext(
      server_context->thread_system()->NewMutex(), request);
  apache_request_context->set_url(url);
  RequestContextPtr request_context(apache_request_context);
  bool using_spdy = request_context->using_spdy();
  RewriteOptions* global_options = server_context->global_options();
  if (using_spdy && (server_context->SpdyConfig() != NULL)) {
    global_options = server_context->SpdyConfig();
  }

  scoped_ptr<RequestHeaders> request_headers(new RequestHeaders);
  // Filter limited request headers into backend fetch.
  // TODO(sligocki): Put this filtering in ResourceFetch and instead use:
  // ApacheRequestToRequestHeaders(*request, request_headers.get());
  for (int i = 0, n = arraysize(RewriteDriver::kPassThroughRequestAttributes);
       i < n; ++i) {
    const char* value = apr_table_get(
        request->headers_in,
        RewriteDriver::kPassThroughRequestAttributes[i]);
    if (value != NULL) {
      request_headers->Add(
          RewriteDriver::kPassThroughRequestAttributes[i], value);
    }
  }

  scoped_ptr<RewriteOptions> custom_options(get_custom_options(
      server_context, request, gurl, request_headers.get(), global_options));

  RewriteOptions* options = custom_options.get();  // Options for this request.
  if (custom_options.get() == NULL) {
    options = global_options;
  }

  // Finally, do the actual handling.
  bool handled = false;
  if (server_context->IsPagespeedResource(*gurl)) {
    handled = true;
    handle_as_pagespeed_resource(request_context, gurl, url,
                                 custom_options.release(), server_context,
                                 request_headers.release(), request);
  } else if (handle_as_proxy(server_context, request, request_context, gurl,
                             options, &custom_options)) {
    handled = true;
  } else if (options->in_place_rewriting_enabled() && options->enabled() &&
             options->IsAllowed(url)) {
    handled = handle_as_in_place(request_context, gurl, url,
                                 custom_options.release(), server_context,
                                 request_headers.release(), request);
  }

  return handled;
}

// Write response headers and send out headers and output, including the option
//     for a custom Content-Type.
void write_handler_response(const StringPiece& output,
                            request_rec* request,
                            ContentType content_type,
                            const StringPiece& cache_control) {
  ResponseHeaders response_headers;
  response_headers.SetStatusAndReason(HttpStatus::kOK);
  response_headers.set_major_version(1);
  response_headers.set_minor_version(1);

  response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
  AprTimer timer;
  int64 now_ms = timer.NowMs();
  response_headers.SetDate(now_ms);
  response_headers.SetLastModified(now_ms);
  response_headers.Add(HttpAttributes::kCacheControl, cache_control);
  send_out_headers_and_body(request, response_headers, output.as_string());
}

void write_handler_response(const StringPiece& output,
                            request_rec* request,
                            ContentType content_type) {
  write_handler_response(output, request, kContentTypeHtml,
                         HttpAttributes::kNoCache);
}

void write_handler_response(const StringPiece& output, request_rec* request) {
  write_handler_response(output, request, kContentTypeHtml);
}

// Returns request URL if it was a .pagespeed. rewritten resource URL.
// Otherwise returns NULL. Since other Apache modules can change request->uri,
// we stow the original request URL in a note. This method reads that note
// and thus should return the URL that the browser actually requested (rather
// than a mod_rewrite altered URL).
const char* get_instaweb_resource_url(request_rec* request,
                                      ApacheServerContext* server_context) {
  const char* resource = apr_table_get(request->notes, kResourceUrlNote);

  // If our translate_name hook, save_url_hook, failed to run because some
  // other module's translate_hook returned OK first, then run it now. The
  // main reason we try to do this early is to save our URL before mod_rewrite
  // mutates it.
  if (resource == NULL) {
    save_url_in_note(request, server_context);
    resource = apr_table_get(request->notes, kResourceUrlNote);
  }

  if (resource != NULL && strcmp(resource, kResourceUrlNo) == 0) {
    return NULL;
  }

  const char* url = apr_table_get(request->notes, kPagespeedOriginalUrl);
  return url;
}

// Used by log_request_headers for testing only.
struct HeaderLoggingData {
  HeaderLoggingData(StringWriter* writer_in, MessageHandler* handler_in)
      : writer(writer_in), handler(handler_in) {}
  StringWriter* writer;
  MessageHandler* handler;
};

// Helper function to support the LogRequestHeadersHandler.  Called once for
// each header to write header data in a form suitable for javascript inlining.
// Used only for tests.
int log_request_headers(void* logging_data,
                        const char* key, const char* value) {
  HeaderLoggingData* hld = static_cast<HeaderLoggingData*>(logging_data);
  StringWriter* writer = hld->writer;
  MessageHandler* handler = hld->handler;

  GoogleString escaped_key;
  GoogleString escaped_value;

  EscapeToJsStringLiteral(key, false, &escaped_key);
  EscapeToJsStringLiteral(value, false, &escaped_value);

  writer->Write("alert(\"", handler);
  writer->Write(escaped_key, handler);
  writer->Write("=", handler);
  writer->Write(escaped_value, handler);
  writer->Write("\");\n", handler);

  return 1;  // Continue iteration.
}

// Writes text wrapped in a <pre> block
void WritePre(StringPiece str, Writer* writer, MessageHandler* handler) {
  writer->Write("<pre>\n", handler);
  writer->Write(str, handler);
  writer->Write("</pre>\n", handler);
}

void instaweb_static_handler(request_rec* request,
                             ApacheServerContext* server_context) {
  StaticAssetManager* static_asset_manager =
      server_context->static_asset_manager();
  StringPiece request_uri_path = request->parsed_uri.path;
  // Strip out the common prefix url before sending to StaticAssetManager.
  StringPiece file_name =
      request_uri_path.substr(
          strlen(ApacheRewriteDriverFactory::kStaticAssetPrefix));
  StringPiece file_contents;
  StringPiece cache_header;
  ContentType content_type;
  if (static_asset_manager->GetAsset(
      file_name, &file_contents, &content_type, &cache_header)) {
    write_handler_response(file_contents, request, content_type, cache_header);
  } else {
    instaweb_404_handler(request->parsed_uri.path, request);
  }
}

apr_status_t instaweb_console_handler(
    request_rec* request, ApacheConfig* config,
    ApacheMessageHandler* message_handler) {
  GoogleString output;
  StringWriter writer(&output);
  writer.Write("<!DOCTYPE html>"
                "<title>mod_pagespeed console</title>",
               message_handler);
  writer.Write("<style>", message_handler);
  writer.Write(CSS_mod_pagespeed_console_css, message_handler);
  writer.Write("</style>", message_handler);
  writer.Write(HTML_mod_pagespeed_console_body, message_handler);
  writer.Write("<script>", message_handler);
  if (config->statistics_logging_charts_js().size() > 0 &&
      config->statistics_logging_charts_css().size() > 0) {
    writer.Write("var chartsOfflineJS = '", message_handler);
    writer.Write(config->statistics_logging_charts_js(), message_handler);
    writer.Write("';", message_handler);
    writer.Write("var chartsOfflineCSS = '", message_handler);
    writer.Write(config->statistics_logging_charts_css(), message_handler);
    writer.Write("';", message_handler);
  } else {
    if (config->statistics_logging_charts_js().size() > 0 ||
        config->statistics_logging_charts_css().size() > 0) {
      message_handler->Message(kWarning, "Using online Charts API.");
    }
    writer.Write("var chartsOfflineJS, chartsOfflineCSS;", message_handler);
  }
  writer.Write(JS_mod_pagespeed_console_js, message_handler);
  writer.Write("</script>", message_handler);
  write_handler_response(output, request);
  return OK;
}

apr_status_t instaweb_statistics_handler(
    request_rec* request, ApacheServerContext* server_context,
    ApacheRewriteDriverFactory* factory, MessageHandler* message_handler) {
  bool general_stats_request =
      (strcmp(request->handler, kStatisticsHandler) == 0);
  bool global_stats_request =
      (strcmp(request->handler, kGlobalStatisticsHandler) == 0);

  int64 start_time, end_time, granularity_ms;
  std::set<GoogleString> var_titles;
  std::set<GoogleString> hist_titles;
  if (general_stats_request && !factory->use_per_vhost_statistics()) {
    global_stats_request = true;
  }

  // Choose the correct statistics.
  Statistics* statistics = global_stats_request ?
      factory->statistics() : server_context->statistics();

  QueryParams params;
  params.Parse(request->args);

  // Parse various mode query params.
  bool print_normal_config = params.Has("config");
  bool print_spdy_config = params.Has("spdy_config");

  // JSON statistics handling is done only if we have a console logger.
  bool json = false;
  if (statistics->console_logger() != NULL) {
    // Default values for start_time, end_time, and granularity_ms in case the
    // query does not include these parameters.
    start_time = 0;
    end_time = statistics->console_logger()->timer()->NowMs();
    // Granularity is the difference in ms between data points. If it is not
    // specified by the query, the default value is 3000 ms, the same as the
    // default logging granularity.
    granularity_ms = 3000;
    for (int i = 0; i < params.size(); ++i) {
      const GoogleString value =
          (params.value(i) == NULL) ? "" : *params.value(i);
      const char* name = params.name(i);
      if (strcmp(name, "json") == 0) {
        json = true;
      } else if (strcmp(name, "start_time") == 0) {
        StringToInt64(value, &start_time);
      } else if (strcmp(name, "end_time") == 0) {
        StringToInt64(value, &end_time);
      } else if (strcmp(name, "var_titles") == 0) {
        std::vector<StringPiece> variable_names;
        SplitStringPieceToVector(value, ",", &variable_names, true);
        for (size_t i = 0; i < variable_names.size(); ++i) {
          var_titles.insert(variable_names[i].as_string());
        }
      } else if (strcmp(name, "hist_titles") == 0) {
        std::vector<StringPiece> histogram_names;
        SplitStringPieceToVector(value, ",", &histogram_names, true);
        for (size_t i = 0; i < histogram_names.size(); ++i) {
          // TODO(morlovich): Cleanup & publicize UrlToFileNameEncoder::Unescape
          // and use it here, instead of this GlobalReplaceSubstring hack.
          GoogleString name = histogram_names[i].as_string();
          GlobalReplaceSubstring("%20", " ", &(name));
          hist_titles.insert(name);
        }
      } else if (strcmp(name, "granularity") == 0) {
        StringToInt64(value, &granularity_ms);
      }
    }
  }
  GoogleString output;
  StringWriter writer(&output);
  if (json) {
    statistics->console_logger()->DumpJSON(var_titles, hist_titles,
                                            start_time, end_time,
                                            granularity_ms, &writer,
                                            message_handler);
  } else {
    // Generate some navigational links to the right to help
    // our users get to other modes.
    writer.Write(
        "<div style='float:right'>View "
        "<a href='?config'>Configuration</a>, "
        "<a href='?spdy_config'>SPDY Configuration</a>, "
        "<a href='?'>Statistics</a> "
        "(<a href='?memcached'>with memcached Stats</a>). "
        "</div>",
        message_handler);

    // Only print stats or configuration, not both.
    if (!print_normal_config && !print_spdy_config) {
      writer.Write(global_stats_request ?
                       "Global Statistics" : "VHost-Specific Statistics",
                   message_handler);

      // Write <pre></pre> for Dump to keep good format.
      writer.Write("<pre>", message_handler);
      statistics->Dump(&writer, message_handler);
      writer.Write("</pre>", message_handler);
      statistics->RenderHistograms(&writer, message_handler);

      int flags = SystemCaches::kDefaultStatFlags;
      if (global_stats_request) {
        flags |= SystemCaches::kGlobalView;
      }

      if (params.Has("memcached")) {
        flags |= SystemCaches::kIncludeMemcached;
      }

      GoogleString backend_stats;
      factory->caches()->PrintCacheStats(
          static_cast<SystemCaches::StatFlags>(flags), &backend_stats);
      if (!backend_stats.empty()) {
        WritePre(backend_stats, &writer, message_handler);
      }
    }

    if (print_normal_config) {
      writer.Write("Configuration:<br>", message_handler);
      WritePre(server_context->config()->OptionsToString(),
               &writer, message_handler);
    }

    if (print_spdy_config) {
      ApacheConfig* spdy_config = server_context->SpdyConfig();
      if (spdy_config == NULL) {
        writer.Write("SPDY-specific configuration missing, using default.",
                      message_handler);
      } else {
        writer.Write("SPDY-specific configuration:<br>", message_handler);
        WritePre(spdy_config->OptionsToString(), &writer, message_handler);
      }
    }
  }

  if (json) {
    write_handler_response(output, request, kContentTypeJson);
  } else {
    write_handler_response(output, request);
  }
  return OK;
}

// Copy the query params from a GET request into data. Return true if
// successful, otherwise, returns false and sets ret to the appropriate status.
bool parse_query_params_from_get(const request_rec* request, GoogleString* data,
                         apr_status_t* ret) {
  if (request->method_number != M_GET) {
    *ret = HTTP_METHOD_NOT_ALLOWED;
    return false;
  }

  // Add a dummy host (www.example.com) to the request URL to make it absolute
  // so that GoogleUrl can be used for parsing.
  GoogleUrl base("http://www.example.com");
  GoogleUrl url(base, request->unparsed_uri);

  if (!url.is_valid() || !url.has_query()) {
    *ret = HTTP_BAD_REQUEST;
    return false;
  }

  url.Query().CopyToString(data);
  return true;
}

// Read the body from a POST request into data. Return true if successful,
// otherwise, returns false and sets ret to the appropriate status.<commit_msg>Revert unintended rollback due to bad repo sync.<commit_after>// Copyright 2010 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: <EMAIL> (<NAME>)
//         <EMAIL> (<NAME>)

#include "net/instaweb/apache/instaweb_handler.h"

#include <cstddef>
#include <set>
#include <vector>

#include "net/instaweb/apache/apache_config.h"
#include "net/instaweb/apache/apache_message_handler.h"
#include "net/instaweb/apache/apache_request_context.h"
#include "net/instaweb/apache/apache_rewrite_driver_factory.h"
#include "net/instaweb/apache/apache_server_context.h"
#include "net/instaweb/apache/apache_slurp.h"
#include "net/instaweb/apache/apache_writer.h"
#include "net/instaweb/apache/apr_timer.h"
#include "net/instaweb/apache/header_util.h"
#include "net/instaweb/apache/in_place_resource_recorder.h"
#include "net/instaweb/apache/instaweb_context.h"
#include "net/instaweb/apache/mod_instaweb.h"
#include "net/instaweb/automatic/public/proxy_fetch.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/meta_data.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/http/public/request_headers.h"
#include "net/instaweb/http/public/response_headers.h"
#include "net/instaweb/http/public/sync_fetcher_adapter_callback.h"
#include "net/instaweb/public/global_constants.h"
#include "net/instaweb/rewriter/public/domain_lawyer.h"
#include "net/instaweb/rewriter/public/resource_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/util/public/abstract_mutex.h"
#include "net/instaweb/util/public/basictypes.h"
#include "net/instaweb/util/public/condvar.h"
#include "net/instaweb/util/public/escaping.h"
#include "net/instaweb/util/public/google_url.h"
#include "net/instaweb/util/public/message_handler.h"
#include "net/instaweb/util/public/query_params.h"
#include "net/instaweb/util/public/ref_counted_ptr.h"
#include "net/instaweb/util/public/scoped_ptr.h"
#include "net/instaweb/util/public/statistics.h"
#include "net/instaweb/util/public/string.h"
#include "net/instaweb/util/public/string_util.h"
#include "net/instaweb/util/public/string_writer.h"
#include "net/instaweb/util/public/thread_system.h"
#include "net/instaweb/util/public/timer.h"
#include "net/instaweb/util/public/writer.h"

#include "http_config.h"
#include "http_core.h"
#include "http_protocol.h"
#include "http_request.h"
#include "net/instaweb/apache/apache_logging_includes.h"

namespace net_instaweb {

extern const char* JS_mod_pagespeed_console_js;
extern const char* CSS_mod_pagespeed_console_css;
extern const char* HTML_mod_pagespeed_console_body;

namespace {

const char kStatisticsHandler[] = "mod_pagespeed_statistics";
const char kConsoleHandler[] = "mod_pagespeed_console";
const char kGlobalStatisticsHandler[] = "mod_pagespeed_global_statistics";
const char kMessageHandler[] = "mod_pagespeed_message";
const char kBeaconHandler[] = "mod_pagespeed_beacon";
const char kLogRequestHeadersHandler[] = "mod_pagespeed_log_request_headers";
const char kGenerateResponseWithOptionsHandler[] =
    "mod_pagespeed_response_options_handler";
const char kResourceUrlNote[] = "mod_pagespeed_resource";
const char kResourceUrlNo[] = "<NO>";
const char kResourceUrlYes[] = "<YES>";

// Set the maximum size we allow for processing a POST body. The limit of 128k
// is based on a best guess for the maximum size of beacons required for
// critical CSS.
// TODO(jud): Factor this out, potentially into an option, and pass the value to
// any filters using beacons with POST requests (CriticalImagesBeaconFilter for
// instance).
const size_t kMaxPostSizeBytes = 131072;

// Links an apache request_rec* to an AsyncFetch, adding the ability to
// block based on a condition variable.
//
// TODO(jmarantz): consider refactoring to share code with ProxyFetch, though
// this implementation does not imply any rewriting; it's just a caching
// proxy.
class ApacheProxyFetch : public AsyncFetchUsingWriter {
 public:
  ApacheProxyFetch(const GoogleString& mapped_url, ThreadSystem* thread_system,
                   RewriteDriver* driver, request_rec* request)
      : AsyncFetchUsingWriter(driver->request_context(), &apache_writer_),
        mapped_url_(mapped_url),
        apache_writer_(request),
        driver_(driver),
        mutex_(thread_system->NewMutex()),
        condvar_(mutex_->NewCondvar()),
        done_(false),
        handle_error_(true),
        status_ok_(false) {
    // We are proxying content, and the caching in the http configuration
    // should not apply; we want to use the caching from the proxy.
    apache_writer_.set_disable_downstream_header_filters(true);
    apache_writer_.set_strip_cookies(true);
    ApacheRequestToRequestHeaders(*request, request_headers());
    request_headers()->RemoveAll(HttpAttributes::kCookie);
    request_headers()->RemoveAll(HttpAttributes::kCookie2);
  }

  virtual ~ApacheProxyFetch() {
  }

  // When used for in-place resource optimization in mod_pagespeed, we have
  // disabled fetching resources that are not in cache, otherwise we may wind
  // up doing a loopback fetch to the same Apache server.  So the
  // CacheUrlAsyncFetcher will return a 501 or 404 but we do not want to
  // send that to the client.  So for ipro we suppress resporting errors
  // in this flow.
  //
  // TODO(jmarantz): consider allowing serf fetches in ipro when running as
  // a reverse-proxy.
  void set_handle_error(bool x) { handle_error_ = x; }

  virtual void HandleHeadersComplete() {
    int status_code = response_headers()->status_code();
    status_ok_ = (status_code != 0) && (status_code < 400);
    if (handle_error_ || status_ok_) {
      // TODO(sligocki): Add X-Mod-Pagespeed header.
      if (content_length_known()) {
        apache_writer_.set_content_length(content_length());
      }
      apache_writer_.OutputHeaders(response_headers());
    }
  }

  virtual void HandleDone(bool success) {
    ScopedMutex lock(mutex_.get());
    done_ = true;
    if (status_ok_ && !success) {
      driver_->message_handler()->Message(
          kWarning,
          "Response for url %s issued with status %d %s but "
          "failed to complete.",
          mapped_url_.c_str(), response_headers()->status_code(),
          response_headers()->reason_phrase());
    }
    condvar_->Signal();
  }

  // Blocks indefinitely waiting for the proxy fetch to complete.
  // Every 'blocking_fetch_timeout_ms', log a message so that if
  // we get stuck there's noise in the logs, but we don't expect this
  // to happen because underlying fetch/cache timeouts should fire.
  //
  // Note that enforcing a timeout in this function makes debugging
  // difficult.
  void Wait() {
    int64 timeout_ms = driver_->options()->blocking_fetch_timeout_ms();
    ServerContext* server_context = driver_->server_context();
    MessageHandler* handler = server_context->message_handler();
    Timer* timer = server_context->timer();
    int64 start_ms = timer->NowMs();
    {
      ScopedMutex lock(mutex_.get());
      while (!done_) {
        condvar_->TimedWait(timeout_ms);
        if (!done_) {
          int64 elapsed_ms = timer->NowMs() - start_ms;
          handler->Message(
              kWarning, "Waiting for in-place ProxyFetch on URL %s for %g sec",
              mapped_url_.c_str(), elapsed_ms / 1000.0);
        }
      }
    }
  }

  // Returning true here tells Serf to use the threaded fetcher.  We can
  // also consider returning false here and polling the fetcher.
  //
  // TODO(jmarantz): delete the non-threaded fetcher support in Serf.
  virtual bool EnableThreaded() const { return true; }

  bool status_ok() const { return status_ok_; }

 private:
  GoogleString mapped_url_;
  ApacheWriter apache_writer_;
  RewriteDriver* driver_;
  scoped_ptr<ThreadSystem::CondvarCapableMutex> mutex_;
  scoped_ptr<ThreadSystem::Condvar> condvar_;
  bool done_;
  bool handle_error_;
  bool status_ok_;

  DISALLOW_COPY_AND_ASSIGN(ApacheProxyFetch);
};

bool IsCompressibleContentType(const char* content_type) {
  if (content_type == NULL) {
    return false;
  }
  GoogleString type = content_type;
  size_t separator_idx = type.find(";");
  if (separator_idx != GoogleString::npos) {
    type.erase(separator_idx);
  }

  bool res = false;
  if (type.find("text/") == 0) {
    res = true;
  } else if (type.find("application/") == 0) {
    if (type.find("javascript") != type.npos ||
        type.find("json") != type.npos ||
        type.find("ecmascript") != type.npos ||
        type == "application/livescript" ||
        type == "application/js" ||
        type == "application/jscript" ||
        type == "application/x-js" ||
        type == "application/xhtml+xml" ||
        type == "application/xml") {
      res = true;
    }
  }

  return res;
}

// Default handler when the file is not found
void instaweb_404_handler(const GoogleString& url, request_rec* request) {
  request->status = HTTP_NOT_FOUND;
  ap_set_content_type(request, "text/html; charset=utf-8");
  ap_rputs("<html><head><title>Not Found</title></head>", request);
  ap_rputs("<body><h1>Apache server with mod_pagespeed</h1>OK", request);
  ap_rputs("<hr>NOT FOUND:", request);
  ap_rputs(url.c_str(), request);
  ap_rputs("</body></html>", request);
}

void send_out_headers_and_body(request_rec* request,
                               const ResponseHeaders& response_headers,
                               const GoogleString& output) {
  // We always disable downstream header filters when sending out
  // pagespeed resources, since we've captured them in the origin fetch.
  ResponseHeadersToApacheRequest(response_headers,
                                 true,  // Disable downstream header filters.
                                 request);
  if (response_headers.status_code() == HttpStatus::kOK &&
      IsCompressibleContentType(request->content_type)) {
    // Make sure compression is enabled for this response.
    ap_add_output_filter("DEFLATE", NULL, request, request->connection);
  }

  // Recompute the content-length, because the content may have changed.
  ap_set_content_length(request, output.size());
  // Send the body
  ap_rwrite(output.c_str(), output.size(), request);
}

// Evaluate custom_options based upon global_options, directory-specific
// options and query-param/request-header options. Returns computed
// custom options (or NULL if global_options should be used).
//
// Caller takes ownership of options.
RewriteOptions* get_custom_options(ApacheServerContext* server_context,
                                   request_rec* request,
                                   GoogleUrl* gurl,
                                   RequestHeaders* request_headers,
                                   RewriteOptions* global_options) {
  // Set directory specific options.  These will be the options for the
  // directory the resource is in, which under some configurations will be
  // different from the options for the directory that the referencing html is
  // in.  This can lead to us using different options here when regenerating
  // the resource than would be used if the resource were generated as part of
  // a rewrite kicked off by a request for the referencing html file.  This is
  // hard to fix, so instead we're documenting that you must make sure the
  // configuration for your resources matches the configuration for your html
  // files.
  RewriteOptions* custom_options = NULL;
  ApacheConfig* directory_options = static_cast<ApacheConfig*>
      ap_get_module_config(request->per_dir_config, &pagespeed_module);
  if ((directory_options != NULL) && directory_options->modified()) {
    custom_options = server_context->apache_factory()->NewRewriteOptions();
    custom_options->Merge(*global_options);
    custom_options->Merge(*directory_options);
  }

  // TODO(sligocki): Move inside PSOL.
  // Merge in query-param or header-based options.
  // Note: We do not generally get response headers in the resource flow,
  // so NULL is passed in instead.
  ServerContext::OptionsBoolPair query_options_success =
      server_context->GetQueryOptions(gurl, request_headers, NULL);
  if (!query_options_success.second) {
    server_context->message_handler()->Message(
        kWarning, "Invalid ModPagespeed query params or headers for "
        "request %s. Serving with default options.", gurl->spec_c_str());
  }
  if (query_options_success.first != NULL) {
    if (custom_options == NULL) {
      custom_options = server_context->apache_factory()->NewRewriteOptions();
      custom_options->Merge(*global_options);
    }
    custom_options->Merge(*query_options_success.first);
    delete query_options_success.first;
    // Don't run any experiments if we're handling a customized request.
    custom_options->set_running_furious_experiment(false);
  }

  return custom_options;
}

// Handle url as .pagespeed. rewritten resource.
void handle_as_pagespeed_resource(const RequestContextPtr& request_context,
                                  GoogleUrl* gurl,
                                  const GoogleString& url,
                                  RewriteOptions* custom_options,
                                  ApacheServerContext* server_context,
                                  RequestHeaders* request_headers,
                                  request_rec* request) {
  RewriteDriver* driver = ResourceFetch::GetDriver(
      *gurl, custom_options, server_context, request_context);

  MessageHandler* message_handler = server_context->message_handler();
  message_handler->Message(kInfo, "Fetching resource %s...", url.c_str());

  GoogleString output;  // TODO(jmarantz): Quit buffering resource output.
  StringWriter writer(&output);

  SyncFetcherAdapterCallback* callback = new SyncFetcherAdapterCallback(
      server_context->thread_system(), &writer, request_context);
  callback->SetRequestHeadersTakingOwnership(request_headers);

  if (ResourceFetch::BlockingFetch(*gurl, server_context, driver, callback)) {
    ResponseHeaders* response_headers = callback->response_headers();
    // TODO(sligocki): Check that this is already done in ResourceFetch
    // and remove redundant setting here.
    response_headers->SetDate(server_context->timer()->NowMs());
    // ResourceFetch adds X-Page-Speed header, old mod_pagespeed code
    // did not. For now, we remove that header for consistency.
    // TODO(sligocki): Consistently use X- headers in MPS and PSOL.
    // I think it would be good to change X-Mod-Pagespeed -> X-Page-Speed
    // and use that for all HTML and resource requests.
    response_headers->RemoveAll(kPageSpeedHeader);
    message_handler->Message(kInfo, "Fetch succeeded for %s, status=%d",
                             url.c_str(), response_headers->status_code());
    send_out_headers_and_body(request, *response_headers, output);
  } else {
    RewriteStats* stats = server_context->rewrite_stats();
    stats->resource_404_count()->Add(1);
    instaweb_404_handler(url, request);
  }

  callback->Release();
}

// Handle url with In Place Resource Optimization (IPRO) flow.
bool handle_as_in_place(const RequestContextPtr& request_context,
                        GoogleUrl* gurl,
                        const GoogleString& url,
                        RewriteOptions* custom_options,
                        ApacheServerContext* server_context,
                        RequestHeaders* owned_headers,
                        request_rec* request) {
  scoped_ptr<RequestHeaders> request_headers(owned_headers);
  bool handled = false;

  RewriteDriver* driver = ResourceFetch::GetDriver(
      *gurl, custom_options, server_context, request_context);

  MessageHandler* message_handler = server_context->message_handler();
  message_handler->Message(kInfo, "Trying to serve rewritten resource "
                           "in-place: %s", url.c_str());

  ApacheProxyFetch fetch(
      url, server_context->thread_system(), driver, request);
  fetch.set_handle_error(false);
  driver->FetchInPlaceResource(*gurl, false /* proxy_mode */, &fetch);

  fetch.Wait();
  if (fetch.status_ok()) {
    server_context->rewrite_stats()->ipro_served()->Add(1);
    message_handler->Message(kInfo, "Serving rewritten resource in-place: %s",
                             url.c_str());
    handled = true;
  } else if (fetch.response_headers()->status_code() ==
             CacheUrlAsyncFetcher::kNotInCacheStatus) {
    server_context->rewrite_stats()->ipro_not_in_cache()->Add(1);
    message_handler->Message(kInfo, "Could not rewrite resource in-place "
                             "because URL is not in cache: %s",
                             url.c_str());
    // This URL was not found in cache (neither the input resource nor
    // a ResourceNotCacheable entry) so we need to get it into cache
    // (or at least a note that it cannot be cached stored there).
    // We do that using an Apache output filter.
    InPlaceResourceRecorder* recorder = new InPlaceResourceRecorder(
        url, request_headers.release(), driver->options()->respect_vary(),
        server_context->http_cache(), server_context->statistics(),
        message_handler);
    ap_add_output_filter(kModPagespeedInPlaceFilterName, recorder,
                         request, request->connection);
    ap_add_output_filter(kModPagespeedInPlaceCheckHeadersName, recorder,
                         request, request->connection);
  } else {
    server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
    message_handler->Message(kInfo, "Could not rewrite resource in-place: %s",
                             url.c_str());
  }
  driver->Cleanup();

  return handled;
}

bool handle_as_proxy(ApacheServerContext* server_context,
                     request_rec* request,
                     const RequestContextPtr& request_context,
                     GoogleUrl* gurl,
                     RewriteOptions* options,
                     scoped_ptr<RewriteOptions>* custom_options) {
  bool handled = false;
  // Consider Issue 609: proxying an external CSS file via
  // ModPagespeedMapProxyDomain, and the CSS file makes reference to
  // a font file, which mod_pagespeed does not know anything about, and
  // does not know how to absolutify.  We need to handle the request for
  // the external font file here, even if IPRO (in place resource
  // optimization) is off.
  bool is_proxy = false;
  GoogleString mapped_url;
  if (options->domain_lawyer()->MapOriginUrl(*gurl, &mapped_url, &is_proxy) &&
      is_proxy) {
    RewriteDriver* driver = ResourceFetch::GetDriver(
        *gurl, custom_options->release(), server_context, request_context);
    ApacheProxyFetch apache_proxy_fetch(
        mapped_url, server_context->thread_system(), driver, request);
    server_context->proxy_fetch_factory()->StartNewProxyFetch(
        mapped_url, &apache_proxy_fetch, driver, NULL, NULL);
    apache_proxy_fetch.Wait();
    handled = true;
  }

  return handled;
}

// Determines whether the url can be handled as a mod_pagespeed or in-place
// optimized resource, and handles it, returning true.  Success status is
// written to the status code in the response headers.
bool handle_as_resource(ApacheServerContext* server_context,
                        request_rec* request,
                        GoogleUrl* gurl,
                        const GoogleString& url) {
  if (!gurl->is_valid()) {
    return false;
  }

  // Flushing the cache mutates global_options, so this has to happen before we
  // construct the options that we use to decide whether IPRO is enabled.
  server_context->FlushCacheIfNecessary();

  ApacheRequestContext* apache_request_context = new ApacheRequestContext(
      server_context->thread_system()->NewMutex(), request);
  apache_request_context->set_url(url);
  RequestContextPtr request_context(apache_request_context);
  bool using_spdy = request_context->using_spdy();
  RewriteOptions* global_options = server_context->global_options();
  if (using_spdy && (server_context->SpdyConfig() != NULL)) {
    global_options = server_context->SpdyConfig();
  }

  scoped_ptr<RequestHeaders> request_headers(new RequestHeaders);
  // Filter limited request headers into backend fetch.
  // TODO(sligocki): Put this filtering in ResourceFetch and instead use:
  // ApacheRequestToRequestHeaders(*request, request_headers.get());
  for (int i = 0, n = arraysize(RewriteDriver::kPassThroughRequestAttributes);
       i < n; ++i) {
    const char* value = apr_table_get(
        request->headers_in,
        RewriteDriver::kPassThroughRequestAttributes[i]);
    if (value != NULL) {
      request_headers->Add(
          RewriteDriver::kPassThroughRequestAttributes[i], value);
    }
  }

  scoped_ptr<RewriteOptions> custom_options(get_custom_options(
      server_context, request, gurl, request_headers.get(), global_options));

  RewriteOptions* options = custom_options.get();  // Options for this request.
  if (custom_options.get() == NULL) {
    options = global_options;
  }

  // Finally, do the actual handling.
  bool handled = false;
  if (server_context->IsPagespeedResource(*gurl)) {
    handled = true;
    handle_as_pagespeed_resource(request_context, gurl, url,
                                 custom_options.release(), server_context,
                                 request_headers.release(), request);
  } else if (handle_as_proxy(server_context, request, request_context, gurl,
                             options, &custom_options)) {
    handled = true;
  } else if (options->in_place_rewriting_enabled() && options->enabled() &&
             options->IsAllowed(url)) {
    handled = handle_as_in_place(request_context, gurl, url,
                                 custom_options.release(), server_context,
                                 request_headers.release(), request);
  }

  return handled;
}

// Write response headers and send out headers and output, including the option
//     for a custom Content-Type.
void write_handler_response(const StringPiece& output,
                            request_rec* request,
                            ContentType content_type,
                            const StringPiece& cache_control) {
  ResponseHeaders response_headers;
  response_headers.SetStatusAndReason(HttpStatus::kOK);
  response_headers.set_major_version(1);
  response_headers.set_minor_version(1);

  response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
  AprTimer timer;
  int64 now_ms = timer.NowMs();
  response_headers.SetDate(now_ms);
  response_headers.SetLastModified(now_ms);
  response_headers.Add(HttpAttributes::kCacheControl, cache_control);
  send_out_headers_and_body(request, response_headers, output.as_string());
}

void write_handler_response(const StringPiece& output, request_rec* request) {
  write_handler_response(output, request,
                         kContentTypeHtml, HttpAttributes::kNoCache);
}

// Returns request URL if it was a .pagespeed. rewritten resource URL.
// Otherwise returns NULL. Since other Apache modules can change request->uri,
// we stow the original request URL in a note. This method reads that note
// and thus should return the URL that the browser actually requested (rather
// than a mod_rewrite altered URL).
const char* get_instaweb_resource_url(request_rec* request,
                                      ApacheServerContext* server_context) {
  const char* resource = apr_table_get(request->notes, kResourceUrlNote);

  // If our translate_name hook, save_url_hook, failed to run because some
  // other module's translate_hook returned OK first, then run it now. The
  // main reason we try to do this early is to save our URL before mod_rewrite
  // mutates it.
  if (resource == NULL) {
    save_url_in_note(request, server_context);
    resource = apr_table_get(request->notes, kResourceUrlNote);
  }

  if (resource != NULL && strcmp(resource, kResourceUrlNo) == 0) {
    return NULL;
  }

  const char* url = apr_table_get(request->notes, kPagespeedOriginalUrl);
  return url;
}

// Used by log_request_headers for testing only.
struct HeaderLoggingData {
  HeaderLoggingData(StringWriter* writer_in, MessageHandler* handler_in)
      : writer(writer_in), handler(handler_in) {}
  StringWriter* writer;
  MessageHandler* handler;
};

// Helper function to support the LogRequestHeadersHandler.  Called once for
// each header to write header data in a form suitable for javascript inlining.
// Used only for tests.
int log_request_headers(void* logging_data,
                        const char* key, const char* value) {
  HeaderLoggingData* hld = static_cast<HeaderLoggingData*>(logging_data);
  StringWriter* writer = hld->writer;
  MessageHandler* handler = hld->handler;

  GoogleString escaped_key;
  GoogleString escaped_value;

  EscapeToJsStringLiteral(key, false, &escaped_key);
  EscapeToJsStringLiteral(value, false, &escaped_value);

  writer->Write("alert(\"", handler);
  writer->Write(escaped_key, handler);
  writer->Write("=", handler);
  writer->Write(escaped_value, handler);
  writer->Write("\");\n", handler);

  return 1;  // Continue iteration.
}

// Writes text wrapped in a <pre> block
void WritePre(StringPiece str, Writer* writer, MessageHandler* handler) {
  writer->Write("<pre>\n", handler);
  writer->Write(str, handler);
  writer->Write("</pre>\n", handler);
}

void instaweb_static_handler(request_rec* request,
                             ApacheServerContext* server_context) {
  StaticAssetManager* static_asset_manager =
      server_context->static_asset_manager();
  StringPiece request_uri_path = request->parsed_uri.path;
  // Strip out the common prefix url before sending to StaticAssetManager.
  StringPiece file_name =
      request_uri_path.substr(
          strlen(ApacheRewriteDriverFactory::kStaticAssetPrefix));
  StringPiece file_contents;
  StringPiece cache_header;
  ContentType content_type;
  if (static_asset_manager->GetAsset(
      file_name, &file_contents, &content_type, &cache_header)) {
    write_handler_response(file_contents, request, content_type, cache_header);
  } else {
    instaweb_404_handler(request->parsed_uri.path, request);
  }
}

apr_status_t instaweb_console_handler(
    request_rec* request, ApacheConfig* config,
    ApacheMessageHandler* message_handler) {
  GoogleString output;
  StringWriter writer(&output);
  writer.Write("<!DOCTYPE html>"
                "<title>mod_pagespeed console</title>",
               message_handler);
  writer.Write("<style>", message_handler);
  writer.Write(CSS_mod_pagespeed_console_css, message_handler);
  writer.Write("</style>", message_handler);
  writer.Write(HTML_mod_pagespeed_console_body, message_handler);
  writer.Write("<script>", message_handler);
  if (config->statistics_logging_charts_js().size() > 0 &&
      config->statistics_logging_charts_css().size() > 0) {
    writer.Write("var chartsOfflineJS = '", message_handler);
    writer.Write(config->statistics_logging_charts_js(), message_handler);
    writer.Write("';", message_handler);
    writer.Write("var chartsOfflineCSS = '", message_handler);
    writer.Write(config->statistics_logging_charts_css(), message_handler);
    writer.Write("';", message_handler);
  } else {
    if (config->statistics_logging_charts_js().size() > 0 ||
        config->statistics_logging_charts_css().size() > 0) {
      message_handler->Message(kWarning, "Using online Charts API.");
    }
    writer.Write("var chartsOfflineJS, chartsOfflineCSS;", message_handler);
  }
  writer.Write(JS_mod_pagespeed_console_js, message_handler);
  writer.Write("</script>", message_handler);
  write_handler_response(output, request);
  return OK;
}

apr_status_t instaweb_statistics_handler(
    request_rec* request, ApacheServerContext* server_context,
    ApacheRewriteDriverFactory* factory, MessageHandler* message_handler) {
  bool general_stats_request =
      (strcmp(request->handler, kStatisticsHandler) == 0);
  bool global_stats_request =
      (strcmp(request->handler, kGlobalStatisticsHandler) == 0);

  int64 start_time, end_time, granularity_ms;
  std::set<GoogleString> var_titles;
  std::set<GoogleString> hist_titles;
  if (general_stats_request && !factory->use_per_vhost_statistics()) {
    global_stats_request = true;
  }

  // Choose the correct statistics.
  Statistics* statistics = global_stats_request ?
      factory->statistics() : server_context->statistics();

  QueryParams params;
  params.Parse(request->args);

  // Parse various mode query params.
  bool print_normal_config = params.Has("config");
  bool print_spdy_config = params.Has("spdy_config");

  // JSON statistics handling is done only if we have a console logger.
  bool json = false;
  if (statistics->console_logger() != NULL) {
    // Default values for start_time, end_time, and granularity_ms in case the
    // query does not include these parameters.
    start_time = 0;
    end_time = statistics->console_logger()->timer()->NowMs();
    // Granularity is the difference in ms between data points. If it is not
    // specified by the query, the default value is 3000 ms, the same as the
    // default logging granularity.
    granularity_ms = 3000;
    for (int i = 0; i < params.size(); ++i) {
      const GoogleString value =
          (params.value(i) == NULL) ? "" : *params.value(i);
      const char* name = params.name(i);
      if (strcmp(name, "json") == 0) {
        json = true;
      } else if (strcmp(name, "start_time") == 0) {
        StringToInt64(value, &start_time);
      } else if (strcmp(name, "end_time") == 0) {
        StringToInt64(value, &end_time);
      } else if (strcmp(name, "var_titles") == 0) {
        std::vector<StringPiece> variable_names;
        SplitStringPieceToVector(value, ",", &variable_names, true);
        for (size_t i = 0; i < variable_names.size(); ++i) {
          var_titles.insert(variable_names[i].as_string());
        }
      } else if (strcmp(name, "hist_titles") == 0) {
        std::vector<StringPiece> histogram_names;
        SplitStringPieceToVector(value, ",", &histogram_names, true);
        for (size_t i = 0; i < histogram_names.size(); ++i) {
          // TODO(morlovich): Cleanup & publicize UrlToFileNameEncoder::Unescape
          // and use it here, instead of this GlobalReplaceSubstring hack.
          GoogleString name = histogram_names[i].as_string();
          GlobalReplaceSubstring("%20", " ", &(name));
          hist_titles.insert(name);
        }
      } else if (strcmp(name, "granularity") == 0) {
        StringToInt64(value, &granularity_ms);
      }
    }
  }
  GoogleString output;
  StringWriter writer(&output);
  if (json) {
    statistics->console_logger()->DumpJSON(var_titles, hist_titles,
                                            start_time, end_time,
                                            granularity_ms, &writer,
                                            message_handler);
  } else {
    // Generate some navigational links to the right to help
    // our users get to other modes.
    writer.Write(
        "<div style='float:right'>View "
        "<a href='?config'>Configuration</a>, "
        "<a href='?spdy_config'>SPDY Configuration</a>, "
        "<a href='?'>Statistics</a> "
        "(<a href='?memcached'>with memcached Stats</a>). "
        "</div>",
        message_handler);

    // Only print stats or configuration, not both.
    if (!print_normal_config && !print_spdy_config) {
      writer.Write(global_stats_request ?
                       "Global Statistics" : "VHost-Specific Statistics",
                   message_handler);

      // Write <pre></pre> for Dump to keep good format.
      writer.Write("<pre>", message_handler);
      statistics->Dump(&writer, message_handler);
      writer.Write("</pre>", message_handler);
      statistics->RenderHistograms(&writer, message_handler);

      int flags = SystemCaches::kDefaultStatFlags;
      if (global_stats_request) {
        flags |= SystemCaches::kGlobalView;
      }

      if (params.Has("memcached")) {
        flags |= SystemCaches::kIncludeMemcached;
      }

      GoogleString backend_stats;
      factory->caches()->PrintCacheStats(
          static_cast<SystemCaches::StatFlags>(flags), &backend_stats);
      if (!backend_stats.empty()) {
        WritePre(backend_stats, &writer, message_handler);
      }
    }

    if (print_normal_config) {
      writer.Write("Configuration:<br>", message_handler);
      WritePre(server_context->config()->OptionsToString(),
               &writer, message_handler);
    }

    if (print_spdy_config) {
      ApacheConfig* spdy_config = server_context->SpdyConfig();
      if (spdy_config == NULL) {
        writer.Write("SPDY-specific configuration missing, using default.",
                      message_handler);
      } else {
        writer.Write("SPDY-specific configuration:<br>", message_handler);
        WritePre(spdy_config->OptionsToString(), &writer, message_handler);
      }
    }
  }

  if (json) {
    write_handler_response(output, request,
                           kContentTypeJson, HttpAttributes::kNoCache);
  } else {
    write_handler_response(output, request);
  }
  return OK;
}

// Copy the query params from a GET request into data. Return true if
// successful, otherwise, returns false and sets ret to the appropriate status.
bool parse_query_params_from_get(const request_rec* request, GoogleString* data,
                         apr_status_t* ret) {
  if (request->method_number != M_GET) {
    *ret = HTTP_METHOD_NOT_ALLOWED;
    return false;
  }

  // Add a dummy host (www.example.com) to the request URL to make it absolute
  // so that GoogleUrl can be used for parsing.
  GoogleUrl base("http://www.example.com");
  GoogleUrl url(base, request->unparsed_uri);

  if (!url.is_valid() || !url.has_query()) {
    *ret = HTTP_BAD_REQUEST;
    return false;
  }

  url.Query().CopyToString(data);
  return true;
}

// Read the body from a POST request into data. Return true if successful,
// otherwise, returns false and sets ret to the appropriate status.
{'fraction_non_alphanumeric': 0.1077717104606567, 'fraction_numerical': 0.0029655398565705243, 'mean_word_length': 3.418824418824419, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 421, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4616343', 'n_tokens_mistral': 20626, 'n_tokens_neox': 20133, 'n_words': 5094}