<commit_before>/*
   This file supports the DVID REST API, breaking down URLs into
   commands and massaging attached data into appropriate data types.
*/

package server

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/janelia-flyem/go/nrsc"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/storage"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
)

const WebHelp = `
<!DOCTYPE html>
<html>

  <head>
	<meta charset='utf-8' />
	<meta http-equiv="X-UA-Compatible" content="chrome=1" />
	<meta name="description" content="DVID Web Server Home Page" />

	<title>Embedded DVID Web Server</title>
  </head>

  <body>

	<!-- HEADER -->
	<div id="header_wrap" class="outer">
		<header class="inner">
		  <h2 id="project_tagline">Stock help page for DVID server currently running on %s</h2>
		</header>
	</div>

	<!-- MAIN CONTENT -->
	<div id="main_content_wrap" class="outer">
	  <section id="main_content" class="inner">
		<h3>Welcome to DVID</h3>

		<p>This page provides an introduction to the currently running DVID server.  
		Developers can visit the <a href="https://github.com/janelia-flyem/dvid">Github repo</a> 
		for more documentation and code.
		The <a href="/">DVID admin console</a> may be available if you have downloaded the
		<a href="https://github.com/janelia-flyem/dvid-console">DVID console web client repo</a>
		and included <i>-webclient=/path/to/console</i> when running the
		<code>dvid serve</code> command.</p>
		
		<h3>HTTP API and command line use</h3>

		<p>All API endpoints follow this layout:
		<pre>
/serverhost:someport/api/...
		</pre>
		The online documentation doesn't show the server host prefixed to the "/api/..." URL,
		but it is required.

		<h4>General commands</h4>

		<pre>
 GET  /api/help

	The current page that lists all general and type-specific commands/HTTP API.

 GET  /api/help/{typename}

	Returns help for the given datatype.

 GET  /api/load

	Returns a JSON of server load statistics.

 GET  /api/storage

 	Returns a JSON object for each backend store where the key is the backend store name.
	The store object has local instance ID keys with the following object value:

    {
		"Name": "grayscale",
		"DataType": "uint8blk",
		"DataUUID": ...,
		"RootUUID": ...,  // this is the UUID of this data's root
		"Bytes": ...
	}

 GET  /api/server/info

	Returns JSON for server properties.

 GET  /api/server/types

	Returns JSON with the datatypes of currently stored data instances.  Datatypes are represented
	by a name and the URL of the reference implementation.  To see all possible datatypes, i.e., the
	list of compiled datatypes, use the "compiled-types" endpoint.

 GET  /api/server/compiled-types

 	Returns JSON of all possible datatypes for this server, i.e., the list of compiled datatypes.

 GET  /api/server/groupcache

 	Returns JSON for groupcache statistics for this server.  See github.com/golang/groupcache package
	Stats and CacheStats for MainCache and HotCache.

POST  /api/server/settings

	Sets server parameters.  Expects JSON to be posted with optional keys denoting parameters:
	{
		"gc": 500,
		"throttle": 2
	}

	 
	Possible keys:
	gc        Garbage collection target percentage.  This is a low-level server tuning
	            request that can affect overall request latency.
	            See: https://golang.org/pkg/runtime/debug/#SetGCPercent

	throttle  Maximum number of CPU-intensive requests that can be executed under throttle mode.
	            See imageblk and labelblk GET 3d voxels and POST voxels.
	            Default = 1.


POST  /api/server/reload-metadata

	Reloads the metadata from storage.  This is useful when using multiple DVID frontends with 
	a shared storage backend.  Typically, one DVID is designated "master" and handles any
	modifications of the repositories and data instances.  Other DVIDs simply reload the metadata 
	when prompted by an external coordinator, allowing the "slave" DVIDs to see changes made by
	the master DVID.


-------------------------
Repo-Level REST endpoints
-------------------------

 POST /api/repos<commit_msg>document memory profiler API<commit_after>/*
   This file supports the DVID REST API, breaking down URLs into
   commands and massaging attached data into appropriate data types.
*/

package server

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/janelia-flyem/go/nrsc"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/storage"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
)

const WebHelp = `
<!DOCTYPE html>
<html>

  <head>
	<meta charset='utf-8' />
	<meta http-equiv="X-UA-Compatible" content="chrome=1" />
	<meta name="description" content="DVID Web Server Home Page" />

	<title>Embedded DVID Web Server</title>
  </head>

  <body>

	<!-- HEADER -->
	<div id="header_wrap" class="outer">
		<header class="inner">
		  <h2 id="project_tagline">Stock help page for DVID server currently running on %s</h2>
		</header>
	</div>

	<!-- MAIN CONTENT -->
	<div id="main_content_wrap" class="outer">
	  <section id="main_content" class="inner">
		<h3>Welcome to DVID</h3>

		<p>This page provides an introduction to the currently running DVID server.  
		Developers can visit the <a href="https://github.com/janelia-flyem/dvid">Github repo</a> 
		for more documentation and code.
		The <a href="/">DVID admin console</a> may be available if you have downloaded the
		<a href="https://github.com/janelia-flyem/dvid-console">DVID console web client repo</a>
		and included <i>-webclient=/path/to/console</i> when running the
		<code>dvid serve</code> command.</p>
		
		<h3>HTTP API and command line use</h3>

		<p>All API endpoints (except for memory profiling) follow this layout:
		<pre>
/serverhost:someport/api/...
		</pre>
		The online documentation doesn't show the server host prefixed to the "/api/..." URL,
		but it is required.

		<h4>General commands</h4>

		<pre>
 GET  /api/help

	The current page that lists all general and type-specific commands/HTTP API.

 GET  /api/help/{typename}

	Returns help for the given datatype.

 GET  /api/load

	Returns a JSON of server load statistics.

 GET  /api/storage

 	Returns a JSON object for each backend store where the key is the backend store name.
	The store object has local instance ID keys with the following object value:

    {
		"Name": "grayscale",
		"DataType": "uint8blk",
		"DataUUID": ...,
		"RootUUID": ...,  // this is the UUID of this data's root
		"Bytes": ...
	}

 GET  /api/server/info

	Returns JSON for server properties.

 GET  /api/server/types

	Returns JSON with the datatypes of currently stored data instances.  Datatypes are represented
	by a name and the URL of the reference implementation.  To see all possible datatypes, i.e., the
	list of compiled datatypes, use the "compiled-types" endpoint.

 GET  /api/server/compiled-types

 	Returns JSON of all possible datatypes for this server, i.e., the list of compiled datatypes.

 GET  /api/server/groupcache

 	Returns JSON for groupcache statistics for this server.  See github.com/golang/groupcache package
	Stats and CacheStats for MainCache and HotCache.

POST  /api/server/settings

	Sets server parameters.  Expects JSON to be posted with optional keys denoting parameters:
	{
		"gc": 500,
		"throttle": 2
	}

	 
	Possible keys:
	gc        Garbage collection target percentage.  This is a low-level server tuning
	            request that can affect overall request latency.
	            See: https://golang.org/pkg/runtime/debug/#SetGCPercent

	throttle  Maximum number of CPU-intensive requests that can be executed under throttle mode.
	            See imageblk and labelblk GET 3d voxels and POST voxels.
	            Default = 1.


POST  /api/server/reload-metadata

	Reloads the metadata from storage.  This is useful when using multiple DVID frontends with 
	a shared storage backend.  Typically, one DVID is designated "master" and handles any
	modifications of the repositories and data instances.  Other DVIDs simply reload the metadata 
	when prompted by an external coordinator, allowing the "slave" DVIDs to see changes made by
	the master DVID.

-------------------------
Memory Profiler endpoints
-------------------------

 GET  /profiler/start

 	Starts a memory profiler and redirects to a web page where you can see real-time
	memory usage graphs.

 GET  /profiler/stop

 	Stops the memory profiler.

 GET  /profiler/info.html

 	Shows real-time memory usage graphs.

 GET  /profiler/info

 	Returns JSON of memory usage data.


-------------------------
Repo-Level REST endpoints
-------------------------

 POST /api/repos
{'fraction_non_alphanumeric': 0.11595982142857143, 'fraction_numerical': 0.003794642857142857, 'mean_word_length': 3.185427370387669, 'pattern_counts': {'":': 14, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2826787', 'n_tokens_mistral': 3022, 'n_tokens_neox': 2756, 'n_words': 1000}