<commit_before>// Copyright (c) 2017, Boise State University All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/client"
	"github.com/shirou/gopsutil/mem"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
)

const (
	// defaultNotebook is used if the request doesn't specify a docker image.
	defaultNotebook = "jupyter/minimal-notebook"
	// sessionKey is the cookie name for session managment
	sessionKey = "sessionKey"
	// redirectKey holds some context on login/oauth
	redirectKey = "redirectFrom"
	// bsuDefaultRegexp is a regular expression allowing all u.boisestate.edu and
	// boisestate.edu users login.  As far as we know, BSU doesn't allow symbols
	// in emails, but we may have to add:
	//
	// !#$%&'*+-/=?^_`{|}~
	//
	// in the future.  See RFC 5322 (https://tools.ietf.org/html/rfc5322).
	bsuRegexp = `^.+@(u\.)?boisestate.edu$`
	// use the hardened TLS config
	hardenTLS = false
)

func init() {
	log.SetFlags(log.Lshortfile | log.LstdFlags)
}

type session struct {
	sync.Mutex
	m     map[string]string
	token *oauth2.Token
}

func newSession() *session {
	return &session{m: make(map[string]string), token: nil}
}

func (s *session) get(key string) string {
	s.Lock()
	v := s.m[key]
	s.Unlock()
	return v
}

func (s *session) set(key, val string) {
	s.Lock()
	s.m[key] = val
	s.Unlock()
}

// notebookServer handles the http tasks for the temporary notebooks.
//
// XXX: note that larger configurations in the docker images, many files can be
// opened.  You may need to set a higher ulimit for files on the server.  On a
// server with ~60 open notebooks, ~20K files were opened.  Setting the limit
// on the order of 1<<18 should be enough, I hope.
type notebookServer struct {
	// pool manages the containers
	pool *notebookPool
	// token is the generated random auth token
	token string
	// sessionLock guards sessions
	sessionMu sync.Mutex
	// sessions holds cookie keys and user emails
	sessions map[string]*session
	// enableDockerPush listens for container updates
	enableDockerPush bool
	// enableOAuth if the
	enableOAuth bool
	// oauthConf is the OAuth2 client configuration
	oauthConf *oauth2.Config
	// oauthState is the string passed to the api to validate on return
	oauthState string
	// oauthToken is the API token
	oauthToken string
	// oauthSecret is the API secret
	oauthSecret string
	// oauthDomainRegexp is used to match whitelisted domains
	oauthMatch *regexp.Regexp
	// oauthWhiteList is automagically enabled user emails
	oauthWhiteList map[string]struct{}
	// buildMu guards buildMap
	buildMu sync.Mutex
	// buildMap holds names of images currently being built
	buildMap map[string]struct{}
	// redirectLock locks the redirectMap
	redirectMu sync.Mutex
	// redirectMap handles initial incoming requests before the user is
	// authenticated through OAuth.  The keys are a session type key, the value
	// is the path and query string of the request.
	redirectMap map[string]string
	// mux routes http traffic
	mux *ServeMux
	// embed a server
	*http.Server
	// html templates
	templates *template.Template
	// accessLogWriter is the access logging Writer<commit_msg>add a holder for the push token<commit_after>// Copyright (c) 2017, Boise State University All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/client"
	"github.com/shirou/gopsutil/mem"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
)

const (
	// defaultNotebook is used if the request doesn't specify a docker image.
	defaultNotebook = "jupyter/minimal-notebook"
	// sessionKey is the cookie name for session managment
	sessionKey = "sessionKey"
	// redirectKey holds some context on login/oauth
	redirectKey = "redirectFrom"
	// bsuDefaultRegexp is a regular expression allowing all u.boisestate.edu and
	// boisestate.edu users login.  As far as we know, BSU doesn't allow symbols
	// in emails, but we may have to add:
	//
	// !#$%&'*+-/=?^_`{|}~
	//
	// in the future.  See RFC 5322 (https://tools.ietf.org/html/rfc5322).
	bsuRegexp = `^.+@(u\.)?boisestate.edu$`
	// use the hardened TLS config
	hardenTLS = false
)

func init() {
	log.SetFlags(log.Lshortfile | log.LstdFlags)
}

type session struct {
	sync.Mutex
	m     map[string]string
	token *oauth2.Token
}

func newSession() *session {
	return &session{m: make(map[string]string), token: nil}
}

func (s *session) get(key string) string {
	s.Lock()
	v := s.m[key]
	s.Unlock()
	return v
}

func (s *session) set(key, val string) {
	s.Lock()
	s.m[key] = val
	s.Unlock()
}

// notebookServer handles the http tasks for the temporary notebooks.
//
// XXX: note that larger configurations in the docker images, many files can be
// opened.  You may need to set a higher ulimit for files on the server.  On a
// server with ~60 open notebooks, ~20K files were opened.  Setting the limit
// on the order of 1<<18 should be enough, I hope.
type notebookServer struct {
	// pool manages the containers
	pool *notebookPool
	// token is the generated random auth token
	token string
	// sessionLock guards sessions
	sessionMu sync.Mutex
	// sessions holds cookie keys and user emails
	sessions map[string]*session
	// enableDockerPush listens for container updates
	enableDockerPush bool
	// enableOAuth if the
	enableOAuth bool
	// oauthConf is the OAuth2 client configuration
	oauthConf *oauth2.Config
	// oauthState is the string passed to the api to validate on return
	oauthState string
	// oauthToken is the API token
	oauthToken string
	// oauthSecret is the API secret
	oauthSecret string
	// oauthDomainRegexp is used to match whitelisted domains
	oauthMatch *regexp.Regexp
	// oauthWhiteList is automagically enabled user emails
	oauthWhiteList map[string]struct{}
	// buildMu guards buildMap
	buildMu sync.Mutex
	// buildMap holds names of images currently being built
	buildMap map[string]struct{}
	// githubToken holds the github secret for the push event
	githubToken string
	// redirectLock locks the redirectMap
	redirectMu sync.Mutex
	// redirectMap handles initial incoming requests before the user is
	// authenticated through OAuth.  The keys are a session type key, the value
	// is the path and query string of the request.
	redirectMap map[string]string
	// mux routes http traffic
	mux *ServeMux
	// embed a server
	*http.Server
	// html templates
	templates *template.Template
	// accessLogWriter is the access logging Writer
{'fraction_non_alphanumeric': 0.10441551469601396, 'fraction_numerical': 0.00644208831029392, 'mean_word_length': 3.6956521739130435, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1107271', 'n_tokens_mistral': 2581, 'n_tokens_neox': 2510, 'n_words': 846}