<commit_before>// Copyright (c) 2016 <NAME> <<EMAIL>>, all rights reserved.
// Use of this document is governed by a license found in the LICENSE document.

// gitgodoc allows you to view `godoc` documentation for different branches of a Git repository
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

// TODO make cross-platform

// TODO implement pruning of remote branches that no longer exist (we should be able to detect after each
// push)

type branchPorts map[string]uint
type branchLocks map[string]*sync.Mutex

const (
	refCopy = "@refcopy"

	debug = false
)

var (
	validBranch = regexp.MustCompile("^[a-z0-9_]+$")
	href        = regexp.MustCompile(`(href|src)="/("|[^/][^"]*")`)

	fServeFrom = flag.String("serveFrom", "", "directory to use as a working directory")
	fRepo      = flag.String("repo", "", "git url from which to clone")
	fPkg       = flag.String("pkg", "", "the package the repo represents")
	fGoPath    = flag.String("gopath", "", "a relative GOPATH that will be used when running the godoc server (prepended with the repo dir)")
	fPortStart = flag.Uint("port", 8080, "the port on which to serve; controlled godoc instances will be started on subsequent ports")
)

type gitLabPushWebook struct {
	Ref string `json:"ref"`
}

type server struct {
	repo, serveFrom, pkg, refCopyDir string
	gopath                           []string

	nextPort uint

	ports     atomic.Value
	portsLock sync.Mutex

	repos     atomic.Value
	reposLock sync.Mutex
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("gitgodoc ")

	flag.Parse()

	s := newServer()

	s.setupRefCopy()
	s.serve()
}

func (s *server) setupRefCopy() {
	infof("setting up reference copy of repo in %v", s.refCopyDir)

	s.reposLock.Lock()
	defer s.reposLock.Unlock()

	if _, err := os.Stat(filepath.Join(s.refCopyDir, ".git")); err != nil {
		// the copy does not exist
		// no need to do a fetch; instead we need to clone

		git(s.serveFrom, "clone", s.repo, s.refCopyDir)
		git(s.refCopyDir, "checkout", "-f", "origin/master")
	} else {
		git(s.refCopyDir, "fetch")
		git(s.refCopyDir, "checkout", "-f", "origin/master")
	}
}

// TODO break this apart
func (s *server) serve() {
	tr := &http.Transport{
		DisableCompression:  true,
		DisableKeepAlives:   false,
		MaxIdleConnsPerHost: 20,
	}
	client := &http.Client{Transport: tr}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

		if !path.IsAbs(r.URL.Path) {
			fatalf("expected absolute URL path, got %v", r.URL.Path)
		}

		if r.URL.Path == "/" {
			if r.Method == http.MethodPost {
				if vs, ok := r.URL.Query()["refresh"]; ok {

					// TODO support more than just gitlab
					if len(vs) == 1 && vs[0] == "gitlab" {
						// we need to parse the branch from the request

						body, err := ioutil.ReadAll(r.Body)
						if err != nil {
							fatalf("failed to read body of POST request")
						}
						var pHook gitLabPushWebook
						err = json.Unmarshal(body, &pHook)

						if err != nil {
							fatalf("could not decode Gitlab web")
						}

						ref := strings.Split(pHook.Ref, "/")
						if len(ref) != 3 {
							fatalf("did not understand format of branch: %v", pHook.Ref)
						}

						branch := ref[2]

						infof("got a request to refresh branch %v", branch)

						s.fetch(branch)

						w.WriteHeader(http.StatusOK)
						w.Write([]byte("OK"))

						return
					} else {
						w.WriteHeader(http.StatusInternalServerError)

						msg := fmt.Sprintf("Did not understand refresh request: %v", r.URL)
						infof(msg)
						fmt.Fprintln(w, msg)

						return
					}
				}
			}

			// we should serve a simple page of links to existing branches
			w.WriteHeader(http.StatusOK)
			var bs []string
			m := s.repos.Load().(branchLocks)
			for k := range m {
				bs = append(bs, k)
			}

			sort.Strings(bs)

			tmpl := struct {
				Branches []string
				Pkg      string
			}{
				Branches: bs,
				Pkg:      s.pkg,
			}
			tpl := `
<!DOCTYPE html>
<html>
	<head>
		<meta charset="UTF-8">
		<title>gitgodoc server for {{.Pkg}}</title>
	</head>
	<body>
		<h1><code>gitgodoc</code> server for <code>{{.Pkg}}</code></h1>
		{{with .Branches}}
			<ul>
			{{ range . }}<li><a href="/{{.}}">{{ . }}</a></li>{{ end }}
			</ul>
		{{else}}
			<div><strong>No branches known to godoc server</strong></div>
		{{end}}
	</body>
</html>`

			t, err := template.New("webpage").Parse(tpl)
			if err != nil {
				fatalf("could not parse branch browser template: %v", err)
			}

			t.Execute(w, tmpl)

			return
		}

		parts := strings.Split(r.URL.Path, "/")
		branch := parts[1]
		actUrl := path.Join(parts[2:]...)

		if !validBranch.MatchString(branch) {
			branch = "master"

			// now we need to redirect to the same URL but with a branch
			newUrl := *r.URL
			newUrl.Path = "/" + path.Join(branch, actUrl)

			http.Redirect(w, r, newUrl.String(), http.StatusFound)
<commit_msg>Support gitlab merge requests<commit_after>// Copyright (c) 2016 <NAME> <<EMAIL>>, all rights reserved.
// Use of this document is governed by a license found in the LICENSE document.

// gitgodoc allows you to view `godoc` documentation for different branches of a Git repository
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

// TODO make cross-platform

// TODO implement pruning of remote branches that no longer exist (we should be able to detect after each
// push)

type branchPorts map[string]uint
type branchLocks map[string]*sync.Mutex

const (
	refCopy = "@refcopy"

	debug = false
)

var (
	validBranch = regexp.MustCompile("^[a-z0-9_]+$")
	href        = regexp.MustCompile(`(href|src)="/("|[^/][^"]*")`)

	fServeFrom = flag.String("serveFrom", "", "directory to use as a working directory")
	fRepo      = flag.String("repo", "", "git url from which to clone")
	fPkg       = flag.String("pkg", "", "the package the repo represents")
	fGoPath    = flag.String("gopath", "", "a relative GOPATH that will be used when running the godoc server (prepended with the repo dir)")
	fPortStart = flag.Uint("port", 8080, "the port on which to serve; controlled godoc instances will be started on subsequent ports")
)

// TODO this should be split into separate types and the correct
// type used based on the header sent from Gitlab
type gitLabWebhook struct {
	ObjectKind       string `json:"object_kind"`
	Ref              string `json:"ref"`
	ObjectAttributes struct {
		MergeStatus  string `json:"merge_status"`
		SourceBranch string `json:"source_branch"`
		TargetBranch string `json:"target_branch"`
	} `json:"object_attributes"`
}

type server struct {
	repo, serveFrom, pkg, refCopyDir string
	gopath                           []string

	nextPort uint

	ports     atomic.Value
	portsLock sync.Mutex

	repos     atomic.Value
	reposLock sync.Mutex
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("gitgodoc ")

	flag.Parse()

	s := newServer()

	s.setupRefCopy()
	s.serve()
}

func (s *server) setupRefCopy() {
	infof("setting up reference copy of repo in %v", s.refCopyDir)

	s.reposLock.Lock()
	defer s.reposLock.Unlock()

	if _, err := os.Stat(filepath.Join(s.refCopyDir, ".git")); err != nil {
		// the copy does not exist
		// no need to do a fetch; instead we need to clone

		git(s.serveFrom, "clone", s.repo, s.refCopyDir)
		git(s.refCopyDir, "checkout", "-f", "origin/master")
	} else {
		git(s.refCopyDir, "fetch")
		git(s.refCopyDir, "checkout", "-f", "origin/master")
	}
}

// TODO break this apart
func (s *server) serve() {
	tr := &http.Transport{
		DisableCompression:  true,
		DisableKeepAlives:   false,
		MaxIdleConnsPerHost: 20,
	}
	client := &http.Client{Transport: tr}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

		if !path.IsAbs(r.URL.Path) {
			fatalf("expected absolute URL path, got %v", r.URL.Path)
		}

		if r.URL.Path == "/" {
			if r.Method == http.MethodPost {
				if vs, ok := r.URL.Query()["refresh"]; ok {

					// TODO support more than just gitlab
					if len(vs) == 1 && vs[0] == "gitlab" {
						// we need to parse the branch from the request

						body, err := ioutil.ReadAll(r.Body)
						if err != nil {
							fatalf("failed to read body of POST request")
						}
						var pHook gitLabWebhook
						err = json.Unmarshal(body, &pHook)

						if err != nil {
							fatalf("could not decode Gitlab web")
						}

						switch pHook.ObjectKind {
						case "merge_request":
							s.fetch(pHook.ObjectAttributes.SourceBranch)

							infof("got a request to refresh branch %v in response to a merge request hook with target %v and merge status %v", pHook.ObjectAttributes.SourceBranch, pHook.ObjectAttributes.TargetBranch, pHook.ObjectAttributes.MergeStatus)

						case "push":
							ref := strings.Split(pHook.Ref, "/")
							if len(ref) != 3 {
								fatalf("did not understand format of branch: %v", pHook.Ref)
							}

							branch := ref[2]

							infof("got a request to refresh branch %v in response to a push hook", branch)

							s.fetch(branch)
						default:
							w.WriteHeader(http.StatusInternalServerError)

							msg := fmt.Sprintf("Did not understand Gitlab refresh request; unknown object_kind: %v", pHook.ObjectKind)
							infof(msg)
							fmt.Fprintln(w, msg)

							return
						}

						w.WriteHeader(http.StatusOK)
						w.Write([]byte("OK"))

						return
					} else if len(vs) == 1 && vs[0] == "gitlab_merge_request" {

					} else {
						w.WriteHeader(http.StatusInternalServerError)

						msg := fmt.Sprintf("Did not understand refresh request: %v", r.URL)
						infof(msg)
						fmt.Fprintln(w, msg)

						return
					}
				}
			}

			// we should serve a simple page of links to existing branches
			w.WriteHeader(http.StatusOK)
			var bs []string
			m := s.repos.Load().(branchLocks)
			for k := range m {
				bs = append(bs, k)
			}

			sort.Strings(bs)

			tmpl := struct {
				Branches []string
				Pkg      string
			}{
				Branches: bs,
				Pkg:      s.pkg,
			}
			tpl := `
<!DOCTYPE html>
<html>
	<head>
		<meta charset="UTF-8">
		<title>gitgodoc server for {{.Pkg}}</title>
	</head>
	<body>
		<h1><code>gitgodoc</code> server for <code>{{.Pkg}}</code></h1>
		{{with .Branches}}
			{{ range . }}{{ if eq . "master" }}<p><a href="/{{.}}">{{ . }}</a></p>{{ end }}{{ end }}
			<ul>
			{{ range . }}{{ if ne . "master" }}<li><a href="/{{.}}">{{ . }}</a></li>{{ end }}{{ end }}
			</ul>
		{{else}}
			<div><strong>No branches known to godoc server</strong></div>
		{{end}}
	</body>
</html>`

			t, err := template.New("webpage").Parse(tpl)
			if err != nil {
				fatalf("could not parse branch browser template: %v", err)
			}

			t.Execute(w, tmpl)

			return
		}

		parts := strings.Split(r.URL.Path, "/")
		branch := parts[1]
		actUrl := path.Join(parts[2:]...)

		if !validBranch.MatchString(branch) {
			branch = "master"

			// now we need to redirect to the same URL but with a branch
			newUrl := *r.URL
			newUrl.Path = "/" + path.Join(branch, actUrl)

			http.Redirect(w, r, newUrl.String(), http.StatusFound)

{'fraction_non_alphanumeric': 0.15269275130866827, 'fraction_numerical': 0.004081270517256676, 'mean_word_length': 2.9592553565156305, 'pattern_counts': {'":': 2, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323578', 'n_tokens_mistral': 4590, 'n_tokens_neox': 3886, 'n_words': 1109}