<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command issueserver is an Upspin server that serves GitHub issues.
//
// To try it out, first create a GitHub Personal Access Token which which to
// access the GitHub API, giving it "repo" privileges.
// See: https://github.com/settings/tokens/new
// Put the token string (a string of hex digits) in the file
// $HOME/upspin/issueserver-access-token and run issueserver with upbox:
//   $ upbox -schema=issueserver.upbox
// If all goes well, upbox will leave you in an 'upspin shell' session as
// <EMAIL>. Type 'ls' to look around.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	linebreak "github.com/dgryski/go-linebreak"
	"golang.org/x/build/maintner"

	"upspin.io/access"
	"upspin.io/cloud/https"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil"
	"upspin.io/upspin"

	_ "upspin.io/key/transports"
	_ "upspin.io/pack/eeintegrity"
)

var (
	watchGithub    = flag.String("watch-github", "", "Comma-separated list of GitHub owner/repo pairs to sync")
	dataDir        = flag.String("data-dir", defaultDataDir, "Local directory in which to write issueserver files")
	defaultDataDir = filepath.Join(os.Getenv("HOME"), "upspin", "issueserver")
)

func main() {
	flags.Parse(flags.Server)

	addr := upspin.NetAddr(flags.NetAddr)
	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   addr,
	}
	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}

	// Set up maintner Corpus.
	corpus := new(maintner.Corpus)
	logger := maintner.NewDiskMutationLogger(*dataDir)
	corpus.EnableLeaderMode(logger, *dataDir)
	if *watchGithub != "" {
		for _, pair := range strings.Split(*watchGithub, ",") {
			splits := strings.SplitN(pair, "/", 2)
			if len(splits) != 2 || splits[1] == "" {
				log.Fatalf("Invalid github repo: %s. Should be 'owner/repo,owner2/repo2'", pair)
			}
			token, err := getGithubToken()
			if err != nil {
				log.Fatalf("getting github token: %v", err)
			}
			corpus.TrackGithub(splits[0], splits[1], token)
		}
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	if err := corpus.Initialize(ctx, logger); err != nil {
		log.Fatal(err)
	}
	if *watchGithub != "" {
		go func() { log.Fatal(fmt.Errorf("Corpus.SyncLoop = %v", corpus.SyncLoop(ctx))) }()
	}

	// Set up DirServer and StoreServer.
	s, err := newServer(ep, cfg, corpus)
	if err != nil {
		log.Fatal(err)
	}
	http.Handle("/api/Store/", storeserver.New(cfg, storeServer{s}, addr))
	http.Handle("/api/Dir/", dirserver.New(cfg, dirServer{s}, addr))

	https.ListenAndServeFromFlags(nil)
}

// getGithubToken reads a GitHub Personal Access Token from the file
// $HOME/upspin/issueserver-github-token of the format "token".
func getGithubToken() (string, error) {
	file := filepath.Join(config.Home(), "upspin", "issueserver-github-token")
	slurp, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}<commit_msg>exp/cmd/issueserver: fix build<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command issueserver is an Upspin server that serves GitHub issues.
//
// To try it out, first create a GitHub Personal Access Token which which to
// access the GitHub API, giving it "repo" privileges.
// See: https://github.com/settings/tokens/new
// Put the token string (a string of hex digits) in the file
// $HOME/upspin/issueserver-access-token and run issueserver with upbox:
//   $ upbox -schema=issueserver.upbox
// If all goes well, upbox will leave you in an 'upspin shell' session as
// <EMAIL>. Type 'ls' to look around.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	linebreak "github.com/dgryski/go-linebreak"
	"golang.org/x/build/maintner"

	"upspin.io/access"
	"upspin.io/cloud/https"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil"
	"upspin.io/upspin"

	_ "upspin.io/key/transports"
	_ "upspin.io/pack/eeintegrity"
)

var (
	watchGithub    = flag.String("watch-github", "", "Comma-separated list of GitHub owner/repo pairs to sync")
	dataDir        = flag.String("data-dir", defaultDataDir, "Local directory in which to write issueserver files")
	defaultDataDir = filepath.Join(os.Getenv("HOME"), "upspin", "issueserver")
)

func main() {
	flags.Parse(flags.Server)

	addr := upspin.NetAddr(flags.NetAddr)
	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   addr,
	}
	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}

	// Set up maintner Corpus.
	corpus := new(maintner.Corpus)
	logger := maintner.NewDiskMutationLogger(*dataDir)
	corpus.EnableLeaderMode(logger, *dataDir)
	if *watchGithub != "" {
		for _, pair := range strings.Split(*watchGithub, ",") {
			splits := strings.SplitN(pair, "/", 2)
			if len(splits) != 2 || splits[1] == "" {
				log.Fatalf("Invalid github repo: %s. Should be 'owner/repo,owner2/repo2'", pair)
			}
			token, err := getGithubToken()
			if err != nil {
				log.Fatalf("getting github token: %v", err)
			}
			corpus.TrackGithub(splits[0], splits[1], token)
		}
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	if err := corpus.Initialize(ctx, logger); err != nil {
		log.Fatal(err)
	}
	if *watchGithub != "" {
		go func() { log.Fatal(fmt.Errorf("Corpus.SyncLoop = %v", corpus.SyncLoop(ctx))) }()
	}

	// Set up DirServer and StoreServer.
	s, err := newServer(ep, cfg, corpus)
	if err != nil {
		log.Fatal(err)
	}
	http.Handle("/api/Store/", storeserver.New(cfg, storeServer{s}, addr))
	http.Handle("/api/Dir/", dirserver.New(cfg, dirServer{s}, addr))

	https.ListenAndServeFromFlags(nil)
}

// getGithubToken reads a GitHub Personal Access Token from the file
// $HOME/upspin/issueserver-github-token of the format "token".
func getGithubToken() (string, error) {
	file := filepath.Join(config.Home(), "upspin", "issueserver-github-token")
	token, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}
{'fraction_non_alphanumeric': 0.15484363081617086, 'fraction_numerical': 0.0033562166285278413, 'mean_word_length': 3.874349442379182, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2095751', 'n_tokens_mistral': 2508, 'n_tokens_neox': 2359, 'n_words': 596}