<commit_before>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"
    "log"
	"syscall"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Setting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    return app
}

// Clean shutdown
func (a *App) Finish() {
    // Start a log flush
    timber.Close()
}

func (a *App) setUserAndGroup() {
    // We do not have logging set up yet. We just panic() on error.

    desiredUserName, _ := a.Cfg.Get("gop", "user", "")
    // Usernames it's ok to run as, in order of preference
    possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}

    doneIt := false
    for _, desiredUserName := range possibleUserNames {
        if runAsUserName(desiredUserName) {
            doneIt = true
            break
        }
    }

    if !doneIt {
        panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
    }

// Can't log at this stage :-/
//    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
}

func (a *App) setProcessGroupForNelly() {
	// Nelly knows our pid and will check that there is always at
	// least one process in the process group with the same id as our pid
	mypid := syscall.Getpid()
	err := syscall.Setpgid(mypid, mypid)
	if err != nil {
        panic(fmt.Sprintf("Failed to setprgp]: %s\n", mypid, mypid, err.Error()))
	}
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we<commit_msg>remove setuid code from gop - not reliable on threaded procs on linux<commit_after>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"
    "log"
	"syscall"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    // Linux setuid() doesn't work with threaded procs :-O
    // and the go runtime threads before we can get going.
    //
    // http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/linux-thread-problems.html
//    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Setting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    return app
}

// Clean shutdown
func (a *App) Finish() {
    // Start a log flush
    timber.Close()
}

//
// Commented out - not reliable - see comment at call site
// func (a *App) setUserAndGroup() {
//     // We do not have logging set up yet. We just panic() on error.
// 
//     desiredUserName, _ := a.Cfg.Get("gop", "user", "")
//     // Usernames it's ok to run as, in order of preference
//     possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}
// 
//     doneIt := false
//     for _, desiredUserName := range possibleUserNames {
//         if runAsUserName(desiredUserName) {
//             doneIt = true
//             break
//         }
//     }
// 
//     if !doneIt {
//         panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
//     }
// 
// // Can't log at this stage :-/
// //    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
// }

func (a *App) setProcessGroupForNelly() {
	// Nelly knows our pid and will check that there is always at
	// least one process in the process group with the same id as our pid
	mypid := syscall.Getpid()
	err := syscall.Setpgid(mypid, mypid)
	if err != nil {
        panic(fmt.Sprintf("Failed to setprgp]: %s\n", mypid, mypid, err.Error()))
	}
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we
{'fraction_non_alphanumeric': 0.10660762942779292, 'fraction_numerical': 0.00045413260672116256, 'mean_word_length': 1.8891439816333224, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4160978', 'n_tokens_mistral': 2748, 'n_tokens_neox': 2635, 'n_words': 806}