<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"koding/db/mongodb/modelhelper"
	"koding/kodingkite"
	"koding/kontrol/kontrolproxy/resolver"
	"koding/kontrol/kontrolproxy/utils"
	"koding/tools/config"
	"koding/tools/logger"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/hoisie/redis"
	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
	"github.com/nranchev/go-libGeoIP"
)

const (
	CookieVM      = "kdproxy-vm"
	CookieUseHTTP = "kdproxy-usehttp"

	// Used as a value for CookieVM
	MagicCookieValue = "KEbPptvE7dGLM5YFtcfz"

	KONTROLPROXY_NAME = "kontrolproxy"

	geoIPDatFile = "files/GeoIP.dat"
)

var (
	proxyName, _ = os.Hostname()

	// used for all our log
	log      = logger.New(KONTROLPROXY_NAME)
	logLevel logger.Level

	// redis client, connects once
	redisClient = redis.Client{
		Addr: "127.0.0.1:6379", // for future reference
	}

	// used for request limiter, counting every hit
	redisIntervals = []interval{
		interval{"second", 1},
		interval{"minute", 60},
		interval{"hour", 3600},
		interval{"day", 86400},
	}

	// cookie handling is used for features like securePage
	store = sessions.NewCookieStore([]byte("kontrolproxy-secret-key"))

	// used for various kinds of use cases like validator, 404 pages,
	// maintenance,...
	templates *template.Template

	// geoip instances
	geo *libgeo.GeoIP

	// readed config
	conf *config.Config

	// flag variables
	flagConfig    = flag.String("c", "", "Configuration profile from file")
	flagRegion    = flag.String("r", "", "Region")
	flagVMProxies = flag.Bool("v", false, "Enable ports for VM users (1024-10000)")
	flagDebug     = flag.Bool("d", false, "Debug mode")
	flagTemplates = flag.String("template", "files/templates", "Change template directory")
	flagCerts     = flag.String("certs", ".", "Change Certificate directory")
)

// Proxy is implementing the http.Handler interface (via ServeHTTP). This is
// used as the main handler for our HTTP and HTTPS listeners.
type Proxy struct {
	// mux implies the http.Handler interface. Currently we use the default
	// http.ServeMux but it can be swapped with any other mux that satisfies the
	// http.Handler
	mux *http.ServeMux

	// resolvers are needed to resolve for different cases
	resolvers map[string]Resolver

	// enableFirewall is used to activate the internal validator that uses the
	// restrictions and filter collections to validate the incoming requests
	// accoding to ip, country, requests and so on..
	enableFirewall bool

	// logDestination specifies the destination of requests log in the
	// Combined Log Format.
	logDestination io.Writer

	// cacheTransports is used to enable cache based roundtrips for certaing
	// request hosts, such as koding.com.
	cacheTransports map[string]http.RoundTripper

	// oskite references
	oskites   map[string]*kite.Client
	oskitesMu sync.Mutex
}

type Resolver func(*http.Request, *resolver.Target) http.Handler

// used by redis counter
type interval struct {
	name     string
	duration int64
}

func main() {
	flag.Parse()
	if *flagConfig == "" || *flagRegion == "" {
		log.Error("No flags defined. -c, -r and -v is not set. Aborting")
		os.Exit(1)
	}

	conf = config.MustConfig(*flagConfig)
	modelhelper.Initialize(conf.Mongo)

	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.GetLoggingLevelFromConfig(KONTROLPROXY_NAME, *flagConfig)
	}
	log.SetLevel(logLevel)

	templates = template.Must(template.ParseFiles(
		*flagTemplates+"/notfound.html",
		*flagTemplates+"/notactiveVM.html",
		*flagTemplates+"/securepage.html",
		*flagTemplates+"/quotaExceeded.html",
		*flagTemplates+"/maintenance.html",
	))

	var err error
	geo, err = libgeo.Load(geoIPDatFile)
	if err != nil {
		log.Warning("geoip db loading err: %s", err.Error())
	}

	log.Info("Kontrolproxy started.")
	log.Info("I'm using %d cpus for GOMACPROCS", runtime.NumCPU())
	runtime.GOMAXPROCS(runtime.NumCPU())

	p := &Proxy{
		mux:             http.NewServeMux(),
		enableFirewall:  true,
		cacheTransports: make(map[string]http.RoundTripper),
		oskites:         make(map[string]*kite.Client),
		resolvers:       make(map[string]Resolver),
	}<commit_msg>kontrolproxy: decide with flag about geoIP dat file<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"koding/db/mongodb/modelhelper"
	"koding/kodingkite"
	"koding/kontrol/kontrolproxy/resolver"
	"koding/kontrol/kontrolproxy/utils"
	"koding/tools/config"
	"koding/tools/logger"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/hoisie/redis"
	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
	"github.com/nranchev/go-libGeoIP"
)

const (
	CookieVM      = "kdproxy-vm"
	CookieUseHTTP = "kdproxy-usehttp"

	// Used as a value for CookieVM
	MagicCookieValue = "KEbPptvE7dGLM5YFtcfz"

	KONTROLPROXY_NAME = "kontrolproxy"
)

var (
	proxyName, _ = os.Hostname()

	// used for all our log
	log      = logger.New(KONTROLPROXY_NAME)
	logLevel logger.Level

	// redis client, connects once
	redisClient = redis.Client{
		Addr: "127.0.0.1:6379", // for future reference
	}

	// used for request limiter, counting every hit
	redisIntervals = []interval{
		interval{"second", 1},
		interval{"minute", 60},
		interval{"hour", 3600},
		interval{"day", 86400},
	}

	// cookie handling is used for features like securePage
	store = sessions.NewCookieStore([]byte("kontrolproxy-secret-key"))

	// used for various kinds of use cases like validator, 404 pages,
	// maintenance,...
	templates *template.Template

	// geoip instances
	geo *libgeo.GeoIP

	// readed config
	conf *config.Config

	// flag variables
	flagConfig       = flag.String("c", "", "Configuration profile from file")
	flagRegion       = flag.String("r", "", "Region")
	flagVMProxies    = flag.Bool("v", false, "Enable ports for VM users (1024-10000)")
	flagDebug        = flag.Bool("d", false, "Debug mode")
	flagTemplates    = flag.String("template", "files/templates", "Change template directory")
	flagCerts        = flag.String("certs", ".", "Change Certificate directory")
	flagGeoIpDatFile = flag.String("g", "/opt/kite/kontrolproxy/files/GeoIP.dat", "Change geoIP dat file path")
)

// Proxy is implementing the http.Handler interface (via ServeHTTP). This is
// used as the main handler for our HTTP and HTTPS listeners.
type Proxy struct {
	// mux implies the http.Handler interface. Currently we use the default
	// http.ServeMux but it can be swapped with any other mux that satisfies the
	// http.Handler
	mux *http.ServeMux

	// resolvers are needed to resolve for different cases
	resolvers map[string]Resolver

	// enableFirewall is used to activate the internal validator that uses the
	// restrictions and filter collections to validate the incoming requests
	// accoding to ip, country, requests and so on..
	enableFirewall bool

	// logDestination specifies the destination of requests log in the
	// Combined Log Format.
	logDestination io.Writer

	// cacheTransports is used to enable cache based roundtrips for certaing
	// request hosts, such as koding.com.
	cacheTransports map[string]http.RoundTripper

	// oskite references
	oskites   map[string]*kite.Client
	oskitesMu sync.Mutex
}

type Resolver func(*http.Request, *resolver.Target) http.Handler

// used by redis counter
type interval struct {
	name     string
	duration int64
}

func main() {
	flag.Parse()
	if *flagConfig == "" || *flagRegion == "" {
		log.Error("No flags defined. -c, -r and -v is not set. Aborting")
		os.Exit(1)
	}

	conf = config.MustConfig(*flagConfig)
	modelhelper.Initialize(conf.Mongo)

	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.GetLoggingLevelFromConfig(KONTROLPROXY_NAME, *flagConfig)
	}
	log.SetLevel(logLevel)

	templates = template.Must(template.ParseFiles(
		*flagTemplates+"/notfound.html",
		*flagTemplates+"/notactiveVM.html",
		*flagTemplates+"/securepage.html",
		*flagTemplates+"/quotaExceeded.html",
		*flagTemplates+"/maintenance.html",
	))

	var err error
	geo, err = libgeo.Load(*flagGeoIpDatFile)
	if err != nil {
		log.Warning("geoip db loading err: %s", err.Error())
	}

	log.Info("Kontrolproxy started.")
	log.Info("I'm using %d cpus for GOMACPROCS", runtime.NumCPU())
	runtime.GOMAXPROCS(runtime.NumCPU())

	p := &Proxy{
		mux:             http.NewServeMux(),
		enableFirewall:  true,
		cacheTransports: make(map[string]http.RoundTripper),
		oskites:         make(map[string]*kite.Client),
		resolvers:       make(map[string]Resolver),
	}
{'fraction_non_alphanumeric': 0.12705428992406212, 'fraction_numerical': 0.00884053043182591, 'mean_word_length': 3.979683972911964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6534713', 'n_tokens_mistral': 3216, 'n_tokens_neox': 3047, 'n_words': 765}