<commit_before>/*
 * Copyright (c) 2013 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	mpg "github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goon"
	"appengine"
	"appengine/blobstore"
	aimage "appengine/image"
	"appengine/memcache"
	"appengine/taskqueue"
	"appengine/urlfetch"
	"appengine/user"

	"atom"
	"rdf"
	"rss"
	"sanitizer"
)

func serveError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

type Includes struct {
	Angular             string
	BootstrapCss        string
	BootstrapJs         string
	FontAwesome         string
	Jquery              string
	JqueryUI            string
	Underscore          string
	MiniProfiler        template.HTML
	User                *User
	Messages            []string
	GoogleAnalyticsId   string
	GoogleAnalyticsHost string
	SubURL              string
	IsDev               bool
	IsAdmin             bool
	StripeKey           string
	StripePlans         []Plan
}

var (
	Angular      string
	BootstrapCss string
	BootstrapJs  string
	FontAwesome  string
	Jquery       string
	JqueryUI     string
	Underscore   string
	isDevServer  bool
	subURL       string
)

func init() {
	angular_ver := "1.2.1"
	bootstrap_ver := "3.0.2"
	font_awesome_ver := "4.0.3"
	jquery_ver := "2.0.3"
	jqueryui_ver := "1.10.3.sortable"
	isDevServer = appengine.IsDevAppServer()

	if appengine.IsDevAppServer() {
		Angular = "/static/js/angular.js"
		BootstrapCss = "/static/css/bootstrap.css"
		BootstrapJs = "/static/js/bootstrap.js"
		FontAwesome = "/static/css/font-awesome.css"
		Jquery = fmt.Sprintf("/static/js/jquery-%v.js", jquery_ver)
		JqueryUI = fmt.Sprintf("/static/js/jquery-ui-%v.js", jqueryui_ver)
		Underscore = "/static/js/underscore.js"
	} else {
		Angular = fmt.Sprintf("//ajax.googleapis.com/ajax/libs/angularjs/%v/angular.min.js", angular_ver)
		BootstrapCss = fmt.Sprintf("//netdna.bootstrapcdn.com/bootstrap/%v/css/bootstrap.min.css", bootstrap_ver)
		BootstrapJs = fmt.Sprintf("//netdna.bootstrapcdn.com/bootstrap/%v/js/bootstrap.min.js", bootstrap_ver)
		FontAwesome = fmt.Sprintf("//netdna.bootstrapcdn.com/font-awesome/%v/css/font-awesome.min.css", font_awesome_ver)
		Jquery = fmt.Sprintf("//ajax.googleapis.com/ajax/libs/jquery/%v/jquery.min.js", jquery_ver)
		JqueryUI = fmt.Sprintf("/static/js/jquery-ui-%v.min.js", jqueryui_ver)
		Underscore = "/static/js/underscore-min.js"
	}
}

func includes(c mpg.Context, w http.ResponseWriter, r *http.Request) *Includes {
	i := &Includes{
		Angular:             Angular,
		BootstrapCss:        BootstrapCss,
		BootstrapJs:         BootstrapJs,
		FontAwesome:         FontAwesome,
		Jquery:              Jquery,
		JqueryUI:            JqueryUI,
		Underscore:          Underscore,
		MiniProfiler:        c.Includes(),
		GoogleAnalyticsId:   GOOGLE_ANALYTICS_ID,
		GoogleAnalyticsHost: GOOGLE_ANALYTICS_HOST,
		SubURL:              subURL,
		IsDev:               isDevServer,
		StripeKey:           STRIPE_KEY,
		StripePlans:         STRIPE_PLANS,
	}

	if cu := user.Current(c); cu != nil {
		gn := goon.FromContext(c)
		user := &User{Id: cu.ID}
		if err := gn.Get(user); err == nil {
			i.User = user
			i.IsAdmin = cu.Admin

			if len(user.Messages) > 0 {
				i.Messages = user.Messages
				user.Messages = nil
				gn.Put(user)
			}

			/*
				if _, err := r.Cookie("update-bug"); err != nil {
					i.Messages = append(i.Messages, "Go Read had some problems updating feeds. It may take a while for new stories to appear again. Sorry about that.")
					http.SetCookie(w, &http.Cookie{
						Name: "update-bug",
						Value: "done",
						Expires: time.Now().Add(time.Hour * 24 * 7),
					})
				}
			*/
		}
	}

	return i
}

var dateFormats = []string{
	"01-02-2006",
	"01/02/2006",
	"01/02/2006 - 15:04",
	"01/02/2006 15:04:05 MST",
	"01/02/2006 3:04 PM",
	"02-01-2006",
	"02/01/2006",
	"02.01.2006 -0700",
	"02/01/2006 - 15:04",
	"02.01.2006 15:04",
	"02/01/2006 15:04:05",
	"02.01.2006 15:04:05",
	"02-01-2006 15:04:05 MST",
	"02 Jan 2006",
	"02 Jan 2006 15:04:05",
	"02 Jan 2006 15:04:05 -0700",
	"02 Jan 2006 15:04:05 MST",
	"02 Jan 2006 15:04:05 UT",
	"02 Jan 2006 15:04 MST",
	"06-1-2 15:04",
	"06/1/2 15:04",
	"1/2/2006",
	"1/2/2006 15:04:05 MST",
	"1/2/2006 3:04:05 PM",
	"1/2/2006 3:04:05 PM MST",
	"15:04 02.01.2006 -0700",
	"2006-01-02",
	"2006/01/02",
	"2006-01-02 00:00:00.0 15:04:05.0 -0700",
	"2006-01-02 15:04",
	"2006-01-02 15:04:05 -0700",
	"2006-01-02 15:04:05-07:00",
	"2006-01-02 15:04:05-0700",
	"2006-01-02 15:04:05 MST",
	"2006-01-02 15:04:05Z",
	"2006-01-02T15:04:05",
	"2006-01-02T15:04:05:00",
	"2006-01-02T15:04:05 -0700",
	"2006-01-02T15:04:05-07:00",
	"2006-01-02T15:04:05-0700",
	"2006-01-02T15:04:05:-0700",
	"2006-01-02T15:04:05-07:00:00",
	"2006-01-02T15:04:05Z",
	"2006-01-02T15:04-07:00",<commit_msg>Date format<commit_after>/*
 * Copyright (c) 2013 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	mpg "github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goon"
	"appengine"
	"appengine/blobstore"
	aimage "appengine/image"
	"appengine/memcache"
	"appengine/taskqueue"
	"appengine/urlfetch"
	"appengine/user"

	"atom"
	"rdf"
	"rss"
	"sanitizer"
)

func serveError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

type Includes struct {
	Angular             string
	BootstrapCss        string
	BootstrapJs         string
	FontAwesome         string
	Jquery              string
	JqueryUI            string
	Underscore          string
	MiniProfiler        template.HTML
	User                *User
	Messages            []string
	GoogleAnalyticsId   string
	GoogleAnalyticsHost string
	SubURL              string
	IsDev               bool
	IsAdmin             bool
	StripeKey           string
	StripePlans         []Plan
}

var (
	Angular      string
	BootstrapCss string
	BootstrapJs  string
	FontAwesome  string
	Jquery       string
	JqueryUI     string
	Underscore   string
	isDevServer  bool
	subURL       string
)

func init() {
	angular_ver := "1.2.1"
	bootstrap_ver := "3.0.2"
	font_awesome_ver := "4.0.3"
	jquery_ver := "2.0.3"
	jqueryui_ver := "1.10.3.sortable"
	isDevServer = appengine.IsDevAppServer()

	if appengine.IsDevAppServer() {
		Angular = "/static/js/angular.js"
		BootstrapCss = "/static/css/bootstrap.css"
		BootstrapJs = "/static/js/bootstrap.js"
		FontAwesome = "/static/css/font-awesome.css"
		Jquery = fmt.Sprintf("/static/js/jquery-%v.js", jquery_ver)
		JqueryUI = fmt.Sprintf("/static/js/jquery-ui-%v.js", jqueryui_ver)
		Underscore = "/static/js/underscore.js"
	} else {
		Angular = fmt.Sprintf("//ajax.googleapis.com/ajax/libs/angularjs/%v/angular.min.js", angular_ver)
		BootstrapCss = fmt.Sprintf("//netdna.bootstrapcdn.com/bootstrap/%v/css/bootstrap.min.css", bootstrap_ver)
		BootstrapJs = fmt.Sprintf("//netdna.bootstrapcdn.com/bootstrap/%v/js/bootstrap.min.js", bootstrap_ver)
		FontAwesome = fmt.Sprintf("//netdna.bootstrapcdn.com/font-awesome/%v/css/font-awesome.min.css", font_awesome_ver)
		Jquery = fmt.Sprintf("//ajax.googleapis.com/ajax/libs/jquery/%v/jquery.min.js", jquery_ver)
		JqueryUI = fmt.Sprintf("/static/js/jquery-ui-%v.min.js", jqueryui_ver)
		Underscore = "/static/js/underscore-min.js"
	}
}

func includes(c mpg.Context, w http.ResponseWriter, r *http.Request) *Includes {
	i := &Includes{
		Angular:             Angular,
		BootstrapCss:        BootstrapCss,
		BootstrapJs:         BootstrapJs,
		FontAwesome:         FontAwesome,
		Jquery:              Jquery,
		JqueryUI:            JqueryUI,
		Underscore:          Underscore,
		MiniProfiler:        c.Includes(),
		GoogleAnalyticsId:   GOOGLE_ANALYTICS_ID,
		GoogleAnalyticsHost: GOOGLE_ANALYTICS_HOST,
		SubURL:              subURL,
		IsDev:               isDevServer,
		StripeKey:           STRIPE_KEY,
		StripePlans:         STRIPE_PLANS,
	}

	if cu := user.Current(c); cu != nil {
		gn := goon.FromContext(c)
		user := &User{Id: cu.ID}
		if err := gn.Get(user); err == nil {
			i.User = user
			i.IsAdmin = cu.Admin

			if len(user.Messages) > 0 {
				i.Messages = user.Messages
				user.Messages = nil
				gn.Put(user)
			}

			/*
				if _, err := r.Cookie("update-bug"); err != nil {
					i.Messages = append(i.Messages, "Go Read had some problems updating feeds. It may take a while for new stories to appear again. Sorry about that.")
					http.SetCookie(w, &http.Cookie{
						Name: "update-bug",
						Value: "done",
						Expires: time.Now().Add(time.Hour * 24 * 7),
					})
				}
			*/
		}
	}

	return i
}

var dateFormats = []string{
	"01-02-2006",
	"01/02/2006",
	"01/02/2006 - 15:04",
	"01/02/2006 15:04:05 MST",
	"01/02/2006 3:04 PM",
	"02-01-2006",
	"02/01/2006",
	"02.01.2006 -0700",
	"02/01/2006 - 15:04",
	"02.01.2006 15:04",
	"02/01/2006 15:04:05",
	"02.01.2006 15:04:05",
	"02-01-2006 15:04:05 MST",
	"02 Jan 2006",
	"02 Jan 2006 15:04:05",
	"02 Jan 2006 15:04:05 -0700",
	"02 Jan 2006 15:04:05 MST",
	"02 Jan 2006 15:04:05 UT",
	"02 Jan 2006 15:04 MST",
	"02 Monday, Jan 2006 15:04",
	"06-1-2 15:04",
	"06/1/2 15:04",
	"1/2/2006",
	"1/2/2006 15:04:05 MST",
	"1/2/2006 3:04:05 PM",
	"1/2/2006 3:04:05 PM MST",
	"15:04 02.01.2006 -0700",
	"2006-01-02",
	"2006/01/02",
	"2006-01-02 00:00:00.0 15:04:05.0 -0700",
	"2006-01-02 15:04",
	"2006-01-02 15:04:05 -0700",
	"2006-01-02 15:04:05-07:00",
	"2006-01-02 15:04:05-0700",
	"2006-01-02 15:04:05 MST",
	"2006-01-02 15:04:05Z",
	"2006-01-02T15:04:05",
	"2006-01-02T15:04:05:00",
	"2006-01-02T15:04:05 -0700",
	"2006-01-02T15:04:05-07:00",
	"2006-01-02T15:04:05-0700",
	"2006-01-02T15:04:05:-0700",
	"2006-01-02T15:04:05-07:00:00",
	"2006-01-02T15:04:05Z",
	"2006-01-02T15:04-07:00",
{'fraction_non_alphanumeric': 0.14617509929200484, 'fraction_numerical': 0.1042997755137282, 'mean_word_length': 3.019083969465649, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7329925', 'n_tokens_mistral': 5550, 'n_tokens_neox': 4509, 'n_words': 959}