<commit_before>// Package pastebin is a simple modern and powerful pastebin service
package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	duration "github.com/channelmeter/iso8601duration"
	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

type Configuration struct {
	// ADDRESS that pastebin will return links for
	Address string
	// LENGTH of paste id
	Length int
	// PORT that pastebin will listen on
	Port string
	// USERNAME for database
	Username string
	// PASS database password
	Password string
	// NAME database name
	Name string
}

var configuration Configuration

// DATABASE connection String
var DATABASE string

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	SUCCESS bool   `json:"success"`
	STATUS  string `json:"status"`
	ID      string `json:"id"`
	TITLE   string `json:"title"`
	SHA1    string `json:"sha1"`
	URL     string `json:"url"`
	SIZE    int    `json:"size"`
	DELKEY  string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	// hardcode this for now until I figure out why json isn't parsing correctly
	id := uniuri.NewLen(6)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err = db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Sha1 hashes paste into a sha1 hash
func Sha1(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}
<commit_msg>Declare previously undeclared variable<commit_after>// Package pastebin is a simple modern and powerful pastebin service
package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	duration "github.com/channelmeter/iso8601duration"
	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

type Configuration struct {
	// ADDRESS that pastebin will return links for
	Address string
	// LENGTH of paste id
	Length int
	// PORT that pastebin will listen on
	Port string
	// USERNAME for database
	Username string
	// PASS database password
	Password string
	// NAME database name
	Name string
}

var configuration Configuration

// DATABASE connection String
var DATABASE string

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	SUCCESS bool   `json:"success"`
	STATUS  string `json:"status"`
	ID      string `json:"id"`
	TITLE   string `json:"title"`
	SHA1    string `json:"sha1"`
	URL     string `json:"url"`
	SIZE    int    `json:"size"`
	DELKEY  string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	// hardcode this for now until I figure out why json isn't parsing correctly
	id := uniuri.NewLen(6)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Sha1 hashes paste into a sha1 hash
func Sha1(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

{'fraction_non_alphanumeric': 0.10938131823695917, 'fraction_numerical': 0.00646987464617873, 'mean_word_length': 3.5636531365313653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2697895', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1616, 'n_words': 533}