<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	mathrand "math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	sqlite "github.com/gwenn/gosqlite"
	"github.com/icza/session"
	"github.com/jackc/pgx"
	"github.com/minio/go-homedir"
	"github.com/minio/minio-go"
	"golang.org/x/crypto/bcrypt"
	valid "gopkg.in/go-playground/validator.v9"
)

type ValType int

const (
	Binary ValType = iota
	Image
	Null
	Text
	Integer
	Float
)

type dataValue struct {
	Name  string
	Type  ValType
	Value interface{}
}
type dataRow []dataValue
type dbInfo struct {
	Database     string
	TableHeaders []string
	Records      []dataRow
	Tables       []string
	Tablename    string
	Watchers     int
	Stars        int
	Forks        int
	Discussions  int
	PRs          int
	Description  string
	Updates      int
	Branches     int
	Releases     int
	Contributors int
	Readme       string
	DateCreated  time.Time
	LastModified time.Time
	Public       bool
	Size         int
	Version      int
}

type metaInfo struct {
	Protocol     string
	Server       string
	Title        string
	Username     string
	Database     string
	LoggedInUser string
}

// Configuration file
type tomlConfig struct {
	Minio minioInfo
	Pg    pgInfo
}

// Minio connection parameters
type minioInfo struct {
	Server    string
	AccessKey string `toml:"access_key"`
	Secret    string
	HTTPS     bool
}

// PostgreSQL connection parameters
type pgInfo struct {
	Server   string
	Port     int
	Username string
	Password string
	Database string
}

var (
	// Our configuration info
	conf tomlConfig

	// PostgreSQL configuration info
	pgConfig = new(pgx.ConnConfig)

	// Connection handles
	db          *pgx.Conn
	minioClient *minio.Client

	// Address to listen on
	listenProtocol = "http"
	listenAddr     = "localhost"
	listenPort     = 8080

	// For input validation
	validate *valid.Validate

	// Our parsed HTML templates
	tmpl *template.Template
)

func downloadCSVHandler(w http.ResponseWriter, req *http.Request) {
	pageName := "Download CSV"

	// Split the request URL into path components
	pathStrings := strings.Split(req.URL.Path, "/")

	// Basic sanity check
	numPieces := len(pathStrings)
	if numPieces < 4 {
		errorPage(w, req, http.StatusBadRequest, "Invalid database requested")
		return
	}

	// Extract the username, database, table, and version requested
	var dbVersion int64
	userName := pathStrings[2]
	dbName := pathStrings[3]
	queryValues := req.URL.Query()
	dbTable := queryValues["table"][0]
	dbVersion, err := strconv.ParseInt(queryValues["version"][0], 10, 0) // This also validates the version input
	if err != nil {
		log.Printf("%s: Invalid version number: %v\n", pageName, err)
		errorPage(w, req, http.StatusBadRequest, "Invalid version number")
		return
	}

	// Validate the user supplied user, database, and table name
	err = validateUserDBTable(userName, dbName, dbTable)
	if err != nil {
		log.Printf("Validation failed for user, database, or table name: %s", err)
		errorPage(w, req, http.StatusBadRequest, "Invalid user, database, or table name")
		return
	}

	// Abort if no table name was given
	if dbTable == "" {
		log.Printf("%s: No table name given\n", pageName)
		errorPage(w, req, http.StatusBadRequest, "No table name given")
		return
	}

	// Verify the given database exists and is ok to be downloaded (and get the MinioID while at it)
	rows, err := db.Query("SELECT minioid FROM public.sqlite_databases "+
		"WHERE dbname = $1 "+
		"AND version = $2 "+
		"AND username = $3 "+
		"AND public = true", dbName, dbVersion, userName)
	if err != nil {
		log.Printf("%s: Database query failed: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}
	defer rows.Close()
	var minioID string
	for rows.Next() {
		err = rows.Scan(&minioID)
		if err != nil {
			log.Printf("%s: Error retrieving MinioID: %v\n", pageName, err)
			errorPage(w, req, http.StatusInternalServerError, "Database query failed")
			return
		}
	}
	if minioID == "" {
		log.Printf("%s: Couldn't retrieve required MinioID\n", pageName)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}

	// Get a handle from Minio for the database object
	userDB, err := minioClient.GetObject(userName, minioID)
	if err != nil {
		log.Printf("%s: Error retrieving DB from Minio: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}

	// Close the object handle when this function finishes
	defer func() {
		err := userDB.Close()
		if err != nil {
			log.Printf("%s: Error closing object handle: %v\n", pageName, err)
		}
	}()

	// Save the database locally to a temporary file
	tempfileHandle, err := ioutil.TempFile("", "databaseViewHandler-")
	if err != nil {
		log.Printf("%s: Error creating tempfile: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	tempfile := tempfileHandle.Name()
	bytesWritten, err := io.Copy(tempfileHandle, userDB)
	if err != nil {
		log.Printf("%s: Error writing database to temporary file: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	if bytesWritten == 0 {
		log.Printf("%s: 0 bytes written to the temporary file: %v\n", pageName, dbName)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	tempfileHandle.Close()
	defer os.Remove(tempfile) // Delete the temporary file when this function finishes

	// Open database
	db, err := sqlite.Open(tempfile, sqlite.OpenReadOnly)
	if err != nil {
		log.Printf("Couldn't open database: %s", err)
		errorPage(w, req, http.StatusInternalServerError, "Internal error")
		return
	}
	defer db.Close()

	// Retrieve all of the data from the selected database table
	stmt, err := db.Prepare("SELECT * FROM " + dbTable)
	if err != nil {
		log.Printf("Error when preparing statement for database: %s\v", err)
		errorPage(w, req, http.StatusInternalServerError, "Internal error")
		return
	}

	// Process each row
	fieldCount := -1
	var resultSet [][]string
	err = stmt.Select(func(s *sqlite.Stmt) error {

		// Get the number of fields in the result
		if fieldCount == -1 {
			fieldCount = stmt.DataCount()
		}

		// Retrieve the data for each row
		var row []string
		for i := 0; i < fieldCount; i++ {
			// Retrieve the data type for the field
			fieldType := stmt.ColumnType(i)

			isNull := false
			switch fieldType {
			case sqlite.Integer:
				var val int
				val, isNull, err = s.ScanInt(i)
				if err != nil {
					log.Printf("Something went wrong with ScanInt(): %v\n", err)
					break
				}
				if !isNull {
					row = append(row, fmt.Sprintf("%d", val))
				}
			case sqlite.Float:
				var val float64
				val, isNull, err = s.ScanDouble(i)
				if err != nil {
					log.Printf("Something went wrong with ScanDouble(): %v\n", err)
					break
				}
				if !isNull {
					row = append(row, strconv.FormatFloat(val, 'f', 4, 64))
				}
			case sqlite.Text:
				var val string
				val, isNull = s.ScanText(i)
				if !isNull {
					row = append(row, val)
				}
			case sqlite.Blob:
				var val []byte
				val, isNull = s.ScanBlob(i)
				if !isNull {
					// Base64 encode the value
					row = append(row, base64.StdEncoding.EncodeToString(val))
				}
			case sqlite.Null:
				isNull = true
			}
			if isNull {
				row = append(row, "NULL")
			}
		}
		resultSet = append(resultSet, row)

		return nil
	})
	if err != nil {
		log.Printf("Error when reading data from database: %s\v", err)
		errorPage(w, req, http.StatusInternalServerError,
			fmt.Sprintf("Error reading data from '%s'.  Possibly malformed?", dbName))
		return
	}
	defer stmt.Finalize()

	// Convert resultSet into CSV and send to the user
	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.csv", dbTable))
	w.Header().Set("Content-Type", "text/csv")
	csvFile := csv.NewWriter(w)
	err = csvFile.WriteAll(resultSet)
	if err != nil {
		log.Printf("%s: Error when generating CSV: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Error when generating CSV")
		return
	}
}

func downloadHandler(w http.ResponseWriter, req *http.Request) {
	pageName := "Download Handler"

	// Split the request URL into path components
	pathStrings := strings.Split(req.URL.Path, "/")

	// Basic sanity check
	numPieces := len(pathStrings)
	if numPieces < 4 {
		errorPage(w, req, http.StatusBadRequest, "Invalid database requested")
		return
	}

	// Extract the username, database, and version requested
	var dbVersion int64
	userName := pathStrings[2]
	dbName := pathStrings[3]
	queryValues := req.URL.Query()
	dbVersion, err := strconv.ParseInt(queryValues["version"][0], 10, 0) // This also validates the version input
	if err != nil {
		log.Printf("%s: Invalid version number: %v\n", pageName, err)
		errorPage(w, req, http.StatusBadRequest, "Invalid version number")
		return
	}

	// Validate the user supplied user and database name
	err = validateUserDB(userName, dbName)
	if err != nil {
		log.Printf("Validation failed for user or database name: %s", err)
		errorPage(w, req, http.StatusBadRequest, "Invalid user or database name")
		return
	}

	// Verify the given database exists and is ok to be downloaded (and get the MinioID while at it)
	rows, err := db.Query("SELECT minioid FROM public.sqlite_databases "+
		"WHERE dbname = $1 "+
		"AND version = $2 "+
		"AND username = $3 "+
		"AND public = true", dbName, dbVersion, userName)
	if err != nil {
		log.Printf("%s: Database query failed: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}<commit_msg>Update the Download and Download CSV buttons for the recent schema change<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	mathrand "math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	sqlite "github.com/gwenn/gosqlite"
	"github.com/icza/session"
	"github.com/jackc/pgx"
	"github.com/minio/go-homedir"
	"github.com/minio/minio-go"
	"golang.org/x/crypto/bcrypt"
	valid "gopkg.in/go-playground/validator.v9"
)

type ValType int

const (
	Binary ValType = iota
	Image
	Null
	Text
	Integer
	Float
)

type dataValue struct {
	Name  string
	Type  ValType
	Value interface{}
}
type dataRow []dataValue
type dbInfo struct {
	Database     string
	TableHeaders []string
	Records      []dataRow
	Tables       []string
	Tablename    string
	Watchers     int
	Stars        int
	Forks        int
	Discussions  int
	PRs          int
	Description  string
	Updates      int
	Branches     int
	Releases     int
	Contributors int
	Readme       string
	DateCreated  time.Time
	LastModified time.Time
	Public       bool
	Size         int
	Version      int
}

type metaInfo struct {
	Protocol     string
	Server       string
	Title        string
	Username     string
	Database     string
	LoggedInUser string
}

// Configuration file
type tomlConfig struct {
	Minio minioInfo
	Pg    pgInfo
}

// Minio connection parameters
type minioInfo struct {
	Server    string
	AccessKey string `toml:"access_key"`
	Secret    string
	HTTPS     bool
}

// PostgreSQL connection parameters
type pgInfo struct {
	Server   string
	Port     int
	Username string
	Password string
	Database string
}

var (
	// Our configuration info
	conf tomlConfig

	// PostgreSQL configuration info
	pgConfig = new(pgx.ConnConfig)

	// Connection handles
	db          *pgx.Conn
	minioClient *minio.Client

	// Address to listen on
	listenProtocol = "http"
	listenAddr     = "localhost"
	listenPort     = 8080

	// For input validation
	validate *valid.Validate

	// Our parsed HTML templates
	tmpl *template.Template
)

func downloadCSVHandler(w http.ResponseWriter, req *http.Request) {
	pageName := "Download CSV"

	// Split the request URL into path components
	pathStrings := strings.Split(req.URL.Path, "/")

	// Basic sanity check
	numPieces := len(pathStrings)
	if numPieces < 4 {
		errorPage(w, req, http.StatusBadRequest, "Invalid database requested")
		return
	}

	// Extract the username, database, table, and version requested
	var dbVersion int64
	userName := pathStrings[2]
	dbName := pathStrings[3]
	queryValues := req.URL.Query()
	dbTable := queryValues["table"][0]
	dbVersion, err := strconv.ParseInt(queryValues["version"][0], 10, 0) // This also validates the version input
	if err != nil {
		log.Printf("%s: Invalid version number: %v\n", pageName, err)
		errorPage(w, req, http.StatusBadRequest, "Invalid version number")
		return
	}

	// Validate the user supplied user, database, and table name
	err = validateUserDBTable(userName, dbName, dbTable)
	if err != nil {
		log.Printf("Validation failed for user, database, or table name: %s", err)
		errorPage(w, req, http.StatusBadRequest, "Invalid user, database, or table name")
		return
	}

	// Abort if no table name was given
	if dbTable == "" {
		log.Printf("%s: No table name given\n", pageName)
		errorPage(w, req, http.StatusBadRequest, "No table name given")
		return
	}

	// Verify the given database exists and is ok to be downloaded (and get the MinioID while at it)
	rows, err := db.Query(`
		SELECT minioid
		FROM database_versions
		WHERE db = (SELECT idnum
			FROM sqlite_databases
			WHERE username = $1
				AND dbname = $2
				AND version = $3)`,
		userName, dbName, dbVersion)
	if err != nil {
		log.Printf("%s: Database query failed: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}
	defer rows.Close()
	var minioID string
	for rows.Next() {
		err = rows.Scan(&minioID)
		if err != nil {
			log.Printf("%s: Error retrieving MinioID: %v\n", pageName, err)
			errorPage(w, req, http.StatusInternalServerError, "Database query failed")
			return
		}
	}
	if minioID == "" {
		log.Printf("%s: Couldn't retrieve required MinioID\n", pageName)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}

	// Get a handle from Minio for the database object
	userDB, err := minioClient.GetObject(userName, minioID)
	if err != nil {
		log.Printf("%s: Error retrieving DB from Minio: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}

	// Close the object handle when this function finishes
	defer func() {
		err := userDB.Close()
		if err != nil {
			log.Printf("%s: Error closing object handle: %v\n", pageName, err)
		}
	}()

	// Save the database locally to a temporary file
	tempfileHandle, err := ioutil.TempFile("", "databaseViewHandler-")
	if err != nil {
		log.Printf("%s: Error creating tempfile: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	tempfile := tempfileHandle.Name()
	bytesWritten, err := io.Copy(tempfileHandle, userDB)
	if err != nil {
		log.Printf("%s: Error writing database to temporary file: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	if bytesWritten == 0 {
		log.Printf("%s: 0 bytes written to the temporary file: %v\n", pageName, dbName)
		errorPage(w, req, http.StatusInternalServerError, "Internal server error")
		return
	}
	tempfileHandle.Close()
	defer os.Remove(tempfile) // Delete the temporary file when this function finishes

	// Open database
	db, err := sqlite.Open(tempfile, sqlite.OpenReadOnly)
	if err != nil {
		log.Printf("Couldn't open database: %s", err)
		errorPage(w, req, http.StatusInternalServerError, "Internal error")
		return
	}
	defer db.Close()

	// Retrieve all of the data from the selected database table
	stmt, err := db.Prepare("SELECT * FROM " + dbTable)
	if err != nil {
		log.Printf("Error when preparing statement for database: %s\v", err)
		errorPage(w, req, http.StatusInternalServerError, "Internal error")
		return
	}

	// Process each row
	fieldCount := -1
	var resultSet [][]string
	err = stmt.Select(func(s *sqlite.Stmt) error {

		// Get the number of fields in the result
		if fieldCount == -1 {
			fieldCount = stmt.DataCount()
		}

		// Retrieve the data for each row
		var row []string
		for i := 0; i < fieldCount; i++ {
			// Retrieve the data type for the field
			fieldType := stmt.ColumnType(i)

			isNull := false
			switch fieldType {
			case sqlite.Integer:
				var val int
				val, isNull, err = s.ScanInt(i)
				if err != nil {
					log.Printf("Something went wrong with ScanInt(): %v\n", err)
					break
				}
				if !isNull {
					row = append(row, fmt.Sprintf("%d", val))
				}
			case sqlite.Float:
				var val float64
				val, isNull, err = s.ScanDouble(i)
				if err != nil {
					log.Printf("Something went wrong with ScanDouble(): %v\n", err)
					break
				}
				if !isNull {
					row = append(row, strconv.FormatFloat(val, 'f', 4, 64))
				}
			case sqlite.Text:
				var val string
				val, isNull = s.ScanText(i)
				if !isNull {
					row = append(row, val)
				}
			case sqlite.Blob:
				var val []byte
				val, isNull = s.ScanBlob(i)
				if !isNull {
					// Base64 encode the value
					row = append(row, base64.StdEncoding.EncodeToString(val))
				}
			case sqlite.Null:
				isNull = true
			}
			if isNull {
				row = append(row, "NULL")
			}
		}
		resultSet = append(resultSet, row)

		return nil
	})
	if err != nil {
		log.Printf("Error when reading data from database: %s\v", err)
		errorPage(w, req, http.StatusInternalServerError,
			fmt.Sprintf("Error reading data from '%s'.  Possibly malformed?", dbName))
		return
	}
	defer stmt.Finalize()

	// Convert resultSet into CSV and send to the user
	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.csv", dbTable))
	w.Header().Set("Content-Type", "text/csv")
	csvFile := csv.NewWriter(w)
	err = csvFile.WriteAll(resultSet)
	if err != nil {
		log.Printf("%s: Error when generating CSV: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Error when generating CSV")
		return
	}
}

func downloadHandler(w http.ResponseWriter, req *http.Request) {
	pageName := "Download Handler"

	// Split the request URL into path components
	pathStrings := strings.Split(req.URL.Path, "/")

	// Basic sanity check
	numPieces := len(pathStrings)
	if numPieces < 4 {
		errorPage(w, req, http.StatusBadRequest, "Invalid database requested")
		return
	}

	// Extract the username, database, and version requested
	var dbVersion int64
	userName := pathStrings[2]
	dbName := pathStrings[3]
	queryValues := req.URL.Query()
	dbVersion, err := strconv.ParseInt(queryValues["version"][0], 10, 0) // This also validates the version input
	if err != nil {
		log.Printf("%s: Invalid version number: %v\n", pageName, err)
		errorPage(w, req, http.StatusBadRequest, "Invalid version number")
		return
	}

	// Validate the user supplied user and database name
	err = validateUserDB(userName, dbName)
	if err != nil {
		log.Printf("Validation failed for user or database name: %s", err)
		errorPage(w, req, http.StatusBadRequest, "Invalid user or database name")
		return
	}

	// Verify the given database exists and is ok to be downloaded (and get the MinioID while at it)
	rows, err := db.Query(`
		SELECT minioid
		FROM database_versions
		WHERE db = (SELECT idnum
			FROM sqlite_databases
			WHERE username = $1
				AND dbname = $2
				AND version = $3)`,
		userName, dbName, dbVersion)
	if err != nil {
		log.Printf("%s: Database query failed: %v\n", pageName, err)
		errorPage(w, req, http.StatusInternalServerError, "Database query failed")
		return
	}
{'fraction_non_alphanumeric': 0.11567655095638094, 'fraction_numerical': 0.004959012245724117, 'mean_word_length': 3.5059279525763793, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4676697', 'n_tokens_mistral': 7127, 'n_tokens_neox': 6542, 'n_words': 2025}