<commit_before>/* Miscellaneous function handlers. */
package v3

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
)

// Flush and close all data files and shutdown entire program.
func Shutdown(w http.ResponseWriter, r *http.Request) {
	V3Sync.Lock()
	defer V3Sync.Unlock()
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	V3DB.Close()
	os.Exit(0)
}

// Pause all activities and make a dump of entire database to another file system location.
func Dump(w http.ResponseWriter, r *http.Request) {
	V3Sync.Lock()
	defer V3Sync.Unlock()
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	var dest string
	if !Require(w, r, "dest", &dest) {
		return
	}
	// Note that symbol links are skipped!
	walkFun := func(currPath string, info os.FileInfo, err error) error {
		if info.IsDir() {
			// Calculate directory path at destination and create it
			relPath, err := filepath.Rel(V3DB.Dir, currPath)
			if err != nil {
				return err
			}
			destDir := path.Join(dest, relPath)
			if err := os.MkdirAll(destDir, 0700); err != nil {
				return err
			}
			log.Printf("Dump created directory %s with permission 0700", destDir)
		} else {
			// Open the file to be copied (collection data/index)
			src, err := os.Open(currPath)
			if err != nil {
				return err
			}
			// Calculate file path at destination and create it
			relPath, err := filepath.Rel(V3DB.Dir, currPath)
			if err != nil {
				return err
			}
			destPath := path.Join(dest, relPath)
			destFile, err := os.Create(destPath)
			if err != nil {
				return err
			}
			// Copy from source to destination
			written, err := io.Copy(destFile, src)
			if err != nil {
				return err
			}
			log.Printf("Dump create file %s with permission 666 (before umask), size is %d", destPath, written)
		}
		return nil
	}
	err := filepath.Walk(V3DB.Dir, walkFun)
	if err != nil {
		http.Error(w, fmt.Sprint(err), 500)
	}
}

// Return server memory statistics.
func MemStats(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	stats := new(runtime.MemStats)
	runtime.ReadMemStats(stats)
	resp, err := json.Marshal(stats)
	if err != nil {
		http.Error(w, "Cannot serialize MemStats to JSON.", 500)
		return
	}
	w.Write(resp)
}

// Return server protocol version number.
func Version(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("3"))
}<commit_msg>sync to file system before dumping database<commit_after>/* Miscellaneous function handlers. */
package v3

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
)

// Flush and close all data files and shutdown entire program.
func Shutdown(w http.ResponseWriter, r *http.Request) {
	V3Sync.Lock()
	defer V3Sync.Unlock()
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	V3DB.Close()
	os.Exit(0)
}

// Pause all activities and make a dump of entire database to another file system location.
func Dump(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	var dest string
	if !Require(w, r, "dest", &dest) {
		return
	}
	// Note that symbol links are skipped!
	walkFun := func(currPath string, info os.FileInfo, err error) error {
		if info.IsDir() {
			// Calculate directory path at destination and create it
			relPath, err := filepath.Rel(V3DB.Dir, currPath)
			if err != nil {
				return err
			}
			destDir := path.Join(dest, relPath)
			if err := os.MkdirAll(destDir, 0700); err != nil {
				return err
			}
			log.Printf("Dump created directory %s with permission 0700", destDir)
		} else {
			// Open the file to be copied (collection data/index)
			src, err := os.Open(currPath)
			if err != nil {
				return err
			}
			// Calculate file path at destination and create it
			relPath, err := filepath.Rel(V3DB.Dir, currPath)
			if err != nil {
				return err
			}
			destPath := path.Join(dest, relPath)
			destFile, err := os.Create(destPath)
			if err != nil {
				return err
			}
			// Copy from source to destination
			written, err := io.Copy(destFile, src)
			if err != nil {
				return err
			}
			log.Printf("Dump create file %s with permission 666 (before umask), size is %d", destPath, written)
		}
		return nil
	}
	V3Sync.Lock()
	defer V3Sync.Unlock()
	V3DB.Flush()
	err := filepath.Walk(V3DB.Dir, walkFun)
	if err != nil {
		http.Error(w, fmt.Sprint(err), 500)
	}
}

// Return server memory statistics.
func MemStats(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "application/json")
	stats := new(runtime.MemStats)
	runtime.ReadMemStats(stats)
	resp, err := json.Marshal(stats)
	if err != nil {
		http.Error(w, "Cannot serialize MemStats to JSON.", 500)
		return
	}
	w.Write(resp)
}

// Return server protocol version number.
func Version(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("3"))
}
{'fraction_non_alphanumeric': 0.14704783258594917, 'fraction_numerical': 0.010650224215246636, 'mean_word_length': 3.5752136752136754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6025073', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1856, 'n_words': 525}