<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// vfs is a temporary copy of src/pkg/http/fs.go until http gets ServeFileFromFS
package vfs

import (
	"fmt"
	"io"
	"os"
	"http"
	"path/filepath"
	"mime"
	"strconv"
	"strings"
	"time"
	"utf8"
)

// The FileSystem interface represents the virtual filesystem subset
// used by ServeFileFromFs to serve files.
type FileSystem interface {
	Open(name string) (File, os.Error)
}

// The File interface is the subset of *os.File methods needed
// by ServeFileFromFs and the FileSystem interface.
type File interface {
	Close() os.Error
	Stat() (*os.FileInfo, os.Error)
	Readdir(count int) ([]os.FileInfo, os.Error)
	Read([]byte) (int, os.Error)
	Seek(offset int64, whence int) (int64, os.Error)
}

type osFileSystem struct{}

func (osFileSystem) Open(name string) (File, os.Error) {
	return os.Open(name)
}

func dirList(w http.ResponseWriter, f File) {
	fmt.Fprintf(w, "<pre>\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name
			if d.IsDirectory() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "<a href=\"%s\">%s</a>\n", name, name)
		}
	}
	fmt.Fprintf(w, "</pre>\n")
}

func serveFile(w http.ResponseWriter, r *http.Request, name string, fs FileSystem, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	if strings.HasSuffix(r.URL.Path, indexPage) {
		http.Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len(indexPage)+1], http.StatusMovedPermanently)
		return
	}

	f, err := fs.Open(name)
	if err != nil {
		// TODO expose actual error?
		http.NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		http.NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDirectory() {
			if url[len(url)-1] != '/' {
				http.Redirect(w, r, url+"/", http.StatusMovedPermanently)
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				http.Redirect(w, r, url[0:len(url)-1], http.StatusMovedPermanently)
				return
			}
		}
	}

	if t, _ := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Set("Last-Modified", time.SecondsToUTC(d.Mtime_ns/1e9).Format(http.TimeFormat))

	// use contents of index.html for directory, if present
	if d.IsDirectory() {
		index := name + filepath.FromSlash(indexPage)
		ff, err := fs.Open(index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDirectory() {
		dirList(w, f)
		return
	}

	// serve file
	size := d.Size
	code := http.StatusOK

	// If Content-Type isn't set, use the file's extension to find it.
	if w.Header().Get("Content-Type") == "" {
		ctype := mime.TypeByExtension(filepath.Ext(name))
		if ctype == "" {
			// read a chunk to decide between utf-8 text and binary
			var buf [1024]byte
			n, _ := io.ReadFull(f, buf[:])
			b := buf[:n]
			if isText(b) {
				ctype = "text/plain; charset=utf-8"
			} else {
				// generic binary
				ctype = "application/octet-stream"
			}
			f.Seek(0, os.SEEK_SET) // rewind to output whole file
		}
		w.Header().Set("Content-Type", ctype)
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	ranges, err := parseRange(r.Header.Get("Range"), size)
	if err == nil && len(ranges) > 1 {
		err = os.ErrorString("multiple ranges not supported")
	}
	if err != nil {
		http.Error(w, err.String(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if len(ranges) == 1 {
		ra := ranges[0]
		if _, err := f.Seek(ra.start, os.SEEK_SET); err != nil {<commit_msg>update for Go change<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// vfs is a temporary copy of src/pkg/http/fs.go until http gets ServeFileFromFS
package vfs

import (
	"fmt"
	"io"
	"os"
	"http"
	"path/filepath"
	"mime"
	"strconv"
	"strings"
	"time"
	"utf8"
)

// The FileSystem interface represents the virtual filesystem subset
// used by ServeFileFromFs to serve files.
type FileSystem interface {
	Open(name string) (File, os.Error)
}

// The File interface is the subset of *os.File methods needed
// by ServeFileFromFs and the FileSystem interface.
type File interface {
	Close() os.Error
	Stat() (*os.FileInfo, os.Error)
	Readdir(count int) ([]os.FileInfo, os.Error)
	Read([]byte) (int, os.Error)
	Seek(offset int64, whence int) (int64, os.Error)
}

type osFileSystem struct{}

func (osFileSystem) Open(name string) (File, os.Error) {
	return os.Open(name)
}

func dirList(w http.ResponseWriter, f File) {
	fmt.Fprintf(w, "<pre>\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name
			if d.IsDirectory() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "<a href=\"%s\">%s</a>\n", name, name)
		}
	}
	fmt.Fprintf(w, "</pre>\n")
}

func serveFile(w http.ResponseWriter, r *http.Request, name string, fs FileSystem, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	if strings.HasSuffix(r.URL.Path, indexPage) {
		http.Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len(indexPage)+1], http.StatusMovedPermanently)
		return
	}

	f, err := fs.Open(name)
	if err != nil {
		// TODO expose actual error?
		http.NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		http.NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDirectory() {
			if url[len(url)-1] != '/' {
				http.Redirect(w, r, url+"/", http.StatusMovedPermanently)
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				http.Redirect(w, r, url[0:len(url)-1], http.StatusMovedPermanently)
				return
			}
		}
	}

	if t, _ := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Set("Last-Modified", time.SecondsToUTC(d.Mtime_ns/1e9).Format(http.TimeFormat))

	// use contents of index.html for directory, if present
	if d.IsDirectory() {
		index := name + filepath.FromSlash(indexPage)
		ff, err := fs.Open(index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDirectory() {
		dirList(w, f)
		return
	}

	// serve file
	size := d.Size
	code := http.StatusOK

	// If Content-Type isn't set, use the file's extension to find it.
	if w.Header().Get("Content-Type") == "" {
		ctype := mime.TypeByExtension(filepath.Ext(name))
		if ctype == "" {
			// read a chunk to decide between utf-8 text and binary
			var buf [1024]byte
			n, _ := io.ReadFull(f, buf[:])
			b := buf[:n]
			if isText(b) {
				ctype = "text/plain; charset=utf-8"
			} else {
				// generic binary
				ctype = "application/octet-stream"
			}
			f.Seek(0, os.SEEK_SET) // rewind to output whole file
		}
		w.Header().Set("Content-Type", ctype)
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	ranges, err := parseRange(r.Header.Get("Range"), size)
	if err == nil && len(ranges) > 1 {
		err = os.NewError("multiple ranges not supported")
	}
	if err != nil {
		http.Error(w, err.String(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if len(ranges) == 1 {
		ra := ranges[0]
		if _, err := f.Seek(ra.start, os.SEEK_SET); err != nil {
{'fraction_non_alphanumeric': 0.14985060905538958, 'fraction_numerical': 0.008963456676626062, 'mean_word_length': 3.1921965317919074, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325351', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3125, 'n_words': 897}