<commit_before>// Copyright (c) 2012, <NAME> <<EMAIL>>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"bytes"
	"fmt"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/errors"
)

// FileStorage provide implementation of file-system backed storage.
type FileStorage struct {
	path string
	lock *os.File
	log  *os.File
	buf  []byte
	mu   sync.Mutex
}

// OpenFile creates new initialized FileStorage for given path. This will also
// hold file lock; thus any subsequent attempt to open same file path will
// fail.
func OpenFile(dbpath string) (d *FileStorage, err error) {
	err = os.MkdirAll(dbpath, 0755)
	if err != nil {
		return
	}

	lock, err := os.OpenFile(path.Join(dbpath, "LOCK"), os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return
	}

	defer func() {
		if err != nil {
			lock.Close()
		}
	}()

	err = setFileLock(lock, true)
	if err != nil {
		return
	}

	os.Rename(path.Join(dbpath, "LOG"), path.Join(dbpath, "LOG.old"))
	log, err := os.OpenFile(path.Join(dbpath, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		setFileLock(lock, false)
		return
	}

	d = &FileStorage{path: dbpath, lock: lock, log: log}
	runtime.SetFinalizer(d, (*FileStorage).Close)

	return
}

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *[]byte, i int, wid int) {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		*buf = append(*buf, '0')
		return
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	*buf = append(*buf, b[bp:]...)
}

// Print write given str to the log file.
func (d *FileStorage) Print(str string) {
	t := time.Now()
	year, month, day := t.Date()
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	d.mu.Lock()
	d.buf = d.buf[:0]

	// date
	itoa(&d.buf, year, 4)
	d.buf = append(d.buf, '/')
	itoa(&d.buf, int(month), 2)
	d.buf = append(d.buf, '/')
	itoa(&d.buf, day, 4)
	d.buf = append(d.buf, ' ')

	// time
	itoa(&d.buf, hour, 2)
	d.buf = append(d.buf, ':')
	itoa(&d.buf, min, 2)
	d.buf = append(d.buf, ':')
	itoa(&d.buf, sec, 2)
	d.buf = append(d.buf, '.')
	itoa(&d.buf, msec, 6)
	d.buf = append(d.buf, ' ')

	// write
	d.log.Write(d.buf)
	d.log.WriteString(str)
	d.log.WriteString("\n")

	d.mu.Unlock()
}

// GetFile get file with given number and type.
func (d *FileStorage) GetFile(number uint64, t FileType) File {
	return &file{stor: d, num: number, t: t}
}

// GetFiles get all files that match given file types; multiple file
// type may OR'ed together.
func (d *FileStorage) GetFiles(t FileType) (r []File) {
	dir, err := os.Open(d.path)
	if err != nil {
		return
	}
	names, err := dir.Readdirnames(0)
	dir.Close()
	if err != nil {
		return
	}
	p := &file{stor: d}
	for _, name := range names {
		if p.parse(name) && (p.t&t) != 0 {
			r = append(r, p)
			p = &file{stor: d}
		}
	}
	return
}

// GetManifest get manifest file.
func (d *FileStorage) GetManifest() (f File, err error) {
	pth := path.Join(d.path, "CURRENT")
	rw, err := os.OpenFile(pth, os.O_RDONLY, 0)
	if err != nil {
		err = err.(*os.PathError).Err
		return
	}
	defer rw.Close()
	buf := new(bytes.Buffer)
	_, err = buf.ReadFrom(rw)
	if err != nil {
		return
	}
	b := buf.Bytes()
	p := &file{stor: d}
	if len(b) < 1 || b[len(b)-1] != '\n' || !p.parse(string(b[:len(b)-1])) {
		return nil, errors.ErrCorrupt("invalid CURRENT file")
	}
	return p, nil
}

// SetManifest set manifest to given file.
func (d *FileStorage) SetManifest(f File) (err error) {
	p, ok := f.(*file)
	if !ok {
		return ErrInvalidFile
	}
	pth := path.Join(d.path, "CURRENT")
	pthTmp := fmt.Sprintf("%s.%d", pth, p.num)
	rw, err := os.OpenFile(pthTmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer rw.Close()
	_, err = fmt.Fprintln(rw, p.name())
	if err != nil {
		return
	}
	return os.Rename(pthTmp, pth)
}

// Close closes the storage and release the lock.
func (d *FileStorage) Close() error {
	if err := setFileLock(d.lock, false); err != nil {
		return err
	}
	return d.lock.Close()
}

type file struct {
	stor *FileStorage
	num  uint64<commit_msg>storage: FileStorage: fixes not properly closed files<commit_after>// Copyright (c) 2012, <NAME> <<EMAIL>>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"bytes"
	"fmt"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/errors"
)

// FileStorage provide implementation of file-system backed storage.
type FileStorage struct {
	path string
	lock *os.File
	log  *os.File
	buf  []byte
	mu   sync.Mutex
}

// OpenFile creates new initialized FileStorage for given path. This will also
// hold file lock; thus any subsequent attempt to open same file path will
// fail.
func OpenFile(dbpath string) (d *FileStorage, err error) {
	err = os.MkdirAll(dbpath, 0755)
	if err != nil {
		return
	}

	lock, err := os.OpenFile(path.Join(dbpath, "LOCK"), os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return
	}

	defer func() {
		if err != nil {
			lock.Close()
		}
	}()

	err = setFileLock(lock, true)
	if err != nil {
		return
	}

	os.Rename(path.Join(dbpath, "LOG"), path.Join(dbpath, "LOG.old"))
	log, err := os.OpenFile(path.Join(dbpath, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		setFileLock(lock, false)
		return
	}

	d = &FileStorage{path: dbpath, lock: lock, log: log}
	runtime.SetFinalizer(d, (*FileStorage).Close)

	return
}

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *[]byte, i int, wid int) {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		*buf = append(*buf, '0')
		return
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	*buf = append(*buf, b[bp:]...)
}

// Print write given str to the log file.
func (d *FileStorage) Print(str string) {
	t := time.Now()
	year, month, day := t.Date()
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	d.mu.Lock()
	d.buf = d.buf[:0]

	// date
	itoa(&d.buf, year, 4)
	d.buf = append(d.buf, '/')
	itoa(&d.buf, int(month), 2)
	d.buf = append(d.buf, '/')
	itoa(&d.buf, day, 4)
	d.buf = append(d.buf, ' ')

	// time
	itoa(&d.buf, hour, 2)
	d.buf = append(d.buf, ':')
	itoa(&d.buf, min, 2)
	d.buf = append(d.buf, ':')
	itoa(&d.buf, sec, 2)
	d.buf = append(d.buf, '.')
	itoa(&d.buf, msec, 6)
	d.buf = append(d.buf, ' ')

	// write
	d.log.Write(d.buf)
	d.log.WriteString(str)
	d.log.WriteString("\n")

	d.mu.Unlock()
}

// GetFile get file with given number and type.
func (d *FileStorage) GetFile(number uint64, t FileType) File {
	return &file{stor: d, num: number, t: t}
}

// GetFiles get all files that match given file types; multiple file
// type may OR'ed together.
func (d *FileStorage) GetFiles(t FileType) (r []File) {
	dir, err := os.Open(d.path)
	if err != nil {
		return
	}
	names, err := dir.Readdirnames(0)
	dir.Close()
	if err != nil {
		return
	}
	p := &file{stor: d}
	for _, name := range names {
		if p.parse(name) && (p.t&t) != 0 {
			r = append(r, p)
			p = &file{stor: d}
		}
	}
	return
}

// GetManifest get manifest file.
func (d *FileStorage) GetManifest() (f File, err error) {
	pth := path.Join(d.path, "CURRENT")
	rw, err := os.OpenFile(pth, os.O_RDONLY, 0)
	if err != nil {
		err = err.(*os.PathError).Err
		return
	}
	defer rw.Close()
	buf := new(bytes.Buffer)
	_, err = buf.ReadFrom(rw)
	if err != nil {
		return
	}
	b := buf.Bytes()
	p := &file{stor: d}
	if len(b) < 1 || b[len(b)-1] != '\n' || !p.parse(string(b[:len(b)-1])) {
		return nil, errors.ErrCorrupt("invalid CURRENT file")
	}
	return p, nil
}

// SetManifest set manifest to given file.
func (d *FileStorage) SetManifest(f File) (err error) {
	p, ok := f.(*file)
	if !ok {
		return ErrInvalidFile
	}
	pth := path.Join(d.path, "CURRENT")
	pthTmp := fmt.Sprintf("%s.%d", pth, p.num)
	rw, err := os.OpenFile(pthTmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	_, err = fmt.Fprintln(rw, p.name())
	if err != nil {
		rw.Close()
		return
	}
	rw.Close()
	return os.Rename(pthTmp, pth)
}

// Close closes the storage and release the lock.
func (d *FileStorage) Close() error {
	d.log.Close()
	if err := setFileLock(d.lock, false); err != nil {
		return err
	}
	return d.lock.Close()
}

type file struct {
	stor *FileStorage
	num  uint64
{'fraction_non_alphanumeric': 0.17561935633248438, 'fraction_numerical': 0.012039824033341051, 'mean_word_length': 3.1774661508704063, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '794970', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3529, 'n_words': 990}