<commit_before>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package file

// package gt

import (
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"strings"

	"io/ioutil"
	"path/filepath"
)

// Read read file and return string
func Read(fileName string) (string, error) {
	fin, err := os.Open(fileName)
	if err != nil {
		// log.Println("os.Open: ", fileName, err)
		return "", err
	}
	defer fin.Close()

	var str string
	buf := make([]byte, 1024)
	for {
		n, _ := fin.Read(buf)
		if 0 == n {
			break
		}
		// os.Stdout.Write(buf[:n])
		strBuf := string(buf[:n])
		str += strBuf
	}

	return str, nil
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func WriteFile(fileName string, data []byte) error {
	os.MkdirAll(path.Dir(fileName), os.ModePerm)
	return ioutil.WriteFile(fileName, data, 0655)
}

// Write writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func Write(fileName, writeStr string) error {
	os.MkdirAll(path.Dir(fileName), os.ModePerm)

	fout, err := os.Create(fileName)
	if err != nil {
		// log.Println("Write file "+fileName, err)
		return err
	}
	defer fout.Close()

	_, err = fout.WriteString(writeStr)
	return err
}

// AppendTo append to file
func AppendTo(fileName, content string) error {
	// write only
	f, err := os.OpenFile(fileName, os.O_WRONLY, 0644)<commit_msg>add io.Reader type file operation<commit_after>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package file

// package gt

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"strings"

	"io/ioutil"
	"path/filepath"
)

// Read read file and return string
func Read(fileName string) (string, error) {
	fin, err := os.Open(fileName)
	if err != nil {
		// log.Println("os.Open: ", fileName, err)
		return "", err
	}
	defer fin.Close()

	var str string
	buf := make([]byte, 1024)
	for {
		n, _ := fin.Read(buf)
		if 0 == n {
			break
		}
		// os.Stdout.Write(buf[:n])
		strBuf := string(buf[:n])
		str += strBuf
	}

	return str, nil
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func WriteFile(fileName string, data []byte) error {
	os.MkdirAll(path.Dir(fileName), os.ModePerm)
	return ioutil.WriteFile(fileName, data, 0655)
}

// Write writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func Write(fileName, writeStr string) error {
	os.MkdirAll(path.Dir(fileName), os.ModePerm)

	fout, err := os.Create(fileName)
	if err != nil {
		// log.Println("Write file "+fileName, err)
		return err
	}
	defer fout.Close()

	_, err = fout.WriteString(writeStr)
	return err
}

// ReadIo read file return io.Reader
func ReadIo(fileName string) (io.Reader, error) {
	f, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	reader := bufio.NewReader(f)
	return reader, err
}

// WriteIo wite file with io.Reader
func WriteIo(fileName string, fio io.Reader) error {
	data, err := ioutil.ReadAll(fio)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(fileName, data, 0655)
}

// ReadFromIO read io.Reader return string and error
func ReadFromIO(fio io.Reader) (string, error) {
	b, err := ioutil.ReadAll(fio)
	if err != nil {
		return "", err
	}

	return string(b), nil
}

// AppendTo append to file
func AppendTo(fileName, content string) error {
	// write only
	f, err := os.OpenFile(fileName, os.O_WRONLY, 0644)
{'fraction_non_alphanumeric': 0.13354531001589826, 'fraction_numerical': 0.010447422212128094, 'mean_word_length': 3.4172517552657973, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5950983', 'n_tokens_mistral': 1583, 'n_tokens_neox': 1540, 'n_words': 498}