<commit_before>package uri

import (
	"errors"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"time"
)

var protocolRegistry map[string]reflect.Type

type Handler func(root Uri, u Uri) error

func init() {
	protocolRegistry = make(map[string]reflect.Type, 4)
	protocolRegistry["local"] = reflect.TypeOf((*UriLocal)(nil)).Elem()
	protocolRegistry["ftp"] = reflect.TypeOf((*UriFtp)(nil)).Elem()
}

type Uri interface {
	Scheme() string
	Host() string
	Path() string
	Uri() string
	Abs() string
	Parent() (Uri, error)

	Create(bool, os.FileMode) error
	OpenRead() (io.ReadCloser, error)
	OpenWrite() (io.WriteCloser, error)
	Remove() error
	Walk(dh, fh Handler) error //dh is the handler to handle a directory,and fh is the handler to handle a file.

	IsDir() bool
	Exist() bool
	IsAbs() bool

	Mode() os.FileMode
	ModTime() time.Time

	setHost(h string)
	setPath(p string)
	setScheme(s string)
}

func Parse(u string) (Uri, error) {
	urlp, err := url.Parse(u)
	if err != nil {
		return nil, ParseError{u, err.Error()}
	}

	UriType, exist := protocolRegistry[urlp.Scheme]
	if !exist {
		return nil, ProtocolError{urlp.Scheme, "protocol not supported."}
	}

	UriVal := reflect.New(UriType)
	i := UriVal.Interface()
	Urip, ok := i.(Uri)
	if !ok {
		return nil, ProtocolError{urlp.Scheme, "protocol not fully supported."}
	}
	Urip.setScheme(urlp.Scheme)
	Urip.setHost(urlp.Host)
	Urip.setPath(urlp.Path)

	return Urip, nil

}

type UriLocal struct {
	scheme string
	path   string
	host   string
}

func (u *UriLocal) Host() string {
	return u.host

}

func (u *UriLocal) Uri() string {
	return u.scheme + "://" + u.host + u.path
}

func (u *UriLocal) Abs() string {
	return u.host + u.path
}

func (u *UriLocal) IsAbs() bool {
	return filepath.IsAbs(u.Abs())
}

func (u *UriLocal) Scheme() string {
	return u.scheme
}

func (u *UriLocal) Mode() os.FileMode {
	fi, err := os.Stat(u.host + u.path)
	if err != nil {
		return os.ModePerm
	}
	return fi.Mode()
}

func (u *UriLocal) Exist() bool {
	fi, _ := os.Stat(u.Abs())
	if fi != nil {
		return true
	}
	return false
}

func (u *UriLocal) ModTime() time.Time {
	fi, err := os.Stat(u.host + u.path)
	if err != nil {
		return time.Now()
	}
	return fi.ModTime()
}

func (u *UriLocal) Create(IsDir bool, m os.FileMode) (err error) {

	if u.Exist() {
		return nil
	}

	if IsDir {
		err = os.Mkdir(u.Abs(), m)
		if err != nil {
			return
		}
	} else {
		var fd *os.File
		fd, err = os.OpenFile(u.Abs(), os.O_CREATE, m)
		defer fd.Close()
		if err != nil {
			return
		}
	}
	return nil
}

func (u *UriLocal) OpenRead() (io.ReadCloser, error) {
	AbsPath := u.host + u.path

	if !filepath.IsAbs(AbsPath) {
		return nil, OpenError{u.Uri(), "is not an absolute path."}
	}
	return os.OpenFile(AbsPath, os.O_RDONLY, u.Mode())

}
func (u *UriLocal) OpenWrite() (io.WriteCloser, error) {
	AbsPath := u.Abs()

	if !filepath.IsAbs(AbsPath) {
		return nil, OpenError{u.Uri(), "is not an absolute path."}
	}

	return os.OpenFile(AbsPath, os.O_WRONLY, u.Mode())

}

func (u *UriLocal) Remove() error {
	return os.Remove(u.Abs())
}

func (u *UriLocal) Walk(dh, fh Handler) error {
	if !u.IsDir() {
		return errors.New("walk " + u.Abs() + ": is not a directory")
	}

	err := filepath.Walk(u.Abs(),
		func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			path = strings.Replace(path, "\\", "/", -1)
			urip, err := Parse(u.Scheme() + "://" + path)
			if err != nil {
				return err
			}
			if urip.IsDir() {
				err = dh(u, urip)
			} else {
				err = fh(u, urip)
			}

			if err != nil {
				return err
			}
			return nil
		},
	)

	return err
}

func (u *UriLocal) Path() string {

	return u.path
}

func (u *UriLocal) Parent() (Uri, error) {
	p := filepath.Dir(u.Abs())
	p = strings.Replace(p, "\\", "/", -1)
	return Parse(u.Scheme() + "://" + p)
}

func (u *UriLocal) IsDir() bool {
	fi, _ := os.Stat(u.Abs())
	if fi == nil {
		return false
		//TODO
	}

	return fi.IsDir()
}

func (u *UriLocal) setHost(h string) {
	u.host = h
}
func (u *UriLocal) setPath(p string) {
	u.path = p
}
func (u *UriLocal) setScheme(s string) {
	u.scheme = s
}

type ParseError struct {
	Uri     string
	Message string
}

func (e ParseError) Error() string {
	return "Error parsing " + e.Uri + " on " + runtime.GOOS + ": " + e.Message
}

type ProtocolError struct {
	Protocol string
	Message  string
}

func (e ProtocolError) Error() string {
	return "When handling protocol " + e.Protocol + ": " + e.Message
}

type OpenError struct {
	Uri     string
	Message string
}

func (e OpenError) Error() string {
	return "Open " + e.Uri + "error: " + e.Message
}
<commit_msg>fix method ModTime.<commit_after>package uri

import (
	"errors"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"time"
)

var protocolRegistry map[string]reflect.Type

type Handler func(root Uri, u Uri) error

func init() {
	protocolRegistry = make(map[string]reflect.Type, 4)
	protocolRegistry["local"] = reflect.TypeOf((*UriLocal)(nil)).Elem()
	protocolRegistry["ftp"] = reflect.TypeOf((*UriFtp)(nil)).Elem()
}

type Uri interface {
	Scheme() string
	Host() string
	Path() string
	Uri() string
	Abs() string
	Parent() (Uri, error)

	Create(bool, os.FileMode) error
	OpenRead() (io.ReadCloser, error)
	OpenWrite() (io.WriteCloser, error)
	Remove() error
	Walk(dh, fh Handler) error //dh is the handler to handle a directory,and fh is the handler to handle a file.

	IsDir() bool
	Exist() bool
	IsAbs() bool

	Mode() os.FileMode
	ModTime() time.Time

	setHost(h string)
	setPath(p string)
	setScheme(s string)
}

func Parse(u string) (Uri, error) {
	urlp, err := url.Parse(u)
	if err != nil {
		return nil, ParseError{u, err.Error()}
	}

	UriType, exist := protocolRegistry[urlp.Scheme]
	if !exist {
		return nil, ProtocolError{urlp.Scheme, "protocol not supported."}
	}

	UriVal := reflect.New(UriType)
	i := UriVal.Interface()
	Urip, ok := i.(Uri)
	if !ok {
		return nil, ProtocolError{urlp.Scheme, "protocol not fully supported."}
	}
	Urip.setScheme(urlp.Scheme)
	Urip.setHost(urlp.Host)
	Urip.setPath(urlp.Path)

	return Urip, nil

}

type UriLocal struct {
	scheme string
	path   string
	host   string
}

func (u *UriLocal) Host() string {
	return u.host

}

func (u *UriLocal) Uri() string {
	return u.scheme + "://" + u.host + u.path
}

func (u *UriLocal) Abs() string {
	return u.host + u.path
}

func (u *UriLocal) IsAbs() bool {
	return filepath.IsAbs(u.Abs())
}

func (u *UriLocal) Scheme() string {
	return u.scheme
}

func (u *UriLocal) Mode() os.FileMode {
	fi, err := os.Stat(u.host + u.path)
	if err != nil {
		return os.ModePerm
	}
	return fi.Mode()
}

func (u *UriLocal) Exist() bool {
	fi, _ := os.Stat(u.Abs())
	if fi != nil {
		return true
	}
	return false
}

func (u *UriLocal) ModTime() time.Time {
	fi, _ := os.Stat(u.host + u.path)
	if fi == nil {
		return time.Date(1970, time.January, 1, 0, 0, 0, 0, time.Local)
	}
	return fi.ModTime()
}

func (u *UriLocal) Create(IsDir bool, m os.FileMode) (err error) {

	if u.Exist() {
		return nil
	}

	if IsDir {
		err = os.Mkdir(u.Abs(), m)
		if err != nil {
			return
		}
	} else {
		var fd *os.File
		fd, err = os.OpenFile(u.Abs(), os.O_CREATE, m)
		defer fd.Close()
		if err != nil {
			return
		}
	}
	return nil
}

func (u *UriLocal) OpenRead() (io.ReadCloser, error) {
	AbsPath := u.host + u.path

	if !filepath.IsAbs(AbsPath) {
		return nil, OpenError{u.Uri(), "is not an absolute path."}
	}
	return os.OpenFile(AbsPath, os.O_RDONLY, u.Mode())

}
func (u *UriLocal) OpenWrite() (io.WriteCloser, error) {
	AbsPath := u.Abs()

	if !filepath.IsAbs(AbsPath) {
		return nil, OpenError{u.Uri(), "is not an absolute path."}
	}

	return os.OpenFile(AbsPath, os.O_WRONLY, u.Mode())

}

func (u *UriLocal) Remove() error {
	return os.Remove(u.Abs())
}

func (u *UriLocal) Walk(dh, fh Handler) error {
	if !u.IsDir() {
		return errors.New("walk " + u.Abs() + ": is not a directory")
	}

	err := filepath.Walk(u.Abs(),
		func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			path = strings.Replace(path, "\\", "/", -1)
			urip, err := Parse(u.Scheme() + "://" + path)
			if err != nil {
				return err
			}
			if urip.IsDir() {
				err = dh(u, urip)
			} else {
				err = fh(u, urip)
			}

			if err != nil {
				return err
			}
			return nil
		},
	)

	return err
}

func (u *UriLocal) Path() string {

	return u.path
}

func (u *UriLocal) Parent() (Uri, error) {
	p := filepath.Dir(u.Abs())
	p = strings.Replace(p, "\\", "/", -1)
	return Parse(u.Scheme() + "://" + p)
}

func (u *UriLocal) IsDir() bool {
	fi, _ := os.Stat(u.Abs())
	if fi == nil {
		return false
		//TODO
	}

	return fi.IsDir()
}

func (u *UriLocal) setHost(h string) {
	u.host = h
}
func (u *UriLocal) setPath(p string) {
	u.path = p
}
func (u *UriLocal) setScheme(s string) {
	u.scheme = s
}

type ParseError struct {
	Uri     string
	Message string
}

func (e ParseError) Error() string {
	return "Error parsing " + e.Uri + " on " + runtime.GOOS + ": " + e.Message
}

type ProtocolError struct {
	Protocol string
	Message  string
}

func (e ProtocolError) Error() string {
	return "When handling protocol " + e.Protocol + ": " + e.Message
}

type OpenError struct {
	Uri     string
	Message string
}

func (e OpenError) Error() string {
	return "Open " + e.Uri + "error: " + e.Message
}

{'fraction_non_alphanumeric': 0.1636461126005362, 'fraction_numerical': 0.0016085790884718498, 'mean_word_length': 3.283876894809371, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136970', 'n_tokens_mistral': 3861, 'n_tokens_neox': 3657, 'n_words': 1002}