<commit_before>package rdirsync

import (
	"io"
	"log"
	"os"

	"bitbucket.org/hnakamur/rdirsync/rpc"
)

type server struct{}

func NewServer() rpc.RDirSyncServer {
	return new(server)
}

func (s *server) FetchFile(req *rpc.FetchRequest, stream rpc.RDirSync_FetchFileServer) error {
	log.Printf("FetchFile start. path=%s, bufSize=%d", req.Path, req.BufSize)
	defer log.Printf("FetchFile exit.")
	buf := make([]byte, req.BufSize)
	file, err := os.Open(req.Path)
	if err != nil {
		log.Printf("failed to open file; err=%+v", err)
		return err
	}
	defer file.Close()

	for {
		n, err := io.ReadFull(file, buf)
		log.Printf("after ReadFull. n=%d, err=%+v", n, err)
		if err == io.EOF {
			break
		} else if err == io.ErrUnexpectedEOF {
			buf = buf[:n]
		} else if err != nil {
			return err
		}

		err = stream.Send(&rpc.FileChunk{Chunk: buf})
		log.Printf("after stream.Send. err=%+v", err)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *server) ReadDir(req *rpc.ReadDirRequest, stream rpc.RDirSync_ReadDirServer) error {
	log.Printf("ReadDir start. path=%s, atMostCount=%d", req.Path, req.AtMostCount)
	defer log.Printf("ReadDir exit.")
	file, err := os.Open(req.Path)
	if err != nil {
		return err
	}
	defer file.Close()

	for {
		osFileInfos, err := file.Readdir(int(req.AtMostCount))
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		infos := newFileInfosFromOS(osFileInfos)
		err = stream.Send(&rpc.FileInfos{Infos: infos})
		if err != nil {
			return err
		}
	}

	return nil
}

func newFileInfosFromOS(fis []os.FileInfo) []*rpc.FileInfo {
	infos := make([]*rpc.FileInfo, 0, len(fis))
	for _, fi := range fis {
		infos = append(infos, newFileInfoFromOS(fi))
	}
	return infos
}

func newFileInfoFromOS(fi os.FileInfo) *rpc.FileInfo {
	return &rpc.FileInfo{
		Name:    fi.Name(),
		Size:    fi.Size(),
		Mode:    int32(fi.Mode()),
		ModTime: fi.ModTime().Unix(),
	}
}
<commit_msg>Change ReadDir to return only regular files and directories<commit_after>package rdirsync

import (
	"io"
	"log"
	"os"

	"bitbucket.org/hnakamur/rdirsync/rpc"
)

type server struct{}

func NewServer() rpc.RDirSyncServer {
	return new(server)
}

func (s *server) FetchFile(req *rpc.FetchRequest, stream rpc.RDirSync_FetchFileServer) error {
	log.Printf("FetchFile start. path=%s, bufSize=%d", req.Path, req.BufSize)
	defer log.Printf("FetchFile exit.")
	buf := make([]byte, req.BufSize)
	file, err := os.Open(req.Path)
	if err != nil {
		log.Printf("failed to open file; err=%+v", err)
		return err
	}
	defer file.Close()

	for {
		n, err := io.ReadFull(file, buf)
		log.Printf("after ReadFull. n=%d, err=%+v", n, err)
		if err == io.EOF {
			break
		} else if err == io.ErrUnexpectedEOF {
			buf = buf[:n]
		} else if err != nil {
			return err
		}

		err = stream.Send(&rpc.FileChunk{Chunk: buf})
		log.Printf("after stream.Send. err=%+v", err)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *server) ReadDir(req *rpc.ReadDirRequest, stream rpc.RDirSync_ReadDirServer) error {
	log.Printf("ReadDir start. path=%s, atMostCount=%d", req.Path, req.AtMostCount)
	defer log.Printf("ReadDir exit.")
	file, err := os.Open(req.Path)
	if err != nil {
		return err
	}
	defer file.Close()

	for {
		osFileInfos, err := file.Readdir(int(req.AtMostCount))
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		infos := newFileInfosFromOS(osFileInfos)
		err = stream.Send(&rpc.FileInfos{Infos: infos})
		if err != nil {
			return err
		}
	}

	return nil
}

func newFileInfosFromOS(fis []os.FileInfo) []*rpc.FileInfo {
	infos := make([]*rpc.FileInfo, 0, len(fis))
	for _, fi := range fis {
		if fi.IsDir() || fi.Mode().IsRegular() {
			infos = append(infos, newFileInfoFromOS(fi))
		}
	}
	return infos
}

func newFileInfoFromOS(fi os.FileInfo) *rpc.FileInfo {
	return &rpc.FileInfo{
		Name:    fi.Name(),
		Size:    fi.Size(),
		Mode:    int32(fi.Mode()),
		ModTime: fi.ModTime().Unix(),
	}
}

{'fraction_non_alphanumeric': 0.1596446700507614, 'fraction_numerical': 0.0015228426395939086, 'mean_word_length': 3.6093567251461987, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1907347', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1547, 'n_words': 363}