<commit_before>#include <dirent.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/inotify.h>
#include <set>
#include <iostream>
#include <boost/log/trivial.hpp>
#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include "InotifyFileWatcher.hpp"
#include "../common/FormattedException.hpp"

using namespace std;
namespace logging = boost::log;

struct RC2::InotifyFileWatcher::FSObject 
{
	string			name;
	struct stat 	sb;
	int				wd;
	inline bool is_dir() const { return S_ISDIR(sb.st_mode); }
};

struct RC2::InotifyFileWatcher::Impl 
{
	int					inotifyFd;
	struct event_base*	eventBase;
	string				dirPath;
	map<int, FSObject>	files;
	FSObject			root;
	set<string>			addedFiles;
	set<string>			modifiedFiles;
	set<string>			deletedFiles;

	void watchFile(string fname) {
		FSObject fo;
		fo.name = fname;
		string fullPath = dirPath + "/" + fo.name;
		if (stat(fullPath.c_str(), &fo.sb) == -1)
			return;
		fo.wd = inotify_add_watch(inotifyFd, fullPath.c_str(), 
					IN_CLOSE_WRITE | IN_DELETE_SELF | IN_MODIFY);
		if (fo.wd == -1)
			throw FormattedException("inotify_add_warched failed %s", fo.name.c_str());
		files[fo.wd] = fo;
		BOOST_LOG_TRIVIAL(info) << "watching " << fo.name << endl;
	}
};


RC2::InotifyFileWatcher::InotifyFileWatcher(struct event_base *eb)
	: _impl(new Impl())
{
	_impl->eventBase = eb;
	_impl->inotifyFd = -1;
}

RC2::InotifyFileWatcher::~InotifyFileWatcher()
{
	if (_impl->inotifyFd != -1)
		close(_impl->inotifyFd);
}

void
RC2::InotifyFileWatcher::initializeWatcher(std::string dirPath)
{
	_impl->dirPath = dirPath;
	DIR *dir;
	class dirent *ent;

	_impl->inotifyFd = inotify_init();
	if (_impl->inotifyFd == -1)
		throw FormattedException("inotify_init() failed for %s", dirPath.c_str());
	_impl->root.wd = inotify_add_watch(_impl->inotifyFd, dirPath.c_str(), 
		IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF | IN_MOVE);
	if (_impl->root.wd == -1)
		throw FormattedException("inotify_add_watched failed");
	stat(dirPath.c_str(), &_impl->root.sb);
	_impl->files[_impl->root.wd] = _impl->root;
	BOOST_LOG_TRIVIAL(info) << "watching " << dirPath << endl;
	
	dir = opendir(dirPath.c_str());
	while ((ent = readdir(dir)) != NULL) {
		//skip "." and ".." but not other files that start with a period
		if (ent->d_name[0] == '.' && (strlen(ent->d_name) == 1 || ent->d_name[1] == '.'))
			continue;
		_impl->watchFile(ent->d_name);
	}
	closedir(dir);
	
	struct bufferevent *evt = bufferevent_socket_new(_impl->eventBase, _impl->inotifyFd, 0);
	bufferevent_setcb(evt, InotifyFileWatcher::handleInotifyEvent, NULL, NULL, static_cast<void*>(this));
	bufferevent_enable(evt, EV_READ);
}

#define I_BUF_LEN (10 * (sizeof(struct inotify_event) + NAME_MAX + 1))

void
RC2::InotifyFileWatcher::handleInotifyEvent(struct bufferevent *bev)
{
	BOOST_LOG_TRIVIAL(info) << "handleInotify called" << endl;
	char buf[I_BUF_LEN];
	size_t numRead = bufferevent_read(bev, buf, I_BUF_LEN);
	char *p;
	for (p=buf; p < buf + numRead; ) {
		struct inotify_event *event = (struct inotify_event*)p;
		int evtype = event->mask & 0xffff;
		cerr << "notify:" << std::hex << evtype << " for " << 
			_impl->files[event->wd].name << endl;
		if(evtype == IN_CREATE) {
			_impl->addedFiles.insert((string)event->name);
			_impl->watchFile(event->name);
			//need to add a watch for this file
		} else if (evtype == IN_CLOSE_WRITE) {
			_impl->modifiedFiles.insert((string)event->name);
		} else if (evtype == IN_DELETE_SELF) {
			cerr << "in delete called for " << event->name << endl;
			_impl->deletedFiles.insert((string)event->name);
			//discard our records of it
			inotify_rm_watch(_impl->inotifyFd, event->wd);
			_impl->files.erase(event->wd);
		}
		//handle event
		p += sizeof(struct inotify_event) + event->len;
	}
}

void
RC2::InotifyFileWatcher::startWatch()
{
	_impl->addedFiles.clear();<commit_msg>raises exception if stat call fails while starting to watch a file.<commit_after>#include <dirent.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/inotify.h>
#include <set>
#include <iostream>
#include <iomanip>
#include <boost/log/trivial.hpp>
#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/format.hpp>
#include "InotifyFileWatcher.hpp"
#include "../common/FormattedException.hpp"

using namespace std;
namespace logging = boost::log;
using boost::format;

struct RC2::InotifyFileWatcher::FSObject 
{
	string			name;
	struct stat 	sb;
	int				wd;
	inline bool is_dir() const { return S_ISDIR(sb.st_mode); }
};

struct RC2::InotifyFileWatcher::Impl 
{
	int					inotifyFd;
	struct event_base*	eventBase;
	string				dirPath;
	map<int, FSObject>	files;
	FSObject			root;
	set<string>			addedFiles;
	set<string>			modifiedFiles;
	set<string>			deletedFiles;
	
	class StatException : public runtime_error {
		using runtime_error::runtime_error;
	};

	void watchFile(string fname) {
		FSObject fo;
		fo.name = fname;
		string fullPath = dirPath + "/" + fo.name;
		if (stat(fullPath.c_str(), &fo.sb) == -1)
			throw StatException((format("stat failed for %s") % fo.name).str());
		fo.wd = inotify_add_watch(inotifyFd, fullPath.c_str(), 
					IN_CLOSE_WRITE | IN_DELETE_SELF | IN_MODIFY);
		if (fo.wd == -1)
			throw FormattedException("inotify_add_warched failed %s", fo.name.c_str());
		files[fo.wd] = fo;
		BOOST_LOG_TRIVIAL(info) << "watching " << fo.name << endl;
	}
};


RC2::InotifyFileWatcher::InotifyFileWatcher(struct event_base *eb)
	: _impl(new Impl())
{
	_impl->eventBase = eb;
	_impl->inotifyFd = -1;
}

RC2::InotifyFileWatcher::~InotifyFileWatcher()
{
	if (_impl->inotifyFd != -1)
		close(_impl->inotifyFd);
}

void
RC2::InotifyFileWatcher::initializeWatcher(std::string dirPath)
{
	_impl->dirPath = dirPath;
	DIR *dir;
	class dirent *ent;

	_impl->inotifyFd = inotify_init();
	if (_impl->inotifyFd == -1)
		throw FormattedException("inotify_init() failed for %s", dirPath.c_str());
	_impl->root.wd = inotify_add_watch(_impl->inotifyFd, dirPath.c_str(), 
		IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF | IN_MOVE);
	if (_impl->root.wd == -1)
		throw FormattedException("inotify_add_watched failed");
	stat(dirPath.c_str(), &_impl->root.sb);
	_impl->files[_impl->root.wd] = _impl->root;
	BOOST_LOG_TRIVIAL(info) << "watching " << dirPath << endl;
	
	dir = opendir(dirPath.c_str());
	while ((ent = readdir(dir)) != NULL) {
		//skip "." and ".." but not other files that start with a period
		if (ent->d_name[0] == '.' && (strlen(ent->d_name) == 1 || ent->d_name[1] == '.'))
			continue;
		try {
			_impl->watchFile(ent->d_name);
		} catch (Impl::StatException &se) {
		}
	}
	closedir(dir);
	
	struct bufferevent *evt = bufferevent_socket_new(_impl->eventBase, _impl->inotifyFd, 0);
	bufferevent_setcb(evt, InotifyFileWatcher::handleInotifyEvent, NULL, NULL, static_cast<void*>(this));
	bufferevent_enable(evt, EV_READ);
}

#define I_BUF_LEN (10 * (sizeof(struct inotify_event) + NAME_MAX + 1))

void
RC2::InotifyFileWatcher::handleInotifyEvent(struct bufferevent *bev)
{
	BOOST_LOG_TRIVIAL(info) << "handleInotify called" << endl;
	char buf[I_BUF_LEN];
	size_t numRead = bufferevent_read(bev, buf, I_BUF_LEN);
	char *p;
	for (p=buf; p < buf + numRead; ) {
		struct inotify_event *event = (struct inotify_event*)p;
		int evtype = event->mask & 0xffff; //events are in lower word, flags in upper
		BOOST_LOG_TRIVIAL(info) << "notify:" << std::hex << evtype << " for " << 
			_impl->files[event->wd].name << endl;
		if(evtype == IN_CREATE) {
			_impl->addedFiles.insert((string)event->name);
			_impl->watchFile(event->name);
			//need to add a watch for this file
		} else if (evtype == IN_CLOSE_WRITE) {
			_impl->modifiedFiles.insert((string)event->name);
		} else if (evtype == IN_DELETE_SELF) {
			_impl->deletedFiles.insert((string)event->name);
			//discard our records of it
			inotify_rm_watch(_impl->inotifyFd, event->wd);
			_impl->files.erase(event->wd);
		}
		//handle event
		p += sizeof(struct inotify_event) + event->len;
	}
}

void
RC2::InotifyFileWatcher::startWatch()
{
	_impl->addedFiles.clear();
{'fraction_non_alphanumeric': 0.16069868995633188, 'fraction_numerical': 0.005240174672489083, 'mean_word_length': 4.758620689655173, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 110, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2614379', 'n_tokens_mistral': 3321, 'n_tokens_neox': 3158, 'n_words': 602}