<commit_before>/*
 * MacFileMonitor.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/system/FileMonitor.hpp>

#include <dirent.h>
#include <sys/stat.h>

#include <CoreServices/CoreServices.h>

#include <list>
#include <algorithm>

#include <boost/foreach.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/classification.hpp>

#include <core/Log.hpp>
#include <core/Error.hpp>
#include <core/FileInfo.hpp>
#include <core/Thread.hpp>

#include <core/system/System.hpp>

namespace core {
namespace system {
namespace file_monitor {

namespace {

int entryFilter(struct dirent *entry)
{
   if (::strcmp(entry->d_name, ".") == 0 || ::strcmp(entry->d_name, "..") == 0)
      return 0;
   else
      return 1;
}

Error scan(tree<FileInfo>* pTree,
           const tree<FileInfo>::iterator_base& node,
           bool recursive)
{
   // clear all existing
   pTree->erase_children(node);

   // create FilePath for root
   FilePath rootPath(node->absolutePath());

   // read directory contents
   struct dirent **namelist;
   int entries = ::scandir(node->absolutePath().c_str(),
                           &namelist,
                           entryFilter,
                           ::alphasort);
   if (entries == -1)
   {
      Error error = systemError(boost::system::errc::no_such_file_or_directory,
                                ERROR_LOCATION);
      error.addProperty("path", node->absolutePath());
      return error;
   }

   // iterate over entries
   for(int i=0; i<entries; i++)
   {
      // get the entry (then free it) and compute the path
      dirent entry = *namelist[i];
      ::free(namelist[i]);
      std::string name(entry.d_name, entry.d_namlen);
      std::string path = rootPath.childPath(name).absolutePath();

      // get the attributes
      struct stat st;
      int res = ::lstat(path.c_str(), &st);
      if (res == -1)
      {
         LOG_ERROR(systemError(errno, ERROR_LOCATION));
         continue;
      }

      // add the correct type of FileEntry
      if ( S_ISDIR(st.st_mode))
      {
         tree<FileInfo>::iterator_base child =
                              pTree->append_child(node, FileInfo(path, true));
         if (recursive)
         {
            Error error = scan(pTree, child, true);
            if (error)
            {
               LOG_ERROR(error);
               continue;
            }
         }
      }
      else
      {
         pTree->append_child(node, FileInfo(path,
                                            false,
                                            st.st_size,
                                            st.st_mtimespec.tv_sec));
      }
   }

   // free the namelist
   ::free(namelist);

   // return success
   return Success();
}

class FileEventContext : boost::noncopyable
{
public:
   FileEventContext() : streamRef(NULL) {}
   virtual ~FileEventContext() {}
   FSEventStreamRef streamRef;
   tree<FileInfo> fileTree;
   Callbacks::FilesChanged onFilesChanged;
};

inline int fileInfoCompare(const FileInfo& a, const FileInfo& b)
{
   // use stcoll because that is what alphasort (comp function passed to
   // scandir) uses for its sorting)
   int result = ::strcoll(a.absolutePath().c_str(), b.absolutePath().c_str());

   if (result != 0)
      return result;

   if (a.isDirectory() == b.isDirectory())
      return 0;

   return a.isDirectory() ? -1 : 1;
}

inline bool fileInfoComparePaths(const FileInfo& a, const FileInfo& b)
{
   return fileInfoCompare(a, b) < 0;
}

template<typename PreviousIterator, typename CurrentIterator>
void collectFileChangeEvents(PreviousIterator prevBegin,
                             PreviousIterator prevEnd,
                             CurrentIterator currBegin,
                             CurrentIterator currEnd,
                             std::vector<FileChangeEvent>* pEvents)
{
   FileInfo noFile;
   while (prevBegin != prevEnd || currBegin != currEnd)
   {
      const FileInfo& prevFile = prevBegin != prevEnd ? *prevBegin : noFile;
      const FileInfo& currFile = currBegin != currEnd ? *currBegin : noFile;

      int comp;
      if (prevFile.empty())
         comp = 1;
      else if (currFile.empty())
         comp = -1;
      else
         comp = fileInfoCompare(prevFile, currFile);

      if (comp == 0)
      {
         if (currFile.lastWriteTime() != prevFile.lastWriteTime())
         {
            pEvents->push_back(FileChangeEvent(FileChangeEvent::FileModified,
                                               currFile));
         }
         prevBegin++;
         currBegin++;
      }
      else if (comp < 0)
      {
         pEvents->push_back(FileChangeEvent(FileChangeEvent::FileRemoved,
                                            prevFile));
         prevBegin++;
      }
      else // comp > 1
      {
         pEvents->push_back(FileChangeEvent(FileChangeEvent::FileAdded,
                                            currFile));
         currBegin++;
      }
   }
}

void addEvent(FileChangeEvent::Type type,
              const FileInfo& fileInfo,
              std::vector<FileChangeEvent>* pEvents)
{
   pEvents->push_back(FileChangeEvent(type, fileInfo));
}
<commit_msg>sort ranges before collecting file change events<commit_after>/*
 * MacFileMonitor.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/system/FileMonitor.hpp>

#include <dirent.h>
#include <sys/stat.h>

#include <CoreServices/CoreServices.h>

#include <list>
#include <algorithm>

#include <boost/foreach.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/classification.hpp>

#include <core/Log.hpp>
#include <core/Error.hpp>
#include <core/FileInfo.hpp>
#include <core/Thread.hpp>

#include <core/system/System.hpp>

namespace core {
namespace system {
namespace file_monitor {

namespace {

int entryFilter(struct dirent *entry)
{
   if (::strcmp(entry->d_name, ".") == 0 || ::strcmp(entry->d_name, "..") == 0)
      return 0;
   else
      return 1;
}

Error scan(tree<FileInfo>* pTree,
           const tree<FileInfo>::iterator_base& node,
           bool recursive)
{
   // clear all existing
   pTree->erase_children(node);

   // create FilePath for root
   FilePath rootPath(node->absolutePath());

   // read directory contents
   struct dirent **namelist;
   int entries = ::scandir(node->absolutePath().c_str(),
                           &namelist,
                           entryFilter,
                           ::alphasort);
   if (entries == -1)
   {
      Error error = systemError(boost::system::errc::no_such_file_or_directory,
                                ERROR_LOCATION);
      error.addProperty("path", node->absolutePath());
      return error;
   }

   // iterate over entries
   for(int i=0; i<entries; i++)
   {
      // get the entry (then free it) and compute the path
      dirent entry = *namelist[i];
      ::free(namelist[i]);
      std::string name(entry.d_name, entry.d_namlen);
      std::string path = rootPath.childPath(name).absolutePath();

      // get the attributes
      struct stat st;
      int res = ::lstat(path.c_str(), &st);
      if (res == -1)
      {
         LOG_ERROR(systemError(errno, ERROR_LOCATION));
         continue;
      }

      // add the correct type of FileEntry
      if ( S_ISDIR(st.st_mode))
      {
         tree<FileInfo>::iterator_base child =
                              pTree->append_child(node, FileInfo(path, true));
         if (recursive)
         {
            Error error = scan(pTree, child, true);
            if (error)
            {
               LOG_ERROR(error);
               continue;
            }
         }
      }
      else
      {
         pTree->append_child(node, FileInfo(path,
                                            false,
                                            st.st_size,
                                            st.st_mtimespec.tv_sec));
      }
   }

   // free the namelist
   ::free(namelist);

   // return success
   return Success();
}

class FileEventContext : boost::noncopyable
{
public:
   FileEventContext() : streamRef(NULL) {}
   virtual ~FileEventContext() {}
   FSEventStreamRef streamRef;
   tree<FileInfo> fileTree;
   Callbacks::FilesChanged onFilesChanged;
};

inline int fileInfoCompare(const FileInfo& a, const FileInfo& b)
{
   // use stcoll because that is what alphasort (comp function passed to
   // scandir) uses for its sorting)
   int result = ::strcoll(a.absolutePath().c_str(), b.absolutePath().c_str());

   if (result != 0)
      return result;

   if (a.isDirectory() == b.isDirectory())
      return 0;

   return a.isDirectory() ? -1 : 1;
}

inline bool fileInfoComparePaths(const FileInfo& a, const FileInfo& b)
{
   return fileInfoCompare(a, b) < 0;
}

template<typename PreviousIterator, typename CurrentIterator>
void collectFileChangeEvents(PreviousIterator prevBegin,
                             PreviousIterator prevEnd,
                             CurrentIterator currBegin,
                             CurrentIterator currEnd,
                             std::vector<FileChangeEvent>* pEvents)
{
   // sort the ranges
   std::vector<FileInfo> prev;
   std::copy(prevBegin, prevEnd, std::back_inserter(prev));
   std::sort(prev.begin(), prev.end(), fileInfoComparePaths);
   std::vector<FileInfo> curr;
   std::copy(currBegin, currEnd, std::back_inserter(curr));
   std::sort(curr.begin(), curr.end(), fileInfoComparePaths);

   // initalize the iterators
   std::vector<FileInfo>::iterator prevIt = prev.begin();
   std::vector<FileInfo>::iterator currIt = curr.begin();

   FileInfo noFile;
   while (prevIt != prev.end() || currIt != curr.end())
   {
      const FileInfo& prevFile = prevIt != prev.end() ? *prevIt : noFile;
      const FileInfo& currFile = currIt != curr.end() ? *currIt : noFile;

      int comp;
      if (prevFile.empty())
         comp = 1;
      else if (currFile.empty())
         comp = -1;
      else
         comp = fileInfoCompare(prevFile, currFile);

      if (comp == 0)
      {
         if (currFile.lastWriteTime() != prevFile.lastWriteTime())
         {
            pEvents->push_back(FileChangeEvent(FileChangeEvent::FileModified,
                                               currFile));
         }
         prevIt++;
         currIt++;
      }
      else if (comp < 0)
      {
         pEvents->push_back(FileChangeEvent(FileChangeEvent::FileRemoved,
                                            prevFile));
         prevIt++;
      }
      else // comp > 1
      {
         pEvents->push_back(FileChangeEvent(FileChangeEvent::FileAdded,
                                            currFile));
         currIt++;
      }
   }
}

void addEvent(FileChangeEvent::Type type,
              const FileInfo& fileInfo,
              std::vector<FileChangeEvent>* pEvents)
{
   pEvents->push_back(FileChangeEvent(type, fileInfo));
}

{'fraction_non_alphanumeric': 0.11659347283217085, 'fraction_numerical': 0.0044777404632739175, 'mean_word_length': 2.1061781224926452, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6608585', 'n_tokens_mistral': 3399, 'n_tokens_neox': 3327, 'n_words': 794}