<commit_before>/* Copyright 2012-present Facebook, Inc.
 * Licensed under the Apache License, Version 2.0 */

#include "watchman_system.h"
#include "watchman.h"
#include <folly/String.h>
#include <exception>
#include <optional>

namespace watchman {

CookieSync::Cookie::Cookie(uint64_t numCookies) : numPending(numCookies) {}

CookieSync::CookieSync(const w_string& dir) {
  char hostname[256];
  gethostname(hostname, sizeof(hostname));
  hostname[sizeof(hostname) - 1] = '\0';

  auto prefix =
      w_string::build(WATCHMAN_COOKIE_PREFIX, hostname, "-", ::getpid(), "-");

  auto guard = cookieDirs_.wlock();
  guard->cookiePrefix_ = prefix;
  guard->dirs_.insert(dir);
}

CookieSync::~CookieSync() {
  // Wake up anyone that might have been waiting on us
  abortAllCookies();
}

void CookieSync::addCookieDir(const w_string& dir) {
  auto guard = cookieDirs_.wlock();
  guard->dirs_.insert(dir);
}

void CookieSync::removeCookieDir(const w_string& dir) {
  {
    auto guard = cookieDirs_.wlock();
    guard->dirs_.erase(dir);
  }

  // Cancel the cookies in the removed directory. These are considered to be
  // serviced.
  auto cookies = cookies_.wlock();
  for (const auto& [cookiePath, cookie] : *cookies) {
    if (w_string_startswith(cookiePath, dir)) {
      cookie->notify();
      cookies->erase(cookiePath);
    }
  }
}

void CookieSync::setCookieDir(const w_string& dir) {
  auto guard = cookieDirs_.wlock();
  guard->dirs_.clear();
  guard->dirs_.insert(dir);
}

std::vector<w_string> CookieSync::getOutstandingCookieFileList() const {
  std::vector<w_string> result;
  for (auto& it : *cookies_.rlock()) {
    result.push_back(it.first);
  }

  return result;
}

folly::Future<folly::Unit> CookieSync::sync() {
  auto prefixes = cookiePrefix();
  auto serial = serial_++;

  auto cookie = std::make_shared<Cookie>(prefixes.size());

  // Even though we only write to the cookie at the end of the function, we
  // need to hold it while the files are written on disk to avoid a race where
  // cookies are detected on disk by the watcher, and notifyCookie is called
  // prior to all the pending cookies being added to cookies_. Holding the lock
  // will make sure that notifyCookie will be serialized with this code.
  auto cookiesLock = cookies_.wlock();

  CookieMap pendingCookies;
  std::optional<std::tuple<w_string, int>> lastError;

  for (const auto& prefix : prefixes) {
    auto path_str = w_string::build(prefix, serial);

    /* then touch the file */
    auto file = w_stm_open(
        path_str.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0700);
    if (!file) {
      auto errCode = errno;
      lastError = {path_str, errCode};
      cookie->numPending.fetch_sub(1, std::memory_order_acq_rel);
      logf(
          ERR,
          "sync cookie {} couldn't be created: {}\n",
          path_str,
          folly::errnoStr(errCode));
      continue;
    }

    /* insert the cookie into the temporary map */
    pendingCookies[path_str] = cookie;
    logf(DBG, "sync created cookie file {}\n", path_str);
  }

  if (pendingCookies.size() == 0) {
    w_assert(lastError.has_value(), "no cookies written, but no errors set");
    auto errCode = std::get<int>(*lastError);
    throw std::system_error(
        errCode,
        std::generic_category(),
        folly::to<std::string>(
            "sync: creat(",
            std::get<w_string>(*lastError),
            ") failed: ",
            folly::errnoStr(errCode)));
  }

  cookiesLock->insert(pendingCookies.begin(), pendingCookies.end());

  return cookie->promise.getFuture();
}

void CookieSync::syncToNow(std::chrono::milliseconds timeout) {
  /* compute deadline */
  using namespace std::chrono;
  auto deadline = system_clock::now() + timeout;

  while (true) {
    auto cookie = sync();

    if (!cookie.wait(timeout).isReady()) {
      auto why = folly::to<std::string>(
          "syncToNow: timed out waiting for cookie file to be "
          "observed by watcher within ",
          timeout.count(),
          " milliseconds");
      log(ERR, why, "\n");
      throw std::system_error(ETIMEDOUT, std::generic_category(), why);
    }

    if (cookie.result().hasValue()) {
      // Success!
      return;
    }

    // Sync was aborted by a recrawl; recompute the timeout
    // and wait again if we still have time
    timeout = duration_cast<milliseconds>(deadline - system_clock::now());
    if (timeout.count() <= 0) {
      cookie.result().throwUnlessValue();
    }
  }
}

void CookieSync::abortAllCookies() {
  std::unordered_map<w_string, std::shared_ptr<Cookie>> cookies;

  {
    auto map = cookies_.wlock();
    std::swap(*map, cookies);
  }

  for (const auto& [path, cookie] : cookies) {
    if (cookie->numPending.fetch_sub(1, std::memory_order_acq_rel) == 1) {
      log(ERR, "syncToNow: aborting cookie ", path, "\n");
      cookie->promise.setException(
          folly::make_exception_wrapper<CookieSyncAborted>());
      unlink(path.c_str());
    }
  }
}

void CookieSync::Cookie::notify() {
  if (numPending.fetch_sub(1, std::memory_order_acq_rel) == 1) {
    promise.setValue();
  }
}

void CookieSync::notifyCookie(const w_string& path) {
  std::shared_ptr<Cookie> cookie;

  {
    auto map = cookies_.wlock();
    auto cookie_iter = map->find(path);
    log(DBG,
        "cookie for ",
        path,
        "? ",
        cookie_iter != map->end() ? "yes" : "no",
        "\n");

    if (cookie_iter != map->end()) {
      cookie = std::move(cookie_iter->second);
      map->erase(cookie_iter);<commit_msg>cookies: remove cookies from disk on abort take 2<commit_after>/* Copyright 2012-present Facebook, Inc.
 * Licensed under the Apache License, Version 2.0 */

#include "watchman_system.h"
#include "watchman.h"
#include <folly/String.h>
#include <exception>
#include <optional>

namespace watchman {

CookieSync::Cookie::Cookie(uint64_t numCookies) : numPending(numCookies) {}

CookieSync::CookieSync(const w_string& dir) {
  char hostname[256];
  gethostname(hostname, sizeof(hostname));
  hostname[sizeof(hostname) - 1] = '\0';

  auto prefix =
      w_string::build(WATCHMAN_COOKIE_PREFIX, hostname, "-", ::getpid(), "-");

  auto guard = cookieDirs_.wlock();
  guard->cookiePrefix_ = prefix;
  guard->dirs_.insert(dir);
}

CookieSync::~CookieSync() {
  // Wake up anyone that might have been waiting on us
  abortAllCookies();
}

void CookieSync::addCookieDir(const w_string& dir) {
  auto guard = cookieDirs_.wlock();
  guard->dirs_.insert(dir);
}

void CookieSync::removeCookieDir(const w_string& dir) {
  {
    auto guard = cookieDirs_.wlock();
    guard->dirs_.erase(dir);
  }

  // Cancel the cookies in the removed directory. These are considered to be
  // serviced.
  auto cookies = cookies_.wlock();
  for (const auto& [cookiePath, cookie] : *cookies) {
    if (w_string_startswith(cookiePath, dir)) {
      cookie->notify();
      cookies->erase(cookiePath);
    }
  }
}

void CookieSync::setCookieDir(const w_string& dir) {
  auto guard = cookieDirs_.wlock();
  guard->dirs_.clear();
  guard->dirs_.insert(dir);
}

std::vector<w_string> CookieSync::getOutstandingCookieFileList() const {
  std::vector<w_string> result;
  for (auto& it : *cookies_.rlock()) {
    result.push_back(it.first);
  }

  return result;
}

folly::Future<folly::Unit> CookieSync::sync() {
  auto prefixes = cookiePrefix();
  auto serial = serial_++;

  auto cookie = std::make_shared<Cookie>(prefixes.size());

  // Even though we only write to the cookie at the end of the function, we
  // need to hold it while the files are written on disk to avoid a race where
  // cookies are detected on disk by the watcher, and notifyCookie is called
  // prior to all the pending cookies being added to cookies_. Holding the lock
  // will make sure that notifyCookie will be serialized with this code.
  auto cookiesLock = cookies_.wlock();

  CookieMap pendingCookies;
  std::optional<std::tuple<w_string, int>> lastError;

  for (const auto& prefix : prefixes) {
    auto path_str = w_string::build(prefix, serial);

    /* then touch the file */
    auto file = w_stm_open(
        path_str.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0700);
    if (!file) {
      auto errCode = errno;
      lastError = {path_str, errCode};
      cookie->numPending.fetch_sub(1, std::memory_order_acq_rel);
      logf(
          ERR,
          "sync cookie {} couldn't be created: {}\n",
          path_str,
          folly::errnoStr(errCode));
      continue;
    }

    /* insert the cookie into the temporary map */
    pendingCookies[path_str] = cookie;
    logf(DBG, "sync created cookie file {}\n", path_str);
  }

  if (pendingCookies.size() == 0) {
    w_assert(lastError.has_value(), "no cookies written, but no errors set");
    auto errCode = std::get<int>(*lastError);
    throw std::system_error(
        errCode,
        std::generic_category(),
        folly::to<std::string>(
            "sync: creat(",
            std::get<w_string>(*lastError),
            ") failed: ",
            folly::errnoStr(errCode)));
  }

  cookiesLock->insert(pendingCookies.begin(), pendingCookies.end());

  return cookie->promise.getFuture();
}

void CookieSync::syncToNow(std::chrono::milliseconds timeout) {
  /* compute deadline */
  using namespace std::chrono;
  auto deadline = system_clock::now() + timeout;

  while (true) {
    auto cookie = sync();

    if (!cookie.wait(timeout).isReady()) {
      auto why = folly::to<std::string>(
          "syncToNow: timed out waiting for cookie file to be "
          "observed by watcher within ",
          timeout.count(),
          " milliseconds");
      log(ERR, why, "\n");
      throw std::system_error(ETIMEDOUT, std::generic_category(), why);
    }

    if (cookie.result().hasValue()) {
      // Success!
      return;
    }

    // Sync was aborted by a recrawl; recompute the timeout
    // and wait again if we still have time
    timeout = duration_cast<milliseconds>(deadline - system_clock::now());
    if (timeout.count() <= 0) {
      cookie.result().throwUnlessValue();
    }
  }
}

void CookieSync::abortAllCookies() {
  std::unordered_map<w_string, std::shared_ptr<Cookie>> cookies;

  {
    auto map = cookies_.wlock();
    std::swap(*map, cookies);
  }

  for (const auto& [path, cookie] : cookies) {
    log(ERR, "syncToNow: aborting cookie ", path, "\n");
    unlink(path.c_str());

    if (cookie->numPending.fetch_sub(1, std::memory_order_acq_rel) == 1) {
      cookie->promise.setException(
          folly::make_exception_wrapper<CookieSyncAborted>());
    }
  }
}

void CookieSync::Cookie::notify() {
  if (numPending.fetch_sub(1, std::memory_order_acq_rel) == 1) {
    promise.setValue();
  }
}

void CookieSync::notifyCookie(const w_string& path) {
  std::shared_ptr<Cookie> cookie;

  {
    auto map = cookies_.wlock();
    auto cookie_iter = map->find(path);
    log(DBG,
        "cookie for ",
        path,
        "? ",
        cookie_iter != map->end() ? "yes" : "no",
        "\n");

    if (cookie_iter != map->end()) {
      cookie = std::move(cookie_iter->second);
      map->erase(cookie_iter);
{'fraction_non_alphanumeric': 0.14319185365414075, 'fraction_numerical': 0.004415607821933856, 'mean_word_length': 3.3250194855806705, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5060830', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3564, 'n_words': 871}