<commit_before>#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/sem.h>
#include <cerrno>
#include <cstring>
#include <cstdlib>
#include <cstdio>
#include <unistd.h>
#include <iostream>
#include "defines.h"
#include "shared_memory.h"
#include "stream.h"
#include "procs.h"

namespace IPC {
  /// Stores a long value of val in network order to the pointer p.
  static void htobl(char * p, long val) {
    p[0] = (val >> 24) & 0xFF;
    p[1] = (val >> 16) & 0xFF;
    p[2] = (val >> 8) & 0xFF;
    p[3] = val & 0xFF;
  }

  /// Stores a short value of val in network order to the pointer p.
  static void htobs(char * p, short val) {
    p[0] = (val >> 8) & 0xFF;
    p[1] = val & 0xFF;
  }


  /// Stores a long long value of val in network order to the pointer p.
  static void htobll(char * p, long long val) {
    p[0] = (val >> 56) & 0xFF;
    p[1] = (val >> 48) & 0xFF;
    p[2] = (val >> 40) & 0xFF;
    p[3] = (val >> 32) & 0xFF;
    p[4] = (val >> 24) & 0xFF;
    p[5] = (val >> 16) & 0xFF;
    p[6] = (val >> 8) & 0xFF;
    p[7] = val & 0xFF;
  }

  /// Reads a long value of p in host order to val.
  static void btohl(char * p, long & val) {
    val = ((long)p[0] << 24) | ((long)p[1] << 16) | ((long)p[2] << 8) | p[3];
  }

  /// Reads a short value of p in host order to val.
  static void btohs(char * p, unsigned short & val) {
    val = ((short)p[0] << 8) | p[1];
  }

  /// Reads a long value of p in host order to val.
  static void btohl(char * p, unsigned int & val) {
    val = ((long)p[0] << 24) | ((long)p[1] << 16) | ((long)p[2] << 8) | p[3];
  }
  
  /// Reads a long long value of p in host order to val.
  static void btohll(char * p, long long & val) {
    val = ((long long)p[0] << 56) | ((long long)p[1] << 48) | ((long long)p[2] << 40) | ((long long)p[3] << 32) | ((long long)p[4] << 24) | ((long long)p[5] << 16) | ((long long)p[6] << 8) | p[7];
  }

  ///\brief Empty semaphore constructor, clears all values
  semaphore::semaphore() {
#ifdef __CYGWIN__
    mySem = 0;
#else
    mySem = SEM_FAILED;
#endif
  }

  ///\brief Constructs a named semaphore
  ///\param name The name of the semaphore
  ///\param oflag The flags with which to open the semaphore
  ///\param mode The mode in which to create the semaphore, if O_CREAT is given in oflag, ignored otherwise
  ///\param value The initial value of the semaphore if O_CREAT is given in oflag, ignored otherwise
  semaphore::semaphore(const char * name, int oflag, mode_t mode, unsigned int value) {
#ifdef __CYGWIN__
    mySem = 0;
#else
    mySem = SEM_FAILED;
#endif
    open(name, oflag, mode, value);
  }

  ///\brief The deconstructor
  semaphore::~semaphore() {
    close();
  }


  ///\brief Returns whether we have a valid semaphore
  semaphore::operator bool() const {
#ifdef __CYGWIN__
    return mySem != 0;
#else
    return mySem != SEM_FAILED;
#endif
  }

  ///\brief Opens a semaphore
  ///
  ///Closes currently opened semaphore if needed
  ///\param name The name of the semaphore
  ///\param oflag The flags with which to open the semaphore
  ///\param mode The mode in which to create the semaphore, if O_CREAT is given in oflag, ignored otherwise
  ///\param value The initial value of the semaphore if O_CREAT is given in oflag, ignored otherwise
  void semaphore::open(const char * name, int oflag, mode_t mode, unsigned int value) {
    close();
    int timer = 0;
    while (!(*this) && timer++ < 10){
#ifdef __CYGWIN__
      mySem = CreateSemaphore(0, value, 1 , std::string("Global\\" + std::string(name)).c_str());
#else
      if (oflag & O_CREAT) {
        mySem = sem_open(name, oflag, mode, value);
      } else {
        mySem = sem_open(name, oflag);
      }
#endif
      if (!(*this)){
        if (errno == ENOENT){
          Util::wait(500);
        }else{
          break;
        }
      }
    }
    if (!(*this)){
      DEBUG_MSG(DLVL_VERYHIGH, "Attempt to open semaphore %s: %s", name, strerror(errno));
    }
    myName = (char *)name;
  }

  ///\brief Returns the current value of the semaphore
  int semaphore::getVal() const {
#ifdef __CYGWIN__
    LONG res;
    ReleaseSemaphore(mySem, 0, &res);//not really release.... just checking to see if I can get the value this way
#else
    int res;
    sem_getvalue(mySem, &res);
#endif
    return res;
  }

  ///\brief Posts to the semaphore, increases its value by one
  void semaphore::post() {
    if (*this) {
#ifdef __CYGWIN__
      ReleaseSemaphore(mySem, 1, 0);
#else
      sem_post(mySem);
#endif
    }
  }

  ///\brief Waits for the semaphore, decreases its value by one
  void semaphore::wait() {
    if (*this) {
#ifdef __CYGWIN__
      WaitForSingleObject(mySem, INFINITE);
#else
      int tmp;
      do {
        tmp = sem_wait(mySem);
      } while (tmp == -1 && errno == EINTR);
#endif
    }
  }

  ///\brief Tries to wait for the semaphore, returns true if successfull, false otherwise
  bool semaphore::tryWait() {
    bool result;
#ifdef __CYGWIN__
    result = WaitForSingleObject(mySem, 0);//wait at most 1ms
#else
    result = sem_trywait(mySem);
#endif
    return (result == 0);
  }

  ///\brief Closes the currently opened semaphore
  void semaphore::close() {
    if (*this) {
#ifdef __CYGWIN__
      CloseHandle(mySem);
      mySem = 0;
#else
      sem_close(mySem);
      mySem = SEM_FAILED;
#endif
    }
  }

  ///\brief Unlinks the previously opened semaphore
  void semaphore::unlink() {
    close();
#ifndef __CYGWIN__
    if (myName.size()) {
      sem_unlink(myName.c_str());
    }
#endif
    myName.clear();
  }


  ///brief Creates a shared page
  ///\param name_ The name of the page to be created
  ///\param len_ The size to make the page
  ///\param master_ Whether to create or merely open the page
  ///\param autoBackoff When only opening the page, wait for it to appear or fail
  sharedPage::sharedPage(std::string name_, unsigned int len_, bool master_, bool autoBackoff){
    handle = 0;
    len = 0;
    master = false;
    mapped = 0;
    init(name_, len_, master_, autoBackoff);
  }

  ///\brief Creates a copy of a shared page
  ///\param rhs The page to copy
  sharedPage::sharedPage(const sharedPage & rhs) {
    handle = 0;
    len = 0;
    master = false;
    mapped = 0;
    init(rhs.name, rhs.len, rhs.master);
  }

  ///\brief Default destructor
  sharedPage::~sharedPage() {
    close();
  }

#ifdef SHM_ENABLED
  ///\brief Unmaps a shared page if allowed
  void sharedPage::unmap() {
    if (mapped && len) {
#ifdef __CYGWIN__
      //under Cygwin, the mapped location is shifted by 4 to contain the page size.
      UnmapViewOfFile(mapped-4);
#else
      munmap(mapped, len);
#endif
      mapped = 0;
      len = 0;
    }
  }

  ///\brief Closes a shared page if allowed
  void sharedPage::close() {
    unmap();
    if (handle > 0) {
#ifdef __CYGWIN__
      CloseHandle(handle);
#else
      ::close(handle);
      if (master && name != "") {
        shm_unlink(name.c_str());
      }
#endif
      handle = 0;
    }
  }

  ///\brief Returns whether the shared page is valid or not
  sharedPage::operator bool() const {
    return mapped != 0;
  }

  ///\brief Assignment operator
  void sharedPage::operator =(sharedPage & rhs) {
    init(rhs.name, rhs.len, rhs.master);
    /// \todo This is bad. The assignment operator changes the rhs value? What the hell?
    rhs.master = false;//Make sure the memory does not get unlinked
  }


  ///\brief Initialize a page, de-initialize before if needed
  ///\param name_ The name of the page to be created
  ///\param len_ The size to make the page
  ///\param master_ Whether to create or merely open the page
  ///\param autoBackoff When only opening the page, wait for it to appear or fail
  void sharedPage::init(std::string name_, unsigned int len_, bool master_, bool autoBackoff) {
    close();
    name = name_;
    len = len_;
    master = master_;
    mapped = 0;
    if (name.size()) {
#ifdef __CYGWIN__
      if (master) {
        //Under cygwin, all pages are 4 bytes longer than claimed.
        handle = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, len+4, name.c_str());
      } else {
        int i = 0;
        do {
          if (i != 0) {
            Util::sleep(1000);
          }
          handle = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, name.c_str());
          i++;
        } while (i < 10 && !handle && autoBackoff);
      }
      if (!handle) {
        FAIL_MSG("%s for page %s failed: %s", (master ? "CreateFileMapping" : "OpenFileMapping"), name.c_str(), strerror(errno));
        return;
      }
      mapped = (char *)MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, 0);
      if (!mapped) {
        FAIL_MSG("MapViewOfFile for page %s failed: %s", name.c_str(), strerror(errno));
        return;
      }<commit_msg>Changed windows shm functions to ANSI equivalents.<commit_after>#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/sem.h>
#include <cerrno>
#include <cstring>
#include <cstdlib>
#include <cstdio>
#include <unistd.h>
#include <iostream>
#include "defines.h"
#include "shared_memory.h"
#include "stream.h"
#include "procs.h"

namespace IPC {
  /// Stores a long value of val in network order to the pointer p.
  static void htobl(char * p, long val) {
    p[0] = (val >> 24) & 0xFF;
    p[1] = (val >> 16) & 0xFF;
    p[2] = (val >> 8) & 0xFF;
    p[3] = val & 0xFF;
  }

  /// Stores a short value of val in network order to the pointer p.
  static void htobs(char * p, short val) {
    p[0] = (val >> 8) & 0xFF;
    p[1] = val & 0xFF;
  }


  /// Stores a long long value of val in network order to the pointer p.
  static void htobll(char * p, long long val) {
    p[0] = (val >> 56) & 0xFF;
    p[1] = (val >> 48) & 0xFF;
    p[2] = (val >> 40) & 0xFF;
    p[3] = (val >> 32) & 0xFF;
    p[4] = (val >> 24) & 0xFF;
    p[5] = (val >> 16) & 0xFF;
    p[6] = (val >> 8) & 0xFF;
    p[7] = val & 0xFF;
  }

  /// Reads a long value of p in host order to val.
  static void btohl(char * p, long & val) {
    val = ((long)p[0] << 24) | ((long)p[1] << 16) | ((long)p[2] << 8) | p[3];
  }

  /// Reads a short value of p in host order to val.
  static void btohs(char * p, unsigned short & val) {
    val = ((short)p[0] << 8) | p[1];
  }

  /// Reads a long value of p in host order to val.
  static void btohl(char * p, unsigned int & val) {
    val = ((long)p[0] << 24) | ((long)p[1] << 16) | ((long)p[2] << 8) | p[3];
  }
  
  /// Reads a long long value of p in host order to val.
  static void btohll(char * p, long long & val) {
    val = ((long long)p[0] << 56) | ((long long)p[1] << 48) | ((long long)p[2] << 40) | ((long long)p[3] << 32) | ((long long)p[4] << 24) | ((long long)p[5] << 16) | ((long long)p[6] << 8) | p[7];
  }

  ///\brief Empty semaphore constructor, clears all values
  semaphore::semaphore() {
#ifdef __CYGWIN__
    mySem = 0;
#else
    mySem = SEM_FAILED;
#endif
  }

  ///\brief Constructs a named semaphore
  ///\param name The name of the semaphore
  ///\param oflag The flags with which to open the semaphore
  ///\param mode The mode in which to create the semaphore, if O_CREAT is given in oflag, ignored otherwise
  ///\param value The initial value of the semaphore if O_CREAT is given in oflag, ignored otherwise
  semaphore::semaphore(const char * name, int oflag, mode_t mode, unsigned int value) {
#ifdef __CYGWIN__
    mySem = 0;
#else
    mySem = SEM_FAILED;
#endif
    open(name, oflag, mode, value);
  }

  ///\brief The deconstructor
  semaphore::~semaphore() {
    close();
  }


  ///\brief Returns whether we have a valid semaphore
  semaphore::operator bool() const {
#ifdef __CYGWIN__
    return mySem != 0;
#else
    return mySem != SEM_FAILED;
#endif
  }

  ///\brief Opens a semaphore
  ///
  ///Closes currently opened semaphore if needed
  ///\param name The name of the semaphore
  ///\param oflag The flags with which to open the semaphore
  ///\param mode The mode in which to create the semaphore, if O_CREAT is given in oflag, ignored otherwise
  ///\param value The initial value of the semaphore if O_CREAT is given in oflag, ignored otherwise
  void semaphore::open(const char * name, int oflag, mode_t mode, unsigned int value) {
    close();
    int timer = 0;
    while (!(*this) && timer++ < 10){
#ifdef __CYGWIN__
      mySem = CreateSemaphore(0, value, 1 , std::string("Global\\" + std::string(name)).c_str());
#else
      if (oflag & O_CREAT) {
        mySem = sem_open(name, oflag, mode, value);
      } else {
        mySem = sem_open(name, oflag);
      }
#endif
      if (!(*this)){
        if (errno == ENOENT){
          Util::wait(500);
        }else{
          break;
        }
      }
    }
    if (!(*this)){
      DEBUG_MSG(DLVL_VERYHIGH, "Attempt to open semaphore %s: %s", name, strerror(errno));
    }
    myName = (char *)name;
  }

  ///\brief Returns the current value of the semaphore
  int semaphore::getVal() const {
#ifdef __CYGWIN__
    LONG res;
    ReleaseSemaphore(mySem, 0, &res);//not really release.... just checking to see if I can get the value this way
#else
    int res;
    sem_getvalue(mySem, &res);
#endif
    return res;
  }

  ///\brief Posts to the semaphore, increases its value by one
  void semaphore::post() {
    if (*this) {
#ifdef __CYGWIN__
      ReleaseSemaphore(mySem, 1, 0);
#else
      sem_post(mySem);
#endif
    }
  }

  ///\brief Waits for the semaphore, decreases its value by one
  void semaphore::wait() {
    if (*this) {
#ifdef __CYGWIN__
      WaitForSingleObject(mySem, INFINITE);
#else
      int tmp;
      do {
        tmp = sem_wait(mySem);
      } while (tmp == -1 && errno == EINTR);
#endif
    }
  }

  ///\brief Tries to wait for the semaphore, returns true if successfull, false otherwise
  bool semaphore::tryWait() {
    bool result;
#ifdef __CYGWIN__
    result = WaitForSingleObject(mySem, 0);//wait at most 1ms
#else
    result = sem_trywait(mySem);
#endif
    return (result == 0);
  }

  ///\brief Closes the currently opened semaphore
  void semaphore::close() {
    if (*this) {
#ifdef __CYGWIN__
      CloseHandle(mySem);
      mySem = 0;
#else
      sem_close(mySem);
      mySem = SEM_FAILED;
#endif
    }
  }

  ///\brief Unlinks the previously opened semaphore
  void semaphore::unlink() {
    close();
#ifndef __CYGWIN__
    if (myName.size()) {
      sem_unlink(myName.c_str());
    }
#endif
    myName.clear();
  }


  ///brief Creates a shared page
  ///\param name_ The name of the page to be created
  ///\param len_ The size to make the page
  ///\param master_ Whether to create or merely open the page
  ///\param autoBackoff When only opening the page, wait for it to appear or fail
  sharedPage::sharedPage(std::string name_, unsigned int len_, bool master_, bool autoBackoff){
    handle = 0;
    len = 0;
    master = false;
    mapped = 0;
    init(name_, len_, master_, autoBackoff);
  }

  ///\brief Creates a copy of a shared page
  ///\param rhs The page to copy
  sharedPage::sharedPage(const sharedPage & rhs) {
    handle = 0;
    len = 0;
    master = false;
    mapped = 0;
    init(rhs.name, rhs.len, rhs.master);
  }

  ///\brief Default destructor
  sharedPage::~sharedPage() {
    close();
  }

#ifdef SHM_ENABLED
  ///\brief Unmaps a shared page if allowed
  void sharedPage::unmap() {
    if (mapped && len) {
#ifdef __CYGWIN__
      //under Cygwin, the mapped location is shifted by 4 to contain the page size.
      UnmapViewOfFile(mapped-4);
#else
      munmap(mapped, len);
#endif
      mapped = 0;
      len = 0;
    }
  }

  ///\brief Closes a shared page if allowed
  void sharedPage::close() {
    unmap();
    if (handle > 0) {
#ifdef __CYGWIN__
      CloseHandle(handle);
#else
      ::close(handle);
      if (master && name != "") {
        shm_unlink(name.c_str());
      }
#endif
      handle = 0;
    }
  }

  ///\brief Returns whether the shared page is valid or not
  sharedPage::operator bool() const {
    return mapped != 0;
  }

  ///\brief Assignment operator
  void sharedPage::operator =(sharedPage & rhs) {
    init(rhs.name, rhs.len, rhs.master);
    /// \todo This is bad. The assignment operator changes the rhs value? What the hell?
    rhs.master = false;//Make sure the memory does not get unlinked
  }


  ///\brief Initialize a page, de-initialize before if needed
  ///\param name_ The name of the page to be created
  ///\param len_ The size to make the page
  ///\param master_ Whether to create or merely open the page
  ///\param autoBackoff When only opening the page, wait for it to appear or fail
  void sharedPage::init(std::string name_, unsigned int len_, bool master_, bool autoBackoff) {
    close();
    name = name_;
    len = len_;
    master = master_;
    mapped = 0;
    if (name.size()) {
#ifdef __CYGWIN__
      if (master) {
        //Under cygwin, all pages are 4 bytes longer than claimed.
        handle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, len+4, name.c_str());
      } else {
        int i = 0;
        do {
          if (i != 0) {
            Util::sleep(1000);
          }
          handle = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, name.c_str());
          i++;
        } while (i < 10 && !handle && autoBackoff);
      }
      if (!handle) {
        FAIL_MSG("%s for page %s failed: %s", (master ? "CreateFileMapping" : "OpenFileMapping"), name.c_str(), strerror(errno));
        return;
      }
      mapped = (char *)MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, 0);
      if (!mapped) {
        FAIL_MSG("MapViewOfFile for page %s failed: %s", name.c_str(), strerror(errno));
        return;
      }
{'fraction_non_alphanumeric': 0.13864709563634295, 'fraction_numerical': 0.015474768162940206, 'mean_word_length': 2.5932134096484054, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2091794', 'n_tokens_mistral': 6342, 'n_tokens_neox': 6095, 'n_words': 2042}