<commit_before>/*
 * SessionRPubs.cpp
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionRPubs.hpp"

#include <boost/bind.hpp>
#include <boost/utility.hpp>
#include <boost/format.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>

#include <core/Error.hpp>
#include <core/Exec.hpp>
#include <core/Log.hpp>
#include <core/Settings.hpp>
#include <core/FileSerializer.hpp>

#include <core/text/CsvParser.hpp>
#include <core/http/Util.hpp>
#include <core/system/Process.hpp>

#include <r/RSexp.hpp>
#include <r/RRoutines.hpp>

#include <session/SessionUserSettings.hpp>
#include <session/SessionModuleContext.hpp>

#include <session/projects/SessionProjects.hpp>

using namespace core ;

namespace session {
namespace modules { 
namespace rpubs {

namespace {

// we get a fresh settings object for each read or write so multiple
// processes can all read and write without cache issues
void getUploadIdSettings(Settings* pSettings)
{
   FilePath rpubsUploadIds =
         module_context::scopedScratchPath().complete("rpubs_upload_ids");
   Error error = pSettings->initialize(rpubsUploadIds);
   if (error)
      LOG_ERROR(error);
}

std::string pathIdentifier(const FilePath& filePath)
{
   // use a relative path if we are in a project
   std::string path;
   projects::ProjectContext& projectContext = projects::projectContext();
   if (projectContext.hasProject() &&
       filePath.isWithin(projectContext.directory()))
   {
      path = filePath.relativePath(projectContext.directory());
   }
   else
   {
      path = filePath.absolutePath();
   }

   // urlencode so we can use it as a key
   return http::util::urlEncode(path);
}

void saveUploadId(const FilePath& filePath, const std::string& uploadId)
{
   Settings settings;
   getUploadIdSettings(&settings);
   settings.set(pathIdentifier(filePath), uploadId);
}


class RPubsUpload : boost::noncopyable,
                    public boost::enable_shared_from_this<RPubsUpload>
{
public:
   static boost::shared_ptr<RPubsUpload> create(const std::string& title,
                                                const FilePath& htmlFile,
                                                bool allowUpdate)
   {
      boost::shared_ptr<RPubsUpload> pUpload(new RPubsUpload());
      pUpload->start(title, htmlFile, allowUpdate);
      return pUpload;
   }

   virtual ~RPubsUpload()
   {
   }

   bool isRunning() const { return isRunning_; }

   void terminate()
   {
      terminationRequested_ = true;
   }

private:
   RPubsUpload()
      : terminationRequested_(false), isRunning_(false)
   {
   }

   void start(const std::string& title, const FilePath& htmlFile, bool allowUpdate)
   {
      using namespace core::string_utils;
      using namespace module_context;

      htmlFile_ = htmlFile;
      csvOutputFile_ = module_context::tempFile("rpubsupload", "csv");
      isRunning_ = true;

      // see if we already know of an upload id for this file
      std::string id = allowUpdate ? previousUploadId(htmlFile_) : std::string();

      // R binary
      FilePath rProgramPath;
      Error error = rScriptPath(&rProgramPath);
      if (error)
      {
         terminateWithError(error);
         return;
      }

      // args
      std::vector<std::string> args;
      args.push_back("--slave");
      args.push_back("--no-save");
      args.push_back("--no-restore");
      args.push_back("-e");

      boost::format fmt(
               "source(\"%1%\"); "
               "result <- rpubsUpload(\"%2%\", \"%3%\", %4%); "
               "utils::write.csv(as.data.frame(result), "
                               " file=\"%5%\", "
                               " row.names=FALSE);");

      FilePath modulesPath = session::options().modulesRSourcePath();;
      std::string scriptPath = utf8ToSystem(
                        modulesPath.complete("SessionRPubs.R").absolutePath());
      std::string htmlPath = utf8ToSystem(htmlFile.absolutePath());
      std::string outputPath = utf8ToSystem(csvOutputFile_.absolutePath());

      std::string escapedScriptPath = string_utils::jsLiteralEscape(scriptPath);
      std::string escapedTitle = string_utils::jsLiteralEscape(title);
      std::string escapedHtmlPath = string_utils::jsLiteralEscape(htmlPath);
      std::string escapedId = string_utils::jsLiteralEscape(id);
      std::string escapedOutputPath = string_utils::jsLiteralEscape(outputPath);

      std::string cmd = boost::str(fmt %
                    escapedScriptPath % escapedTitle % escapedHtmlPath %
                    (!escapedId.empty() ? "\"" + escapedId + "\"" : "NULL") %
                    escapedOutputPath);
      args.push_back(cmd);

      // options
      core::system::ProcessOptions options;
      options.terminateChildren = true;
      options.workingDir = htmlFile.parent();

      // callbacks
      core::system::ProcessCallbacks cb;
      cb.onContinue = boost::bind(&RPubsUpload::onContinue,
                                  RPubsUpload::shared_from_this());
      cb.onStdout = boost::bind(&RPubsUpload::onStdOut,
                                RPubsUpload::shared_from_this(), _2);
      cb.onStderr = boost::bind(&RPubsUpload::onStdErr,<commit_msg>use single quotes in rpubsUpload call (required for windows)<commit_after>/*
 * SessionRPubs.cpp
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionRPubs.hpp"

#include <boost/bind.hpp>
#include <boost/utility.hpp>
#include <boost/format.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>

#include <core/Error.hpp>
#include <core/Exec.hpp>
#include <core/Log.hpp>
#include <core/Settings.hpp>
#include <core/FileSerializer.hpp>

#include <core/text/CsvParser.hpp>
#include <core/http/Util.hpp>
#include <core/system/Process.hpp>

#include <r/RSexp.hpp>
#include <r/RRoutines.hpp>

#include <session/SessionUserSettings.hpp>
#include <session/SessionModuleContext.hpp>

#include <session/projects/SessionProjects.hpp>

using namespace core ;

namespace session {
namespace modules { 
namespace rpubs {

namespace {

// we get a fresh settings object for each read or write so multiple
// processes can all read and write without cache issues
void getUploadIdSettings(Settings* pSettings)
{
   FilePath rpubsUploadIds =
         module_context::scopedScratchPath().complete("rpubs_upload_ids");
   Error error = pSettings->initialize(rpubsUploadIds);
   if (error)
      LOG_ERROR(error);
}

std::string pathIdentifier(const FilePath& filePath)
{
   // use a relative path if we are in a project
   std::string path;
   projects::ProjectContext& projectContext = projects::projectContext();
   if (projectContext.hasProject() &&
       filePath.isWithin(projectContext.directory()))
   {
      path = filePath.relativePath(projectContext.directory());
   }
   else
   {
      path = filePath.absolutePath();
   }

   // urlencode so we can use it as a key
   return http::util::urlEncode(path);
}

void saveUploadId(const FilePath& filePath, const std::string& uploadId)
{
   Settings settings;
   getUploadIdSettings(&settings);
   settings.set(pathIdentifier(filePath), uploadId);
}


class RPubsUpload : boost::noncopyable,
                    public boost::enable_shared_from_this<RPubsUpload>
{
public:
   static boost::shared_ptr<RPubsUpload> create(const std::string& title,
                                                const FilePath& htmlFile,
                                                bool allowUpdate)
   {
      boost::shared_ptr<RPubsUpload> pUpload(new RPubsUpload());
      pUpload->start(title, htmlFile, allowUpdate);
      return pUpload;
   }

   virtual ~RPubsUpload()
   {
   }

   bool isRunning() const { return isRunning_; }

   void terminate()
   {
      terminationRequested_ = true;
   }

private:
   RPubsUpload()
      : terminationRequested_(false), isRunning_(false)
   {
   }

   void start(const std::string& title, const FilePath& htmlFile, bool allowUpdate)
   {
      using namespace core::string_utils;
      using namespace module_context;

      htmlFile_ = htmlFile;
      csvOutputFile_ = module_context::tempFile("rpubsupload", "csv");
      isRunning_ = true;

      // see if we already know of an upload id for this file
      std::string id = allowUpdate ? previousUploadId(htmlFile_) : std::string();

      // R binary
      FilePath rProgramPath;
      Error error = rScriptPath(&rProgramPath);
      if (error)
      {
         terminateWithError(error);
         return;
      }

      // args
      std::vector<std::string> args;
      args.push_back("--slave");
      args.push_back("--no-save");
      args.push_back("--no-restore");
      args.push_back("-e");

      boost::format fmt(
               "source('%1%'); "
               "result <- rpubsUpload('%2%', '%3%', %4%); "
               "utils::write.csv(as.data.frame(result), "
                               " file='%5%', "
                               " row.names=FALSE);");

      FilePath modulesPath = session::options().modulesRSourcePath();;
      std::string scriptPath = utf8ToSystem(
                        modulesPath.complete("SessionRPubs.R").absolutePath());
      std::string htmlPath = utf8ToSystem(htmlFile.absolutePath());
      std::string outputPath = utf8ToSystem(csvOutputFile_.absolutePath());

      std::string escapedScriptPath = string_utils::jsLiteralEscape(scriptPath);
      std::string escapedTitle = string_utils::jsLiteralEscape(title);
      std::string escapedHtmlPath = string_utils::jsLiteralEscape(htmlPath);
      std::string escapedId = string_utils::jsLiteralEscape(id);
      std::string escapedOutputPath = string_utils::jsLiteralEscape(outputPath);

      std::string cmd = boost::str(fmt %
                    escapedScriptPath % escapedTitle % escapedHtmlPath %
                    (!escapedId.empty() ? "'" + escapedId + "'" : "NULL") %
                    escapedOutputPath);
      args.push_back(cmd);

      // options
      core::system::ProcessOptions options;
      options.terminateChildren = true;
      options.workingDir = htmlFile.parent();

      // callbacks
      core::system::ProcessCallbacks cb;
      cb.onContinue = boost::bind(&RPubsUpload::onContinue,
                                  RPubsUpload::shared_from_this());
      cb.onStdout = boost::bind(&RPubsUpload::onStdOut,
                                RPubsUpload::shared_from_this(), _2);
      cb.onStderr = boost::bind(&RPubsUpload::onStdErr,
{'fraction_non_alphanumeric': 0.11960311075355323, 'fraction_numerical': 0.003218020917135961, 'mean_word_length': 2.960353982300885, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6332232', 'n_tokens_mistral': 3298, 'n_tokens_neox': 3181, 'n_words': 713}