<commit_before>/*
 * SessionRnwWeave.cpp
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionRnwWeave.hpp"

#include <boost/utility.hpp>
#include <boost/foreach.hpp>
#include <boost/format.hpp>

#include <boost/algorithm/string/split.hpp>

#include <core/FileSerializer.hpp>

#include <core/tex/TexLogParser.hpp>
#include <core/tex/TexMagicComment.hpp>

#include <r/RExec.hpp>
#include <r/RRoutines.hpp>
#include <r/RJson.hpp>
#include <r/session/RSessionUtils.hpp>

#include <session/SessionUserSettings.hpp>
#include <session/projects/SessionProjects.hpp>
#include <session/SessionModuleContext.hpp>

#include "SessionRnwConcordance.hpp"
#include "SessionCompilePdfSupervisor.hpp"

using namespace core;
using namespace session::modules::tex::rnw_concordance;

namespace session {
namespace modules { 
namespace tex {
namespace rnw_weave {

namespace {

Error rBinDir(FilePath* pRBinDir)
{
   std::string rHomeBin;
   r::exec::RFunction rHomeBinFunc("R.home", "bin");
   Error error = rHomeBinFunc.call(&rHomeBin);
   if (error)
      return error;

   *pRBinDir = FilePath(rHomeBin);
   return Success();
}


class RnwWeave : boost::noncopyable
{
public:
   explicit RnwWeave(const std::string& name,
                     const std::string& packageName = "")
   {
      name_ = name;
      packageName_ = !packageName.empty() ? packageName : name;
   }

   virtual ~RnwWeave()
   {
   }

   // COPYING: noncopyable (to prevent slicing)

   const std::string& name() const { return name_; }
   const std::string& packageName() const { return packageName_; }

   virtual bool injectConcordance() const = 0;

   virtual bool isInstalled() const = 0;

   virtual core::json::Value chunkOptions() const = 0;

   // tangle the passed file (note that the implementation can assume
   // that the working directory is already set to that of the file)
   virtual core::Error tangle(const std::string& file) = 0;

   virtual std::vector<std::string> commandArgs(
                                    const std::string& file) const = 0;

   virtual core::Error parseOutputForErrors(
                                    const std::string& output,
                                    const core::FilePath& rnwFilePath,
                                    core::tex::LogEntries* pLogEntries) const
   {
      // split into lines so we can determine the line numbers for the chunks
      // NOTE: will need to read this using global/project encoding if we
      // want to look for text outside of theh orignal error parsing
      // scenario (which only required ascii)
      std::string rnwContents;
      Error error = core::readStringFromFile(rnwFilePath, &rnwContents);
      if (error)
         return error;
      std::vector<std::string> lines;
      boost::algorithm::split(lines, rnwContents, boost::is_any_of("\n"));

      // determine line numbers
      boost::regex re("^<<(.*)>>=.*");
      boost::smatch match;
      std::vector<int> chunkLineNumbers;
      for (std::size_t i=0; i<lines.size(); i++)
      {
         if (boost::regex_match(lines[i], match, re))
            chunkLineNumbers.push_back(i+1);
      }

      // determine chunk number and error message
      boost::regex cre(
         "[\\w]+:[\\s]+chunk[\\s]+(\\d+)[^\n]+\n[^:]+:[\\s]*[\n]?([^\n]+)\n");
      if (boost::regex_search(output, match, cre))
      {
         std::string match1(match[1]);
         std::string match2(match[2]);
         std::size_t chunk = core::safe_convert::stringTo<int>(match1, 0);
         std::string msg = boost::algorithm::trim_copy(match2);
         if (chunk > 0 && chunk <= chunkLineNumbers.size())
         {
            boost::format fmt("(chunk %1%) %2%");
            core::tex::LogEntry logEntry(FilePath(),
                                         -1,
                                         core::tex::LogEntry::Error,
                                         rnwFilePath,
                                         chunkLineNumbers[chunk-1],
                                         boost::str(fmt % chunk % msg));
            pLogEntries->push_back(logEntry);
         }
      }

      return Success();
   }

protected:
   core::json::Value chunkOptions(const std::string& chunkFunction) const
   {
      SEXP optionsSEXP;
      r::sexp::Protect rProtect;
      r::exec::RFunction optionsFunc(chunkFunction);
      Error error = optionsFunc.call(&optionsSEXP, &rProtect);
      if (error)
      {
         LOG_ERROR(error);
         return json::Value();
      }

      core::json::Value optionsJson;
      error = r::json::jsonValueFromList(optionsSEXP, &optionsJson);
      if (error)
         LOG_ERROR(error);

      return optionsJson;
   }

private:
   std::string name_;
   std::string packageName_;
};

class RnwSweave : public RnwWeave
{
public:
   RnwSweave()
      : RnwWeave("Sweave")
   {
   }

   virtual bool isInstalled() const { return true; }

   virtual bool injectConcordance() const { return true; }

   virtual core::json::Value chunkOptions() const
   {
      return RnwWeave::chunkOptions(".rs.sweaveChunkOptions");
   }

   virtual core::Error tangle(const std::string& file)
   {
      return r::exec::RFunction("utils:::Stangle", file).call();
   }

#ifdef _WIN32
   virtual std::vector<std::string> commandArgs(const std::string& file) const
   {
      std::vector<std::string> args;
      std::string sweaveCmd = "\"Sweave('" + file + "')\"";
      args.push_back("-e");
      args.push_back(sweaveCmd);
      args.push_back("--silent");
      return args;
   }
#else
   virtual std::vector<std::string> commandArgs(const std::string& file) const
   {
      std::vector<std::string> args;
      args.push_back("CMD");
      args.push_back("Sweave");
      args.push_back(file);
      return args;
   }
#endif
};

class RnwExternalWeave : public RnwWeave
{
public:
   RnwExternalWeave(const std::string& name,
                    const std::string& packageName,
                    const std::string& cmdFmt)
     : RnwWeave(name, packageName), cmdFmt_(cmdFmt)
   {
   }

   virtual bool isInstalled() const
   {
      r::session::utils::SuppressOutputInScope suppressOutput;

      bool installed;
      r::exec::RFunction func(".rs.isPackageInstalled", packageName());
      Error error = func.call(&installed);
      return !error ? installed : false;
   }

   virtual std::vector<std::string> commandArgs(const std::string& file) const
   {
      std::vector<std::string> args;
      args.push_back("--silent");
      args.push_back("-e");
      std::string cmd = boost::str(boost::format(cmdFmt_) % file);
      args.push_back(cmd);
      return args;
   }

private:
   std::string cmdFmt_;
};


class RnwKnitr : public RnwExternalWeave
{
public:
   RnwKnitr()
      : RnwExternalWeave("knitr", "knitr", cmdFormat())
   {
   }

   virtual bool injectConcordance() const { return false; }

   virtual core::Error parseOutputForErrors(
                                    const std::string& output,
                                    const core::FilePath& rnwFilePath,
                                    core::tex::LogEntries* pLogEntries) const
   {
      boost::regex errRe("^\\s*Quitting from lines ([0-9]+)-([0-9]+): "
                          "(?:Error in [a-z]+\\([a-z=, ]+\\) : \n?)?"
                          "([^\n]+)$");
      boost::smatch match;
      if (boost::regex_search(output, match, errRe))
      {
         // extract error info
         int lineBegin = safe_convert::stringTo<int>(match[1], -1);
         std::string message = match[3];

         // check to see if there is a parse error which provides more
         // precise pinpointing of the line
         boost::regex parseRe("^\\s*<text>:([0-9]+):[0-9]+: (.+)$");
         if (boost::regex_match(message, match, parseRe))
         {
            lineBegin += safe_convert::stringTo<int>(match[1], -1);
            message = match[2];
         }

         core::tex::LogEntry logEntry(FilePath(),
                                      -1,
                                      core::tex::LogEntry::Error,
                                      rnwFilePath,
                                      lineBegin,
                                      message);
         pLogEntries->push_back(logEntry);
      }

      return Success();
   }

   virtual core::json::Value chunkOptions() const
   {
      if (isInstalled())
         return RnwWeave::chunkOptions(".rs.knitrChunkOptions");
      else
         return json::Value();
   }

   virtual core::Error tangle(const std::string& file)
   {
      r::session::utils::SuppressOutputInScope suppressOutput;
      r::exec::RFunction purlFunc("knitr:::purl");
      purlFunc.addParam("input", file);
      purlFunc.addParam("output", file + ".R");
      return purlFunc.call();
   }

private:
   static std::string cmdFormat()
   {
      std::string format = "require(knitr); ";
      if (userSettings().alwaysEnableRnwCorcordance())
         format += "opts_knit$set(concordance = TRUE); ";
      format += "knit('%1%')";
      return format;
   }
};


class RnwWeaveRegistry : boost::noncopyable
{
private:
   RnwWeaveRegistry()
   {
      weaveTypes_.push_back(boost::shared_ptr<RnwWeave>(new RnwSweave()));
      weaveTypes_.push_back(boost::shared_ptr<RnwWeave>(new RnwKnitr()));
   }
   friend const RnwWeaveRegistry& weaveRegistry();

public:
   typedef std::vector<boost::shared_ptr<RnwWeave> > RnwWeaveTypes;


public:
   std::string printableTypeNames() const
   {
      std::string str;
      for (std::size_t i=0; i<weaveTypes_.size(); i++)<commit_msg>prevent incorrect lamda characters in pdf preview (don't use dingbats)<commit_after>/*
 * SessionRnwWeave.cpp
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionRnwWeave.hpp"

#include <boost/utility.hpp>
#include <boost/foreach.hpp>
#include <boost/format.hpp>

#include <boost/algorithm/string/split.hpp>

#include <core/FileSerializer.hpp>

#include <core/tex/TexLogParser.hpp>
#include <core/tex/TexMagicComment.hpp>

#include <r/RExec.hpp>
#include <r/RRoutines.hpp>
#include <r/RJson.hpp>
#include <r/session/RSessionUtils.hpp>

#include <session/SessionUserSettings.hpp>
#include <session/projects/SessionProjects.hpp>
#include <session/SessionModuleContext.hpp>

#include "SessionRnwConcordance.hpp"
#include "SessionCompilePdfSupervisor.hpp"

using namespace core;
using namespace session::modules::tex::rnw_concordance;

namespace session {
namespace modules { 
namespace tex {
namespace rnw_weave {

namespace {

Error rBinDir(FilePath* pRBinDir)
{
   std::string rHomeBin;
   r::exec::RFunction rHomeBinFunc("R.home", "bin");
   Error error = rHomeBinFunc.call(&rHomeBin);
   if (error)
      return error;

   *pRBinDir = FilePath(rHomeBin);
   return Success();
}


class RnwWeave : boost::noncopyable
{
public:
   explicit RnwWeave(const std::string& name,
                     const std::string& packageName = "")
   {
      name_ = name;
      packageName_ = !packageName.empty() ? packageName : name;
   }

   virtual ~RnwWeave()
   {
   }

   // COPYING: noncopyable (to prevent slicing)

   const std::string& name() const { return name_; }
   const std::string& packageName() const { return packageName_; }

   virtual bool injectConcordance() const = 0;

   virtual bool isInstalled() const = 0;

   virtual core::json::Value chunkOptions() const = 0;

   // tangle the passed file (note that the implementation can assume
   // that the working directory is already set to that of the file)
   virtual core::Error tangle(const std::string& file) = 0;

   virtual std::vector<std::string> commandArgs(const std::string& file) const
   {
      std::vector<std::string> args;
      args.push_back("--silent");
      args.push_back("-e");
      std::string cmd = "grDevices::pdf.options(useDingbats = FALSE); "
                        + weaveCommand(file);
      args.push_back(cmd);
      return args;
   }

   virtual std::string weaveCommand(const std::string& file) const = 0;

   virtual core::Error parseOutputForErrors(
                                    const std::string& output,
                                    const core::FilePath& rnwFilePath,
                                    core::tex::LogEntries* pLogEntries) const
   {
      // split into lines so we can determine the line numbers for the chunks
      // NOTE: will need to read this using global/project encoding if we
      // want to look for text outside of theh orignal error parsing
      // scenario (which only required ascii)
      std::string rnwContents;
      Error error = core::readStringFromFile(rnwFilePath, &rnwContents);
      if (error)
         return error;
      std::vector<std::string> lines;
      boost::algorithm::split(lines, rnwContents, boost::is_any_of("\n"));

      // determine line numbers
      boost::regex re("^<<(.*)>>=.*");
      boost::smatch match;
      std::vector<int> chunkLineNumbers;
      for (std::size_t i=0; i<lines.size(); i++)
      {
         if (boost::regex_match(lines[i], match, re))
            chunkLineNumbers.push_back(i+1);
      }

      // determine chunk number and error message
      boost::regex cre(
         "[\\w]+:[\\s]+chunk[\\s]+(\\d+)[^\n]+\n[^:]+:[\\s]*[\n]?([^\n]+)\n");
      if (boost::regex_search(output, match, cre))
      {
         std::string match1(match[1]);
         std::string match2(match[2]);
         std::size_t chunk = core::safe_convert::stringTo<int>(match1, 0);
         std::string msg = boost::algorithm::trim_copy(match2);
         if (chunk > 0 && chunk <= chunkLineNumbers.size())
         {
            boost::format fmt("(chunk %1%) %2%");
            core::tex::LogEntry logEntry(FilePath(),
                                         -1,
                                         core::tex::LogEntry::Error,
                                         rnwFilePath,
                                         chunkLineNumbers[chunk-1],
                                         boost::str(fmt % chunk % msg));
            pLogEntries->push_back(logEntry);
         }
      }

      return Success();
   }

protected:
   core::json::Value chunkOptions(const std::string& chunkFunction) const
   {
      SEXP optionsSEXP;
      r::sexp::Protect rProtect;
      r::exec::RFunction optionsFunc(chunkFunction);
      Error error = optionsFunc.call(&optionsSEXP, &rProtect);
      if (error)
      {
         LOG_ERROR(error);
         return json::Value();
      }

      core::json::Value optionsJson;
      error = r::json::jsonValueFromList(optionsSEXP, &optionsJson);
      if (error)
         LOG_ERROR(error);

      return optionsJson;
   }

private:
   std::string name_;
   std::string packageName_;
};

class RnwSweave : public RnwWeave
{
public:
   RnwSweave()
      : RnwWeave("Sweave")
   {
   }

   virtual bool isInstalled() const { return true; }

   virtual bool injectConcordance() const { return true; }

   virtual core::json::Value chunkOptions() const
   {
      return RnwWeave::chunkOptions(".rs.sweaveChunkOptions");
   }

   virtual core::Error tangle(const std::string& file)
   {
      return r::exec::RFunction("utils:::Stangle", file).call();
   }

   virtual std::string weaveCommand(const std::string& file) const
   {
      return "utils::Sweave('" + file + "')";
   }
};


class RnwKnitr : public RnwWeave
{
public:
   RnwKnitr()
      : RnwWeave("knitr", "knitr")
   {
   }

   virtual bool injectConcordance() const { return false; }

   virtual bool isInstalled() const
   {
      r::session::utils::SuppressOutputInScope suppressOutput;

      bool installed;
      r::exec::RFunction func(".rs.isPackageInstalled", packageName());
      Error error = func.call(&installed);
      return !error ? installed : false;
   }

   virtual std::string weaveCommand(const std::string& file) const
   {
      std::string format = "require(knitr); ";
      if (userSettings().alwaysEnableRnwCorcordance())
         format += "opts_knit$set(concordance = TRUE); ";
      format += "knit('%1%')";

      return boost::str(boost::format(format) % file);
   }

   virtual core::Error parseOutputForErrors(
                                    const std::string& output,
                                    const core::FilePath& rnwFilePath,
                                    core::tex::LogEntries* pLogEntries) const
   {
      boost::regex errRe("^\\s*Quitting from lines ([0-9]+)-([0-9]+): "
                          "(?:Error in [a-z]+\\([a-z=, ]+\\) : \n?)?"
                          "([^\n]+)$");
      boost::smatch match;
      if (boost::regex_search(output, match, errRe))
      {
         // extract error info
         int lineBegin = safe_convert::stringTo<int>(match[1], -1);
         std::string message = match[3];

         // check to see if there is a parse error which provides more
         // precise pinpointing of the line
         boost::regex parseRe("^\\s*<text>:([0-9]+):[0-9]+: (.+)$");
         if (boost::regex_match(message, match, parseRe))
         {
            lineBegin += safe_convert::stringTo<int>(match[1], -1);
            message = match[2];
         }

         core::tex::LogEntry logEntry(FilePath(),
                                      -1,
                                      core::tex::LogEntry::Error,
                                      rnwFilePath,
                                      lineBegin,
                                      message);
         pLogEntries->push_back(logEntry);
      }

      return Success();
   }

   virtual core::json::Value chunkOptions() const
   {
      if (isInstalled())
         return RnwWeave::chunkOptions(".rs.knitrChunkOptions");
      else
         return json::Value();
   }

   virtual core::Error tangle(const std::string& file)
   {
      r::session::utils::SuppressOutputInScope suppressOutput;
      r::exec::RFunction purlFunc("knitr:::purl");
      purlFunc.addParam("input", file);
      purlFunc.addParam("output", file + ".R");
      return purlFunc.call();
   }
};


class RnwWeaveRegistry : boost::noncopyable
{
private:
   RnwWeaveRegistry()
   {
      weaveTypes_.push_back(boost::shared_ptr<RnwWeave>(new RnwSweave()));
      weaveTypes_.push_back(boost::shared_ptr<RnwWeave>(new RnwKnitr()));
   }
   friend const RnwWeaveRegistry& weaveRegistry();

public:
   typedef std::vector<boost::shared_ptr<RnwWeave> > RnwWeaveTypes;


public:
   std::string printableTypeNames() const
   {
      std::string str;
      for (std::size_t i=0; i<weaveTypes_.size(); i++)
{'fraction_non_alphanumeric': 0.13588541666666668, 'fraction_numerical': 0.004791666666666666, 'mean_word_length': 2.4633838383838382, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1208348', 'n_tokens_mistral': 5914, 'n_tokens_neox': 5683, 'n_words': 1291}