<commit_before>// terrasync.cxx -- scenery fetcher
//
// Started by <NAME>, November 2002.
//
// Copyright (C) 2002  <NAME>  - http://www.flightgear.org/~curt
// Copyright (C) 2008  <NAME> <<EMAIL>>
// Copyright (C) 2011  <NAME> <<EMAIL>>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
//
// $Id$

#ifdef HAVE_CONFIG_H
#  include <simgear_config.h>
#endif

#include <simgear/compiler.h>

#ifdef HAVE_WINDOWS_H
#include <windows.h>
#endif

#ifdef __MINGW32__
#include <time.h>
#include <unistd.h>
#elif defined(_MSC_VER)
#   include <io.h>
#   ifndef HAVE_SVN_CLIENT_H
#       include <time.h>
#       include <process.h>
#   endif
#endif

#include <stdlib.h>             // atoi() atof() abs() system()
#include <signal.h>             // signal()
#include <string.h>

#include <iostream>
#include <fstream>
#include <string>
#include <map>

#include <simgear/compiler.h>

#include "terrasync.hxx"
#include <simgear/bucket/newbucket.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/threads/SGQueue.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/debug/BufferedLogCallback.hxx>

#ifdef SG_SVN_CLIENT
#  include <simgear/io/HTTPClient.hxx>
#  include <simgear/io/SVNRepository.hxx>
#endif

#ifdef HAVE_SVN_CLIENT_H
#  ifdef HAVE_LIBSVN_CLIENT_1
#    include <svn_version.h>
#    include <svn_auth.h>
#    include <svn_client.h>
#    include <svn_cmdline.h>
#    include <svn_pools.h>
#  else
#    undef HAVE_SVN_CLIENT_H
#  endif
#endif

#ifdef HAVE_SVN_CLIENT_H
    static const svn_version_checklist_t mysvn_checklist[] = {
        { "svn_subr",   svn_subr_version },
        { "svn_client", svn_client_version },
        { NULL, NULL }
    };
#endif
    
#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
    static const bool svn_built_in_available = true;
#else
    static const bool svn_built_in_available = false;
#endif

using namespace simgear;
using namespace std;

const char* rsync_cmd = 
        "rsync --verbose --archive --delete --perms --owner --group";

const char* svn_options =
        "checkout -q";

namespace UpdateInterval
{
    // interval in seconds to allow an update to repeat after a successful update (=daily)
    static const double SuccessfulAttempt = 24*60*60;
    // interval in seconds to allow another update after a failed attempt (10 minutes)
    static const double FailedAttempt     = 10*60;
}

typedef map<string,time_t> CompletedTiles;

///////////////////////////////////////////////////////////////////////////////
// helper functions ///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
string stripPath(string path)
{
    // svn doesn't like trailing white-spaces or path separators - strip them!
    path = simgear::strutils::strip(path);
    size_t slen = path.length();
    while ((slen>0)&&
            ((path[slen-1]=='/')||(path[slen-1]=='\\')))
    {
        slen--;
    }
    return path.substr(0,slen);
}

bool hasWhitespace(string path)
{
    return path.find(' ')!=string::npos;
}

///////////////////////////////////////////////////////////////////////////////
// WaitingTile ////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
class  WaitingTile
{
public:
    WaitingTile(string dir,bool refresh) :
        _dir(dir), _refreshScenery(refresh) {}
    string _dir;
    bool _refreshScenery;
};

///////////////////////////////////////////////////////////////////////////////
// SGTerraSync::SvnThread /////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
class SGTerraSync::SvnThread : public SGThread
{
public:
   SvnThread();
   virtual ~SvnThread( ) { stop(); }

   void stop();
   bool start();

   bool isIdle() {return waitingTiles.empty();}
   void request(const WaitingTile& dir) {waitingTiles.push_front(dir);}
   bool isDirty() { bool r = _is_dirty;_is_dirty = false;return r;}
   bool hasNewTiles() { return !_freshTiles.empty();}
   WaitingTile getNewTile() { return _freshTiles.pop_front();}

   void   setSvnServer(string server)       { _svn_server   = stripPath(server);}
   void   setExtSvnUtility(string svn_util) { _svn_command  = simgear::strutils::strip(svn_util);}
   void   setRsyncServer(string server)     { _rsync_server = simgear::strutils::strip(server);}
   void   setLocalDir(string dir)           { _local_dir    = stripPath(dir);}
   string getLocalDir()                     { return _local_dir;}
   void   setUseSvn(bool use_svn)           { _use_svn = use_svn;}
   void   setAllowedErrorCount(int errors)  {_allowed_errors = errors;}

#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
   void setUseBuiltin(bool built_in) { _use_built_in = built_in;}
#endif

   volatile bool _active;
   volatile bool _running;
   volatile bool _busy;
   volatile bool _stalled;
   volatile int  _fail_count;
   volatile int  _updated_tile_count;
   volatile int  _success_count;
   volatile int  _consecutive_errors;
   volatile int  _allowed_errors;

private:
   virtual void run();
   bool syncTree(const char* dir, bool& isNewDirectory);
   bool syncTreeExternal(const char* dir);


#if defined(SG_SVN_CLIENT)
   bool syncTreeInternal(const char* dir);
   bool _use_built_in;
   HTTP::Client _http;
   std::auto_ptr<SVNRepository> _repository;
#elif defined(HAVE_SVN_CLIENT_H)
   static int svnClientSetup(void);
   bool syncTreeInternal(const char* dir);

   bool _use_built_in;

   // Things we need for doing subversion checkout - often
   static apr_pool_t *_svn_pool;
   static svn_client_ctx_t *_svn_ctx;
   static svn_opt_revision_t *_svn_rev;
   static svn_opt_revision_t *_svn_rev_peg;
#endif

   volatile bool _is_dirty;
   volatile bool _stop;
   SGBlockingDeque <WaitingTile> waitingTiles;
   CompletedTiles _completedTiles;
   SGBlockingDeque <WaitingTile> _freshTiles;
   bool _use_svn;
   string _svn_server;
   string _svn_command;
   string _rsync_server;
   string _local_dir;
};

#ifdef HAVE_SVN_CLIENT_H
    apr_pool_t* SGTerraSync::SvnThread::_svn_pool = NULL;
    svn_client_ctx_t* SGTerraSync::SvnThread::_svn_ctx = NULL;
    svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev = NULL;
    svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev_peg = NULL;
#endif

SGTerraSync::SvnThread::SvnThread() :
    _active(false),
    _running(false),
    _busy(false),
    _stalled(false),
    _fail_count(0),
    _updated_tile_count(0),
    _success_count(0),
    _consecutive_errors(0),
    _allowed_errors(6),
#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
    _use_built_in(true),
#endif
    _is_dirty(false),
    _stop(false),
    _use_svn(true)
{
#ifdef HAVE_SVN_CLIENT_H
    int errCode = SGTerraSync::SvnThread::svnClientSetup();
    if (errCode != EXIT_SUCCESS)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Failed to initialize built-in SVN client, error = " << errCode);
    }
#endif
}

void SGTerraSync::SvnThread::stop()
{
    // drop any pending requests
    waitingTiles.clear();

    if (!_running)
        return;

    // set stop flag and wake up the thread with an empty request
    _stop = true;
    WaitingTile w("",false);
    request(w);
    join();
    _running = false;
}

bool SGTerraSync::SvnThread::start()
{
    if (_running)
        return false;

    if (_local_dir=="")
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Local cache directory is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    SGPath path(_local_dir);
    if (!path.exists())
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Directory '" << _local_dir <<
               "' does not exist. Set correct directory path or create directory folder.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    path.append("version");
    if (path.exists())
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Directory '" << _local_dir <<
               "' contains the base package. Use a separate directory.");
        _fail_count++;
        _stalled = true;
        return false;
    }

#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
    _use_svn |= _use_built_in;
#endif

    if ((_use_svn)&&(_svn_server==""))
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Subversion scenery server is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }
    if ((!_use_svn)&&(_rsync_server==""))
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Rsync scenery server is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    _fail_count = 0;
    _updated_tile_count = 0;
    _success_count = 0;
    _consecutive_errors = 0;
    _stop = false;
    _stalled = false;
    _running = true;

    string status;
#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
    if (_use_svn && _use_built_in)
        status = "Using built-in SVN support. ";
    else
#endif
    if (_use_svn)
    {
        status = "Using external SVN utility '";
        status += _svn_command;
        status += "'. ";
    }
    else
    {
        status = "Using RSYNC. ";
    }

    // not really an alert - but we want to (always) see this message, so user is
    // aware we're downloading scenery (and using bandwidth).
    SG_LOG(SG_TERRAIN,SG_ALERT,
           "Starting automatic scenery download/synchronization. "
           << status
           << "Directory: '" << _local_dir << "'.");

    SGThread::start();
    return true;
}

// sync one directory tree
bool SGTerraSync::SvnThread::syncTree(const char* dir, bool& isNewDirectory)
{
    int rc;
    SGPath path( _local_dir );

    path.append( dir );
    isNewDirectory = !path.exists();
    if (isNewDirectory)
    {
        rc = path.create_dir( 0755 );
        if (rc)
        {
            SG_LOG(SG_TERRAIN,SG_ALERT,
                   "Cannot create directory '" << dir << "', return code = " << rc );
            return false;
        }
    }

#if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
    if (_use_built_in)
        return syncTreeInternal(dir);
    else
#endif
    {
        return syncTreeExternal(dir);
    }
}

#if defined(SG_SVN_CLIENT)
    
bool SGTerraSync::SvnThread::syncTreeInternal(const char* dir)
{
    ostringstream command;
    command << _svn_server << "/" << dir;

    SGPath path(_local_dir);
    path.append(dir);
    _repository.reset(new SVNRepository(path, &_http));
    _repository->setBaseUrl(command.str());
    
    SGTimeStamp st;
    st.stamp();
    SG_LOG(SG_IO, SG_DEBUG, "terrasync: will sync " << command.str());
    _repository->update();
    
    bool result = true;
    while (!_stop && _repository->isDoingSync()) {
        _http.update(100);
    }
    
    if (_repository->failure() == SVNRepository::SVN_ERROR_NOT_FOUND) {
        // this is fine, but maybe we should use a different return code
        // in the future to higher layers can distuinguish this case
    } else if (_repository->failure() != SVNRepository::SVN_NO_ERROR) {
        result = false;
    } else {
        SG_LOG(SG_IO, SG_DEBUG, "sync of " << command.str() << " finished ("
            << st.elapsedMSec() << " msec");
    }
    
    _repository.reset();
    return result;
}
    
#elif defined(HAVE_SVN_CLIENT_H)

bool SGTerraSync::SvnThread::syncTreeInternal(const char* dir)
{
    SG_LOG(SG_TERRAIN,SG_DEBUG, "Synchronizing scenery directory " << dir);
    if (!_svn_pool)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Built-in SVN client failed to initialize.");
        return false;
    }

    ostringstream command;
    command << _svn_server << "/" << dir;

    ostringstream dest_base_dir;
    dest_base_dir << _local_dir << "/" << dir;

    apr_pool_t *subpool = svn_pool_create(_svn_pool);

    svn_error_t *err = NULL;
#if (SVN_VER_MINOR >= 5)
    err = svn_client_checkout3(NULL,
            command.str().c_str(),
            dest_base_dir.str().c_str(),
            _svn_rev_peg,
            _svn_rev,
            svn_depth_infinity,
            0, // ignore-externals = false
            0, // allow unver obstructions = false
            _svn_ctx,
            subpool);
#else
    // version 1.4 API
    err = svn_client_checkout2(NULL,
            command.str().c_str(),
            dest_base_dir.str().c_str(),
            _svn_rev_peg,
            _svn_rev,
            1, // recurse=true - same as svn_depth_infinity for checkout3 above
            0, // ignore externals = false
            _svn_ctx,
            subpool);
#endif

    bool ReturnValue = true;
    if (err)
    {
        // Report errors from the checkout attempt
        if (err->apr_err == SVN_ERR_RA_ILLEGAL_URL)
        {
            // ignore errors when remote path doesn't exist (no scenery data for ocean areas)
        }
        else
        {
            SG_LOG(SG_TERRAIN,SG_ALERT,
                    "Failed to synchronize directory '" << dir << "', " <<
                    err->message << " (code " << err->apr_err << ").");
            svn_error_clear(err);
            // try to clean up
            err = svn_client_cleanup(dest_base_dir.str().c_str(),
                    _svn_ctx,subpool);
            if (!err)
            {
                SG_LOG(SG_TERRAIN,SG_ALERT,
                       "SVN repository cleanup successful for '" << dir << "'.");
            }
            ReturnValue = false;
        }
    } else
    {
        SG_LOG(SG_TERRAIN,SG_DEBUG, "Done with scenery directory " << dir);
    }
    svn_pool_destroy(subpool);
    return ReturnValue;
}
#endif // of HAVE_SVN_CLIENT_H

bool SGTerraSync::SvnThread::syncTreeExternal(const char* dir)
{
    ostringstream buf;
    SGPath localPath( _local_dir );
    localPath.append( dir );

    if (_use_svn)
    {
        buf << "\"" << _svn_command << "\" "
            << svn_options << " "
            << "\"" << _svn_server << "/" << dir << "\" "
            << "\"" << localPath.str_native() << "\"";
    } else {
        buf << rsync_cmd << " "
            << "\"" << _rsync_server << "/" << dir << "/\" "
            << "\"" << localPath.str_native() << "/\"";
    }

    string command;
#ifdef SG_WINDOWS
        // windows command line parsing is just lovely...
        // to allow white spaces, the system call needs this:
        // ""C:\Program Files\something.exe" somearg "some other arg""
        // Note: whitespace strings quoted by a pair of "" _and_ the 
        //       entire string needs to be wrapped by "" too.
        // The svn url needs forward slashes (/) as a path separator while
        // the local path needs windows-native backslash as a path separator.
    command = "\"" + buf.str() + "\"";
#else
    command = buf.str();
#endif
    SG_LOG(SG_TERRAIN,SG_DEBUG, "sync command '" << command << "'");

#ifdef SG_WINDOWS
    // tbd: does Windows support "popen"?
    int rc = system( command.c_str() );
#else
    FILE* pipe = popen( command.c_str(), "r");
    int rc=-1;
    // wait for external process to finish
    if (pipe)
        rc = pclose(pipe);
#endif

    if (rc)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Failed to synchronize directory '" << dir << "', " <<
               "error code= " << rc);
        return false;
    }
    return true;
}

void SGTerraSync::SvnThread::run()
{
    _active = true;
    while (!_stop)
    {
        WaitingTile next = waitingTiles.pop_front();
        if (_stop)
           break;

        CompletedTiles::iterator ii =
            _completedTiles.find( next._dir );
        time_t now = time(0);
        if ((ii == _completedTiles.end())||
            (ii->second < now ))
        {
            bool isNewDirectory = false;

            _busy = true;
            if (!syncTree(next._dir.c_str(),isNewDirectory))
            {
                _consecutive_errors++;
                _fail_count++;
                _completedTiles[ next._dir ] = now + UpdateInterval::FailedAttempt;
            }
            else
            {
                _consecutive_errors = 0;
                _success_count++;
                SG_LOG(SG_TERRAIN,SG_INFO,
                       "Successfully synchronized directory '" << next._dir << "'");
                if (next._refreshScenery)
                {
                    // updated a tile
                    _updated_tile_count++;
                    if (isNewDirectory)
                    {
                        // for now only report new directories to refresh display
                        // (i.e. only when ocean needs to be replaced with actual data)
                        _freshTiles.push_back(next);
                        _is_dirty = true;
                    }
                }
                _completedTiles[ next._dir ] = now + UpdateInterval::SuccessfulAttempt;
            }
            _busy = false;
        }

        if ((_allowed_errors >= 0)&&
            (_consecutive_errors >= _allowed_errors))
        {
            _stalled = true;
            _stop = true;
        }
    }

    _active = false;
    _running = false;
    _is_dirty = true;
}

#if defined(HAVE_SVN_CLIENT_H)
// Configure our subversion session
int SGTerraSync::SvnThread::svnClientSetup(void)
{
    // Are we already prepared?
    if (_svn_pool) return EXIT_SUCCESS;
    // No, so initialize svn internals generally

#ifdef _MSC_VER
    // there is a segfault when providing an error stream.
    //  Apparently, calling setvbuf with a nul buffer is
    //  not supported under msvc 7.1 ( code inside svn_cmdline_init )
    if (svn_cmdline_init("terrasync", 0) != EXIT_SUCCESS)
        return EXIT_FAILURE;

    // revert locale setting
    setlocale(LC_ALL,"C");
#else
    /* svn_cmdline_init configures the locale. Setup environment to ensure the
     * default "C" locale remains active, since fgfs isn't locale aware - especially
     * requires "." as decimal point in strings containing floating point varibales. */
    setenv("LC_ALL", "C", 1);

    if (svn_cmdline_init("terrasync", stderr) != EXIT_SUCCESS)
        return EXIT_FAILURE;
#endif

    apr_pool_t *pool = NULL;
    
    apr_allocator_t* allocator = NULL;
    int aprErr = apr_allocator_create(&allocator);
    if (aprErr != APR_SUCCESS)
        return EXIT_FAILURE;
    
    apr_pool_create_ex(&pool, NULL /* parent pool */, NULL /* abort func */, allocator);
    
    svn_error_t *err = NULL;
    SVN_VERSION_DEFINE(_svn_version);
    err = svn_ver_check_list(&_svn_version, mysvn_checklist);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_ra_initialize(pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    char *config_dir = NULL;
    err = svn_config_ensure(config_dir, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_client_create_context(&_svn_ctx, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_config_get_config(&(_svn_ctx->config),
        config_dir, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    svn_config_t *cfg;
    cfg = ( svn_config_t*) apr_hash_get(
        _svn_ctx->config,
        SVN_CONFIG_CATEGORY_CONFIG,
        APR_HASH_KEY_STRING);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");

    svn_auth_baton_t *ab=NULL;

#if (SVN_VER_MINOR >= 6)
    err = svn_cmdline_create_auth_baton  (&ab,
            TRUE, NULL, NULL, config_dir, TRUE, FALSE, cfg,
            _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
#else
    err = svn_cmdline_setup_auth_baton(&ab,
        TRUE, NULL, NULL, config_dir, TRUE, cfg,
        _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
#endif

    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    
    _svn_ctx->auth_baton = ab;
#if (SVN_VER_MINOR >= 5)
    _svn_ctx->conflict_func = NULL;
    _svn_ctx->conflict_baton = NULL;
#endif

    // Now our magic revisions
    _svn_rev = (svn_opt_revision_t*) apr_palloc(pool, 
        sizeof(svn_opt_revision_t));
    if (!_svn_rev)
        return EXIT_FAILURE;
    _svn_rev_peg = (svn_opt_revision_t*) apr_palloc(pool, 
        sizeof(svn_opt_revision_t));
    if (!_svn_rev_peg)
        return EXIT_FAILURE;
    _svn_rev->kind = svn_opt_revision_head;
    _svn_rev_peg->kind = svn_opt_revision_unspecified;
    // Success if we got this far
    _svn_pool = pool;
    return EXIT_SUCCESS;
}
#endif // of defined(HAVE_SVN_CLIENT_H)

///////////////////////////////////////////////////////////////////////////////
// SGTerraSync ////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
SGTerraSync::SGTerraSync(SGPropertyNode_ptr root) :
    _svnThread(NULL),
    last_lat(NOWHERE),
    last_lon(NOWHERE),
    _terraRoot(root->getNode("/sim/terrasync",true)),
    _refreshCb(NULL),
    _userCbData(NULL)
{
    _svnThread = new SvnThread();
    _log = new BufferedLogCallback(SG_TERRAIN, SG_INFO);
    _log->truncateAt(255);
    
    sglog().addCallback(_log);
}

SGTerraSync::~SGTerraSync()
{
    _tiedProperties.Untie();
    delete _svnThread;
    _svnThread = NULL;
    sglog().removeCallback(_log);
    delete _log;
}

void SGTerraSync::init()
{
    _refreshDisplay = _terraRoot->getNode("refresh-display",true);
    _terraRoot->setBoolValue("built-in-svn-available",svn_built_in_available);
    reinit();
}

void SGTerraSync::reinit()
{
    // do not reinit when enabled and we're already up and running
    if ((_terraRoot->getBoolValue("enabled",false))&&
         (_svnThread->_active && _svnThread->_running))
        return;

    _svnThread->stop();

    if (_terraRoot->getBoolValue("enabled",false))
    {
        _svnThread->setSvnServer(_terraRoot->getStringValue("svn-server",""));
        _svnThread->setRsyncServer(_terraRoot->getStringValue("rsync-server",""));
        _svnThread->setLocalDir(_terraRoot->getStringValue("scenery-dir",""));
        _svnThread->setAllowedErrorCount(_terraRoot->getIntValue("max-errors",5));

    #if defined(HAVE_SVN_CLIENT_H) or defined(SG_SVN_CLIENT)
        _svnThread->setUseBuiltin(_terraRoot->getBoolValue("use-built-in-svn",true));
    #else
        _terraRoot->setBoolValue("use-built-in-svn",false);
    #endif<commit_msg>Fix preprocessor boolean ops for MSVC<commit_after>// terrasync.cxx -- scenery fetcher
//
// Started by <NAME>, November 2002.
//
// Copyright (C) 2002  <NAME>  - http://www.flightgear.org/~curt
// Copyright (C) 2008  <NAME> <<EMAIL>>
// Copyright (C) 2011  <NAME> <<EMAIL>>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
//
// $Id$

#ifdef HAVE_CONFIG_H
#  include <simgear_config.h>
#endif

#include <simgear/compiler.h>

#ifdef HAVE_WINDOWS_H
#include <windows.h>
#endif

#ifdef __MINGW32__
#include <time.h>
#include <unistd.h>
#elif defined(_MSC_VER)
#   include <io.h>
#   ifndef HAVE_SVN_CLIENT_H
#       include <time.h>
#       include <process.h>
#   endif
#endif

#include <stdlib.h>             // atoi() atof() abs() system()
#include <signal.h>             // signal()
#include <string.h>

#include <iostream>
#include <fstream>
#include <string>
#include <map>

#include <simgear/compiler.h>

#include "terrasync.hxx"
#include <simgear/bucket/newbucket.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/threads/SGQueue.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/debug/BufferedLogCallback.hxx>

#ifdef SG_SVN_CLIENT
#  include <simgear/io/HTTPClient.hxx>
#  include <simgear/io/SVNRepository.hxx>
#endif

#ifdef HAVE_SVN_CLIENT_H
#  ifdef HAVE_LIBSVN_CLIENT_1
#    include <svn_version.h>
#    include <svn_auth.h>
#    include <svn_client.h>
#    include <svn_cmdline.h>
#    include <svn_pools.h>
#  else
#    undef HAVE_SVN_CLIENT_H
#  endif
#endif

#ifdef HAVE_SVN_CLIENT_H
    static const svn_version_checklist_t mysvn_checklist[] = {
        { "svn_subr",   svn_subr_version },
        { "svn_client", svn_client_version },
        { NULL, NULL }
    };
#endif
    
#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
    static const bool svn_built_in_available = true;
#else
    static const bool svn_built_in_available = false;
#endif

using namespace simgear;
using namespace std;

const char* rsync_cmd = 
        "rsync --verbose --archive --delete --perms --owner --group";

const char* svn_options =
        "checkout -q";

namespace UpdateInterval
{
    // interval in seconds to allow an update to repeat after a successful update (=daily)
    static const double SuccessfulAttempt = 24*60*60;
    // interval in seconds to allow another update after a failed attempt (10 minutes)
    static const double FailedAttempt     = 10*60;
}

typedef map<string,time_t> CompletedTiles;

///////////////////////////////////////////////////////////////////////////////
// helper functions ///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
string stripPath(string path)
{
    // svn doesn't like trailing white-spaces or path separators - strip them!
    path = simgear::strutils::strip(path);
    size_t slen = path.length();
    while ((slen>0)&&
            ((path[slen-1]=='/')||(path[slen-1]=='\\')))
    {
        slen--;
    }
    return path.substr(0,slen);
}

bool hasWhitespace(string path)
{
    return path.find(' ')!=string::npos;
}

///////////////////////////////////////////////////////////////////////////////
// WaitingTile ////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
class  WaitingTile
{
public:
    WaitingTile(string dir,bool refresh) :
        _dir(dir), _refreshScenery(refresh) {}
    string _dir;
    bool _refreshScenery;
};

///////////////////////////////////////////////////////////////////////////////
// SGTerraSync::SvnThread /////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
class SGTerraSync::SvnThread : public SGThread
{
public:
   SvnThread();
   virtual ~SvnThread( ) { stop(); }

   void stop();
   bool start();

   bool isIdle() {return waitingTiles.empty();}
   void request(const WaitingTile& dir) {waitingTiles.push_front(dir);}
   bool isDirty() { bool r = _is_dirty;_is_dirty = false;return r;}
   bool hasNewTiles() { return !_freshTiles.empty();}
   WaitingTile getNewTile() { return _freshTiles.pop_front();}

   void   setSvnServer(string server)       { _svn_server   = stripPath(server);}
   void   setExtSvnUtility(string svn_util) { _svn_command  = simgear::strutils::strip(svn_util);}
   void   setRsyncServer(string server)     { _rsync_server = simgear::strutils::strip(server);}
   void   setLocalDir(string dir)           { _local_dir    = stripPath(dir);}
   string getLocalDir()                     { return _local_dir;}
   void   setUseSvn(bool use_svn)           { _use_svn = use_svn;}
   void   setAllowedErrorCount(int errors)  {_allowed_errors = errors;}

#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
   void setUseBuiltin(bool built_in) { _use_built_in = built_in;}
#endif

   volatile bool _active;
   volatile bool _running;
   volatile bool _busy;
   volatile bool _stalled;
   volatile int  _fail_count;
   volatile int  _updated_tile_count;
   volatile int  _success_count;
   volatile int  _consecutive_errors;
   volatile int  _allowed_errors;

private:
   virtual void run();
   bool syncTree(const char* dir, bool& isNewDirectory);
   bool syncTreeExternal(const char* dir);


#if defined(SG_SVN_CLIENT)
   bool syncTreeInternal(const char* dir);
   bool _use_built_in;
   HTTP::Client _http;
   std::auto_ptr<SVNRepository> _repository;
#elif defined(HAVE_SVN_CLIENT_H)
   static int svnClientSetup(void);
   bool syncTreeInternal(const char* dir);

   bool _use_built_in;

   // Things we need for doing subversion checkout - often
   static apr_pool_t *_svn_pool;
   static svn_client_ctx_t *_svn_ctx;
   static svn_opt_revision_t *_svn_rev;
   static svn_opt_revision_t *_svn_rev_peg;
#endif

   volatile bool _is_dirty;
   volatile bool _stop;
   SGBlockingDeque <WaitingTile> waitingTiles;
   CompletedTiles _completedTiles;
   SGBlockingDeque <WaitingTile> _freshTiles;
   bool _use_svn;
   string _svn_server;
   string _svn_command;
   string _rsync_server;
   string _local_dir;
};

#ifdef HAVE_SVN_CLIENT_H
    apr_pool_t* SGTerraSync::SvnThread::_svn_pool = NULL;
    svn_client_ctx_t* SGTerraSync::SvnThread::_svn_ctx = NULL;
    svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev = NULL;
    svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev_peg = NULL;
#endif

SGTerraSync::SvnThread::SvnThread() :
    _active(false),
    _running(false),
    _busy(false),
    _stalled(false),
    _fail_count(0),
    _updated_tile_count(0),
    _success_count(0),
    _consecutive_errors(0),
    _allowed_errors(6),
#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
    _use_built_in(true),
#endif
    _is_dirty(false),
    _stop(false),
    _use_svn(true)
{
#ifdef HAVE_SVN_CLIENT_H
    int errCode = SGTerraSync::SvnThread::svnClientSetup();
    if (errCode != EXIT_SUCCESS)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Failed to initialize built-in SVN client, error = " << errCode);
    }
#endif
}

void SGTerraSync::SvnThread::stop()
{
    // drop any pending requests
    waitingTiles.clear();

    if (!_running)
        return;

    // set stop flag and wake up the thread with an empty request
    _stop = true;
    WaitingTile w("",false);
    request(w);
    join();
    _running = false;
}

bool SGTerraSync::SvnThread::start()
{
    if (_running)
        return false;

    if (_local_dir=="")
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Local cache directory is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    SGPath path(_local_dir);
    if (!path.exists())
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Directory '" << _local_dir <<
               "' does not exist. Set correct directory path or create directory folder.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    path.append("version");
    if (path.exists())
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Directory '" << _local_dir <<
               "' contains the base package. Use a separate directory.");
        _fail_count++;
        _stalled = true;
        return false;
    }

#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
    _use_svn |= _use_built_in;
#endif

    if ((_use_svn)&&(_svn_server==""))
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Subversion scenery server is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }
    if ((!_use_svn)&&(_rsync_server==""))
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Cannot start scenery download. Rsync scenery server is undefined.");
        _fail_count++;
        _stalled = true;
        return false;
    }

    _fail_count = 0;
    _updated_tile_count = 0;
    _success_count = 0;
    _consecutive_errors = 0;
    _stop = false;
    _stalled = false;
    _running = true;

    string status;
#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
    if (_use_svn && _use_built_in)
        status = "Using built-in SVN support. ";
    else
#endif
    if (_use_svn)
    {
        status = "Using external SVN utility '";
        status += _svn_command;
        status += "'. ";
    }
    else
    {
        status = "Using RSYNC. ";
    }

    // not really an alert - but we want to (always) see this message, so user is
    // aware we're downloading scenery (and using bandwidth).
    SG_LOG(SG_TERRAIN,SG_ALERT,
           "Starting automatic scenery download/synchronization. "
           << status
           << "Directory: '" << _local_dir << "'.");

    SGThread::start();
    return true;
}

// sync one directory tree
bool SGTerraSync::SvnThread::syncTree(const char* dir, bool& isNewDirectory)
{
    int rc;
    SGPath path( _local_dir );

    path.append( dir );
    isNewDirectory = !path.exists();
    if (isNewDirectory)
    {
        rc = path.create_dir( 0755 );
        if (rc)
        {
            SG_LOG(SG_TERRAIN,SG_ALERT,
                   "Cannot create directory '" << dir << "', return code = " << rc );
            return false;
        }
    }

#if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
    if (_use_built_in)
        return syncTreeInternal(dir);
    else
#endif
    {
        return syncTreeExternal(dir);
    }
}

#if defined(SG_SVN_CLIENT)
    
bool SGTerraSync::SvnThread::syncTreeInternal(const char* dir)
{
    ostringstream command;
    command << _svn_server << "/" << dir;

    SGPath path(_local_dir);
    path.append(dir);
    _repository.reset(new SVNRepository(path, &_http));
    _repository->setBaseUrl(command.str());
    
    SGTimeStamp st;
    st.stamp();
    SG_LOG(SG_IO, SG_DEBUG, "terrasync: will sync " << command.str());
    _repository->update();
    
    bool result = true;
    while (!_stop && _repository->isDoingSync()) {
        _http.update(100);
    }
    
    if (_repository->failure() == SVNRepository::SVN_ERROR_NOT_FOUND) {
        // this is fine, but maybe we should use a different return code
        // in the future to higher layers can distuinguish this case
    } else if (_repository->failure() != SVNRepository::SVN_NO_ERROR) {
        result = false;
    } else {
        SG_LOG(SG_IO, SG_DEBUG, "sync of " << command.str() << " finished ("
            << st.elapsedMSec() << " msec");
    }
    
    _repository.reset();
    return result;
}
    
#elif defined(HAVE_SVN_CLIENT_H)

bool SGTerraSync::SvnThread::syncTreeInternal(const char* dir)
{
    SG_LOG(SG_TERRAIN,SG_DEBUG, "Synchronizing scenery directory " << dir);
    if (!_svn_pool)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Built-in SVN client failed to initialize.");
        return false;
    }

    ostringstream command;
    command << _svn_server << "/" << dir;

    ostringstream dest_base_dir;
    dest_base_dir << _local_dir << "/" << dir;

    apr_pool_t *subpool = svn_pool_create(_svn_pool);

    svn_error_t *err = NULL;
#if (SVN_VER_MINOR >= 5)
    err = svn_client_checkout3(NULL,
            command.str().c_str(),
            dest_base_dir.str().c_str(),
            _svn_rev_peg,
            _svn_rev,
            svn_depth_infinity,
            0, // ignore-externals = false
            0, // allow unver obstructions = false
            _svn_ctx,
            subpool);
#else
    // version 1.4 API
    err = svn_client_checkout2(NULL,
            command.str().c_str(),
            dest_base_dir.str().c_str(),
            _svn_rev_peg,
            _svn_rev,
            1, // recurse=true - same as svn_depth_infinity for checkout3 above
            0, // ignore externals = false
            _svn_ctx,
            subpool);
#endif

    bool ReturnValue = true;
    if (err)
    {
        // Report errors from the checkout attempt
        if (err->apr_err == SVN_ERR_RA_ILLEGAL_URL)
        {
            // ignore errors when remote path doesn't exist (no scenery data for ocean areas)
        }
        else
        {
            SG_LOG(SG_TERRAIN,SG_ALERT,
                    "Failed to synchronize directory '" << dir << "', " <<
                    err->message << " (code " << err->apr_err << ").");
            svn_error_clear(err);
            // try to clean up
            err = svn_client_cleanup(dest_base_dir.str().c_str(),
                    _svn_ctx,subpool);
            if (!err)
            {
                SG_LOG(SG_TERRAIN,SG_ALERT,
                       "SVN repository cleanup successful for '" << dir << "'.");
            }
            ReturnValue = false;
        }
    } else
    {
        SG_LOG(SG_TERRAIN,SG_DEBUG, "Done with scenery directory " << dir);
    }
    svn_pool_destroy(subpool);
    return ReturnValue;
}
#endif // of HAVE_SVN_CLIENT_H

bool SGTerraSync::SvnThread::syncTreeExternal(const char* dir)
{
    ostringstream buf;
    SGPath localPath( _local_dir );
    localPath.append( dir );

    if (_use_svn)
    {
        buf << "\"" << _svn_command << "\" "
            << svn_options << " "
            << "\"" << _svn_server << "/" << dir << "\" "
            << "\"" << localPath.str_native() << "\"";
    } else {
        buf << rsync_cmd << " "
            << "\"" << _rsync_server << "/" << dir << "/\" "
            << "\"" << localPath.str_native() << "/\"";
    }

    string command;
#ifdef SG_WINDOWS
        // windows command line parsing is just lovely...
        // to allow white spaces, the system call needs this:
        // ""C:\Program Files\something.exe" somearg "some other arg""
        // Note: whitespace strings quoted by a pair of "" _and_ the 
        //       entire string needs to be wrapped by "" too.
        // The svn url needs forward slashes (/) as a path separator while
        // the local path needs windows-native backslash as a path separator.
    command = "\"" + buf.str() + "\"";
#else
    command = buf.str();
#endif
    SG_LOG(SG_TERRAIN,SG_DEBUG, "sync command '" << command << "'");

#ifdef SG_WINDOWS
    // tbd: does Windows support "popen"?
    int rc = system( command.c_str() );
#else
    FILE* pipe = popen( command.c_str(), "r");
    int rc=-1;
    // wait for external process to finish
    if (pipe)
        rc = pclose(pipe);
#endif

    if (rc)
    {
        SG_LOG(SG_TERRAIN,SG_ALERT,
               "Failed to synchronize directory '" << dir << "', " <<
               "error code= " << rc);
        return false;
    }
    return true;
}

void SGTerraSync::SvnThread::run()
{
    _active = true;
    while (!_stop)
    {
        WaitingTile next = waitingTiles.pop_front();
        if (_stop)
           break;

        CompletedTiles::iterator ii =
            _completedTiles.find( next._dir );
        time_t now = time(0);
        if ((ii == _completedTiles.end())||
            (ii->second < now ))
        {
            bool isNewDirectory = false;

            _busy = true;
            if (!syncTree(next._dir.c_str(),isNewDirectory))
            {
                _consecutive_errors++;
                _fail_count++;
                _completedTiles[ next._dir ] = now + UpdateInterval::FailedAttempt;
            }
            else
            {
                _consecutive_errors = 0;
                _success_count++;
                SG_LOG(SG_TERRAIN,SG_INFO,
                       "Successfully synchronized directory '" << next._dir << "'");
                if (next._refreshScenery)
                {
                    // updated a tile
                    _updated_tile_count++;
                    if (isNewDirectory)
                    {
                        // for now only report new directories to refresh display
                        // (i.e. only when ocean needs to be replaced with actual data)
                        _freshTiles.push_back(next);
                        _is_dirty = true;
                    }
                }
                _completedTiles[ next._dir ] = now + UpdateInterval::SuccessfulAttempt;
            }
            _busy = false;
        }

        if ((_allowed_errors >= 0)&&
            (_consecutive_errors >= _allowed_errors))
        {
            _stalled = true;
            _stop = true;
        }
    }

    _active = false;
    _running = false;
    _is_dirty = true;
}

#if defined(HAVE_SVN_CLIENT_H)
// Configure our subversion session
int SGTerraSync::SvnThread::svnClientSetup(void)
{
    // Are we already prepared?
    if (_svn_pool) return EXIT_SUCCESS;
    // No, so initialize svn internals generally

#ifdef _MSC_VER
    // there is a segfault when providing an error stream.
    //  Apparently, calling setvbuf with a nul buffer is
    //  not supported under msvc 7.1 ( code inside svn_cmdline_init )
    if (svn_cmdline_init("terrasync", 0) != EXIT_SUCCESS)
        return EXIT_FAILURE;

    // revert locale setting
    setlocale(LC_ALL,"C");
#else
    /* svn_cmdline_init configures the locale. Setup environment to ensure the
     * default "C" locale remains active, since fgfs isn't locale aware - especially
     * requires "." as decimal point in strings containing floating point varibales. */
    setenv("LC_ALL", "C", 1);

    if (svn_cmdline_init("terrasync", stderr) != EXIT_SUCCESS)
        return EXIT_FAILURE;
#endif

    apr_pool_t *pool = NULL;
    
    apr_allocator_t* allocator = NULL;
    int aprErr = apr_allocator_create(&allocator);
    if (aprErr != APR_SUCCESS)
        return EXIT_FAILURE;
    
    apr_pool_create_ex(&pool, NULL /* parent pool */, NULL /* abort func */, allocator);
    
    svn_error_t *err = NULL;
    SVN_VERSION_DEFINE(_svn_version);
    err = svn_ver_check_list(&_svn_version, mysvn_checklist);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_ra_initialize(pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    char *config_dir = NULL;
    err = svn_config_ensure(config_dir, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_client_create_context(&_svn_ctx, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    err = svn_config_get_config(&(_svn_ctx->config),
        config_dir, pool);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    svn_config_t *cfg;
    cfg = ( svn_config_t*) apr_hash_get(
        _svn_ctx->config,
        SVN_CONFIG_CATEGORY_CONFIG,
        APR_HASH_KEY_STRING);
    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");

    svn_auth_baton_t *ab=NULL;

#if (SVN_VER_MINOR >= 6)
    err = svn_cmdline_create_auth_baton  (&ab,
            TRUE, NULL, NULL, config_dir, TRUE, FALSE, cfg,
            _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
#else
    err = svn_cmdline_setup_auth_baton(&ab,
        TRUE, NULL, NULL, config_dir, TRUE, cfg,
        _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
#endif

    if (err)
        return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
    
    _svn_ctx->auth_baton = ab;
#if (SVN_VER_MINOR >= 5)
    _svn_ctx->conflict_func = NULL;
    _svn_ctx->conflict_baton = NULL;
#endif

    // Now our magic revisions
    _svn_rev = (svn_opt_revision_t*) apr_palloc(pool, 
        sizeof(svn_opt_revision_t));
    if (!_svn_rev)
        return EXIT_FAILURE;
    _svn_rev_peg = (svn_opt_revision_t*) apr_palloc(pool, 
        sizeof(svn_opt_revision_t));
    if (!_svn_rev_peg)
        return EXIT_FAILURE;
    _svn_rev->kind = svn_opt_revision_head;
    _svn_rev_peg->kind = svn_opt_revision_unspecified;
    // Success if we got this far
    _svn_pool = pool;
    return EXIT_SUCCESS;
}
#endif // of defined(HAVE_SVN_CLIENT_H)

///////////////////////////////////////////////////////////////////////////////
// SGTerraSync ////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
SGTerraSync::SGTerraSync(SGPropertyNode_ptr root) :
    _svnThread(NULL),
    last_lat(NOWHERE),
    last_lon(NOWHERE),
    _terraRoot(root->getNode("/sim/terrasync",true)),
    _refreshCb(NULL),
    _userCbData(NULL)
{
    _svnThread = new SvnThread();
    _log = new BufferedLogCallback(SG_TERRAIN, SG_INFO);
    _log->truncateAt(255);
    
    sglog().addCallback(_log);
}

SGTerraSync::~SGTerraSync()
{
    _tiedProperties.Untie();
    delete _svnThread;
    _svnThread = NULL;
    sglog().removeCallback(_log);
    delete _log;
}

void SGTerraSync::init()
{
    _refreshDisplay = _terraRoot->getNode("refresh-display",true);
    _terraRoot->setBoolValue("built-in-svn-available",svn_built_in_available);
    reinit();
}

void SGTerraSync::reinit()
{
    // do not reinit when enabled and we're already up and running
    if ((_terraRoot->getBoolValue("enabled",false))&&
         (_svnThread->_active && _svnThread->_running))
        return;

    _svnThread->stop();

    if (_terraRoot->getBoolValue("enabled",false))
    {
        _svnThread->setSvnServer(_terraRoot->getStringValue("svn-server",""));
        _svnThread->setRsyncServer(_terraRoot->getStringValue("rsync-server",""));
        _svnThread->setLocalDir(_terraRoot->getStringValue("scenery-dir",""));
        _svnThread->setAllowedErrorCount(_terraRoot->getIntValue("max-errors",5));

    #if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT)
        _svnThread->setUseBuiltin(_terraRoot->getBoolValue("use-built-in-svn",true));
    #else
        _terraRoot->setBoolValue("use-built-in-svn",false);
    #endif
{'fraction_non_alphanumeric': 0.1499485155311481, 'fraction_numerical': 0.003732623991762485, 'mean_word_length': 2.277809028266067, 'pattern_counts': {'":': 0, '<': 355, '<?xml version=': 0, '>': 177, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '502420', 'n_tokens_mistral': 15410, 'n_tokens_neox': 14999, 'n_words': 3387}