<commit_before>/*
 * main.c: Subversion server inspection tool.
 *
 * ====================================================================
 * Copyright (c) 2000-2001 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 * ====================================================================
 */

#include <apr_general.h>
#include <apr_pools.h>
#include <apr_time.h>
#include <apr_thread_proc.h>
#include <apr_file_io.h>

#define APR_WANT_STDIO
#define APR_WANT_STRFUNC
#include <apr_want.h>

#include "svn_types.h"
#include "svn_pools.h"
#include "svn_error.h"
#include "svn_path.h"
#include "svn_fs.h"
#include "svn_repos.h"
#include "svn_time.h"
#include "svn_private_config.h"         /* for SVN_CLIENT_DIFF */



/*** Some convenience macros and types. ***/

/* Names used for directories that the `diff' subcommand creates. */
#define SVNLOOK_DIFF_TMPDIR_BASE  "OLD"
#define SVNLOOK_DIFF_TMPDIR_NEW   "NEW"

#define INT_ERR(expr)                              \
  do {                                             \
    svn_error_t *svn_err__temp = (expr);           \
    if (svn_err__temp) {                           \
      svn_handle_error (svn_err__temp, stdout, 0); \
      return (1); }                                \
  } while (0)


typedef enum svnlook_cmd_t
{
  svnlook_cmd_default = 0,
  svnlook_cmd_log,
  svnlook_cmd_author,
  svnlook_cmd_date,
  svnlook_cmd_dirs_changed,
  svnlook_cmd_changed,
  svnlook_cmd_diff,
  svnlook_cmd_ids,
  
} svnlook_cmd_t;


typedef struct svnlook_ctxt_t
{
  svn_fs_t *fs;
  svn_boolean_t is_revision;
  svn_revnum_t rev_id;
  svn_fs_txn_t *txn;
  char *txn_name;

} svnlook_ctxt_t;




/*** Helper functions. ***/
static svn_error_t *
get_property (svn_stringbuf_t **prop_value, 
              svnlook_ctxt_t *c, 
              svn_string_t *prop_name,
              apr_pool_t *pool)
{
  /* Fetch transaction property... */
  if (! c->is_revision)
    return svn_fs_txn_prop (prop_value, c->txn, prop_name, pool);

  /* ...or revision property -- it's your call. */
  return svn_fs_revision_prop (prop_value, c->fs, c->rev_id, prop_name, pool);
}


static svn_error_t *
get_root (svn_fs_root_t **root,
          svnlook_ctxt_t *c,
          apr_pool_t *pool)
{
  /* Open up the appropriate root (revision or transaction). */
  if (c->is_revision)
    {
      /* If we didn't get a valid revision number, we'll look at the
         youngest revision. */
      if (! SVN_IS_VALID_REVNUM (c->rev_id))
        SVN_ERR (svn_fs_youngest_rev (&(c->rev_id), c->fs, pool));

      SVN_ERR (svn_fs_revision_root (root, c->fs, c->rev_id, pool));
    }
  else
    {
      SVN_ERR (svn_fs_txn_root (root, c->txn, pool));
    }

  return SVN_NO_ERROR;
}




/*** Tree Routines ***/

/* Generate a generic delta tree. */
static svn_error_t *
generate_delta_tree (svn_repos_node_t **tree,
                     svn_fs_t *fs,
                     svn_fs_root_t *root, 
                     svn_revnum_t base_rev, 
                     apr_pool_t *pool)
{
  svn_fs_root_t *base_root;
  const svn_delta_edit_fns_t *editor;
  void *edit_baton;
  apr_hash_t *src_revs = apr_hash_make (pool);

  /* Get the current root. */
  apr_hash_set (src_revs, "", APR_HASH_KEY_STRING, &base_rev);

  /* Get the base root. */
  SVN_ERR (svn_fs_revision_root (&base_root, fs, base_rev, pool));

  /* Request our editor. */
  SVN_ERR (svn_repos_node_editor (&editor, &edit_baton, fs,
                                  root, base_root, pool));
  
  /* Drive our editor. */
  SVN_ERR (svn_repos_dir_delta (base_root, 
                                svn_stringbuf_create ("", pool), 
                                NULL, src_revs, root, 
                                svn_stringbuf_create ("", pool), 
                                editor, edit_baton, pool));

  /* Return the tree we just built. */
  *tree = svn_repos_node_from_baton (edit_baton);
  return SVN_NO_ERROR;
}




/*** Tree Printing Routines ***/

/* Recursively print only directory nodes that either a) have property
   mods, or b) contains files that have changed. */
static void
print_dirs_changed_tree (svn_repos_node_t *node,
                         svn_stringbuf_t *path,
                         apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  int print_me = 0;
  svn_stringbuf_t *full_path;

  if (! node)
    return;

  /* Not a directory?  We're not interested. */
  if (node->kind != svn_node_dir)
    return;

  /* Got prop mods?  Excellent. */
  if (node->prop_mod)
    print_me = 1;

  if (! print_me)
    {
      /* Fly through the list of children, checking for modified files. */
      tmp_node = node->child;
      if (tmp_node)
        {
          if ((tmp_node->kind == svn_node_file)
              || (tmp_node->text_mod)
              || (tmp_node->action == 'A')
              || (tmp_node->action == 'D'))
            {
              print_me = 1;
            }
          while (tmp_node->sibling && (! print_me ))
            {
              tmp_node = tmp_node->sibling;
              if ((tmp_node->kind == svn_node_file)
                  || (tmp_node->text_mod)
                  || (tmp_node->action == 'A')
                  || (tmp_node->action == 'D'))
                {
                  print_me = 1;
                }
            }
        }
    }
  
  /* Print the node if it qualifies. */
  if (print_me)
    {
      printf ("%s/\n", path->data);
    }

  /* Recursively handle the node's children. */
  tmp_node = node->child;
  if (! tmp_node)
    return;

  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  print_dirs_changed_tree (tmp_node, full_path, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      print_dirs_changed_tree (tmp_node, full_path, pool);
    }

  return;
}


/* Recursively print all nodes in the tree that have been modified
   (do not include directories affected only by "bubble-up"). */
static void
print_changed_tree (svn_repos_node_t *node,
                    svn_stringbuf_t *path,
                    apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  svn_stringbuf_t *full_path;
  char status[3] = "_ ";
  int print_me = 1;

  if (! node)
    return;

  /* Print the node. */
  tmp_node = node;
  if (tmp_node->action == 'A')
    status[0] = 'A';
  else if (tmp_node->action == 'D')
    status[0] = 'D';
  else if (tmp_node->action == 'R')
    {
      if ((! tmp_node->text_mod) && (! tmp_node->prop_mod))
        print_me = 0;
      if (tmp_node->text_mod)
        status[0] = 'U';
      if (tmp_node->prop_mod)
        status[1] = 'U';
    }
  else
    print_me = 0;

  /* Print this node unless told to skip it. */
  if (print_me)
    printf ("%s  %s%s\n",
            status,
            path->data,
            tmp_node->kind == svn_node_dir ? "/" : "");
  
  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return;

  /* Recursively handle the node's children. */
  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  print_changed_tree (tmp_node, full_path, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      print_changed_tree (tmp_node, full_path, pool);
    }

  return;
}


static svn_error_t *
open_writable_binary_file (apr_file_t **fh, 
                           svn_stringbuf_t *path, 
                           apr_pool_t *pool)
{
  apr_array_header_t *path_pieces;
  apr_status_t apr_err;
  int i;
  svn_stringbuf_t *full_path, *dir, *basename;
  
  /* Try the easy way to open the file. */
  apr_err = apr_file_open (fh, path->data, 
                           APR_WRITE | APR_CREATE | APR_TRUNCATE | APR_BINARY,
                           APR_OS_DEFAULT, pool);
  if (! apr_err)
    return SVN_NO_ERROR;

  svn_path_split (path, &dir, &basename, svn_path_local_style, pool);

  /* If the file path has no parent, then we've already tried to open
     it as best as we care to try above. */
  if (svn_path_is_empty (dir, svn_path_local_style))
    return svn_error_createf (apr_err, 0, NULL, pool,
                              "Error opening writable file %s", path->data);

  path_pieces = svn_path_decompose (dir,
                                    svn_path_local_style,
                                    pool);
  if (! path_pieces->nelts)
    return APR_SUCCESS;

  full_path = svn_stringbuf_create ("", pool);
  for (i = 0; i < path_pieces->nelts; i++)
    {
      enum svn_node_kind kind;
      svn_stringbuf_t *piece = ((svn_stringbuf_t **) (path_pieces->elts))[i];
      svn_path_add_component (full_path, piece, svn_path_local_style);
      SVN_ERR (svn_io_check_path (full_path, &kind, pool));

      /* Does this path component exist at all? */
      if (kind == svn_node_none)
        {
          apr_err = apr_dir_make (full_path->data, APR_OS_DEFAULT, pool);
          if (apr_err)
            return svn_error_createf (apr_err, 0, NULL, pool,
                                      "Error creating dir %s", 
                                      full_path->data);
        }
      else if (kind != svn_node_dir)
        {
          if (apr_err)
            return svn_error_createf (apr_err, 0, NULL, pool,
                                      "Error creating dir %s (path exists)", 
                                      full_path->data);
        }
    }

  /* Now that we are ensured that the parent path for this file
     exists, try once more to open it. */
  apr_err = apr_file_open (fh, path->data, 
                           APR_WRITE | APR_CREATE | APR_TRUNCATE | APR_BINARY,
                           APR_OS_DEFAULT, pool);
  if (apr_err)
    return svn_error_createf (apr_err, 0, NULL, pool,
                              "Error opening writable file %s", path->data);
    
  return SVN_NO_ERROR;
}


static svn_error_t *
dump_contents (apr_file_t *fh,
               svn_fs_root_t *root,
               svn_stringbuf_t *path,
               apr_pool_t *pool)
{
  apr_status_t apr_err;
  apr_size_t len, len2;
  svn_stream_t *stream;
  unsigned char buffer[1024];

  /* Get a stream to the current file's contents. */
  SVN_ERR (svn_fs_file_contents (&stream, root, path->data, pool));
  
  /* Now, route that data into our temporary file. */
  while (1)
    {
      len = sizeof (buffer);
      SVN_ERR (svn_stream_read (stream, buffer, &len));
      len2 = len;
      apr_err = apr_file_write (fh, buffer, &len2);
      if ((apr_err) || (len2 != len))
        return svn_error_createf 
          (apr_err ? apr_err : SVN_ERR_INCOMPLETE_DATA, 0, NULL, pool,
               "Error writing contents of %s", path->data);
      if (len != sizeof (buffer))
        break;
    }

  /* And close the file. */
  apr_file_close (fh);
  return SVN_NO_ERROR;
}


/* Recursively print all nodes in the tree that have been modified
   (do not include directories affected only by "bubble-up"). */
static svn_error_t *
print_diff_tree (svn_fs_root_t *root,
                 svn_fs_root_t *base_root,
                 svn_repos_node_t *node, 
                 svn_stringbuf_t *path,
                 apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  svn_stringbuf_t *full_path;
  svn_stringbuf_t *orig_path = NULL, *new_path = NULL;
  apr_file_t *fh1, *fh2;
      
  if (! node)
    return SVN_NO_ERROR;

  /* Print the node. */
  tmp_node = node;

  /* ### todo:  Need a plan for printing diffs here.  */
  /* First, we'll just print file content diffs. */
  if (tmp_node->kind == svn_node_file)
    {
      if ((tmp_node->action == 'R') && (tmp_node->text_mod))
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          SVN_ERR (dump_contents (fh2, base_root, path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          SVN_ERR (dump_contents (fh1, root, path, pool));
          apr_file_close (fh1);
        }
      if (tmp_node->action == 'A')
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          SVN_ERR (dump_contents (fh1, root, path, pool));
          apr_file_close (fh1);
        }
      if (tmp_node->action == 'D')
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          SVN_ERR (dump_contents (fh2, base_root, path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          apr_file_close (fh1);
        }
    }

  if (orig_path && new_path)
    {
      const char *args[5];
      apr_file_t *outhandle;
      apr_status_t apr_err;

      printf ("%s: %s\n", 
              ((tmp_node->action == 'A') ? "Added" : 
               ((tmp_node->action == 'D') ? "Deleted" :
                ((tmp_node->action == 'R') ? "Modified" : "Index"))),
              path->data);
      printf ("===============================================================\
===============\n");
      fflush (stdout);

      args[0] = SVN_CLIENT_DIFF;
      args[1] = "-c";
      args[2] = orig_path->data;
      args[3] = new_path->data;
      args[4] = NULL;

      /* Get an apr_file_t representing stdout, which is where we'll have
         the diff program print to. */
      apr_err = apr_file_open_stdout (&outhandle, pool);
      if (apr_err)
        return svn_error_create 
          (apr_err, 0, NULL, pool,
           "print_diff_tree: can't open handle to stdout");

      SVN_ERR (svn_io_run_cmd (".", SVN_CLIENT_DIFF, args, 
                               NULL, outhandle, NULL, pool));
      printf ("\n");
      fflush (stdout);
    }
  
  /* Now, delete any temporary files. */
  if (orig_path)
    apr_file_remove (orig_path->data, pool);
  if (new_path)
    apr_file_remove (new_path->data, pool);

  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return SVN_NO_ERROR;

  /* Recursively handle the node's children. */
  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  SVN_ERR (print_diff_tree (root, base_root, tmp_node, full_path, pool));
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      SVN_ERR (print_diff_tree (root, base_root, tmp_node, full_path, pool));
    }

  return SVN_NO_ERROR;
}


/* Recursively print all nodes in the tree.  If SHOW_IDS is non-zero,
   print the id of each node next to its name. */
static void
print_tree (svn_repos_node_t *node,
            svn_boolean_t show_ids,
            int indentation,
            apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  int i;

  if (! node)
    return;

  /* Print the indentation. */
  for (i = 0; i < indentation; i++)
    {
      printf (" ");
    }

  /* Print the node. */
  tmp_node = node;
  if (show_ids)
    {
      svn_stringbuf_t *unparsed_id;

      if (tmp_node->id)
        unparsed_id = svn_fs_unparse_id (tmp_node->id, pool);

      printf ("%s%s <%s>\n", 
              tmp_node->name, 
              tmp_node->kind == svn_node_dir ? "/" : "",
              tmp_node->id ? unparsed_id->data : "unknown");
    }
  else
    {
      printf ("%s%s \n", 
              tmp_node->name, 
              tmp_node->kind == svn_node_dir ? "/" : "");
    }

  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return;

  /* Recursively handle the node's children. */
  print_tree (tmp_node, show_ids, indentation + 1, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      print_tree (tmp_node, show_ids, indentation + 1, pool);
    }

  return;
}





/*** Subcommand handlers. ***/

/* Print the revision's log message to stdout, followed by a newline. */
static svn_error_t *
do_log (svnlook_ctxt_t *c, svn_boolean_t print_size, apr_pool_t *pool)
{
  svn_stringbuf_t *prop_value;
  svn_string_t prop_name;
  const char *name = SVN_PROP_REVISION_LOG;

  prop_name.data = name;
  prop_name.len = strlen (name);

  SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

  if (prop_value && prop_value->data)
    {
      if (print_size)
        {
          printf ("%lu\n", (unsigned long)prop_value->len);
        }

      printf ("%s", prop_value->data);
    }
  else if (print_size)
    {
      printf ("0");
    }
  
  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print the timestamp of the commit (in the revision case) or the
   empty string (in the transaction case) to stdout, followed by a
   newline. */
static svn_error_t *
do_date (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  if (c->is_revision)
    {
      svn_stringbuf_t *prop_value;
      svn_string_t prop_name;
      const char *name = SVN_PROP_REVISION_DATE;
      
      prop_name.data = name;
      prop_name.len = strlen (name);
      
      SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

      if (prop_value && prop_value->data)
        {
          /* The date stored in the repository is in a really complex
             and precise format...and we don't care.  Let's convert
             that to just "YYYY-MM-DD hh:mm".

             ### todo: Right now, "svn dates" are not GMT, but the
             results of svn_time_from_string are.  This sucks. */
          apr_exploded_time_t extime;
          apr_time_t aprtime;
          apr_status_t apr_err;
              
          aprtime = svn_time_from_string (prop_value);
          apr_err = apr_explode_time (&extime, aprtime, 0);
          if (apr_err)
            return svn_error_create (apr_err, 0, NULL, pool,
                                     "do_date: error exploding time");
              
          printf ("%04lu-%02lu-%02lu %02lu:%02lu GMT",
                  (unsigned long)(extime.tm_year + 1900),
                  (unsigned long)(extime.tm_mon + 1),
                  (unsigned long)(extime.tm_mday),
                  (unsigned long)(extime.tm_hour),
                  (unsigned long)(extime.tm_min));
        }
    }

  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print the author of the commit to stdout, followed by a newline. */
static svn_error_t *
do_author (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_stringbuf_t *prop_value;
  svn_string_t prop_name;
  const char *name = SVN_PROP_REVISION_AUTHOR;

  prop_name.data = name;
  prop_name.len = strlen (name);

  SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

  if (prop_value && prop_value->data)
    printf ("%s", prop_value->data);
  
  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print a list of all directories in which files, or directory
   properties, have been modified. */
static svn_error_t *
do_dirs_changed (svnlook_ctxt_t *c, apr_pool_t *pool)
{ 
  svn_fs_root_t *root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    print_dirs_changed_tree (tree, svn_stringbuf_create ("", pool), pool);

  return SVN_NO_ERROR;
}


/* Print a list of all paths modified in a format compatible with `svn
   update'. */
static svn_error_t *
do_changed (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_fs_root_t *root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    print_changed_tree (tree, svn_stringbuf_create ("", pool), pool);

  return SVN_NO_ERROR;
}


/* Print some diff-y stuff in a TBD way. :-) */
static svn_error_t *
do_diff (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_fs_root_t *root, *base_root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    {
      apr_status_t apr_err;

      SVN_ERR (svn_fs_revision_root (&base_root, c->fs, base_rev_id, pool));
      SVN_ERR (print_diff_tree 
               (root, base_root, tree, svn_stringbuf_create ("", pool), pool));
      apr_dir_remove_recursively (SVNLOOK_DIFF_TMPDIR_BASE, pool);
      apr_dir_remove_recursively (SVNLOOK_DIFF_TMPDIR_NEW, pool);
    }<commit_msg>* main.c<commit_after>/*
 * main.c: Subversion server inspection tool.
 *
 * ====================================================================
 * Copyright (c) 2000-2001 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 * ====================================================================
 */

#include <apr_general.h>
#include <apr_pools.h>
#include <apr_time.h>
#include <apr_thread_proc.h>
#include <apr_file_io.h>

#define APR_WANT_STDIO
#define APR_WANT_STRFUNC
#include <apr_want.h>

#include "svn_types.h"
#include "svn_pools.h"
#include "svn_error.h"
#include "svn_path.h"
#include "svn_fs.h"
#include "svn_repos.h"
#include "svn_time.h"
#include "svn_private_config.h"         /* for SVN_CLIENT_DIFF */



/*** Some convenience macros and types. ***/

/* Names used for directories that the `diff' subcommand creates. */
#define SVNLOOK_DIFF_TMPDIR_BASE  "OLD"
#define SVNLOOK_DIFF_TMPDIR_NEW   "NEW"

#define INT_ERR(expr)                              \
  do {                                             \
    svn_error_t *svn_err__temp = (expr);           \
    if (svn_err__temp) {                           \
      svn_handle_error (svn_err__temp, stdout, 0); \
      return (1); }                                \
  } while (0)


typedef enum svnlook_cmd_t
{
  svnlook_cmd_default = 0,
  svnlook_cmd_log,
  svnlook_cmd_author,
  svnlook_cmd_date,
  svnlook_cmd_dirs_changed,
  svnlook_cmd_changed,
  svnlook_cmd_diff,
  svnlook_cmd_ids,
  
} svnlook_cmd_t;


typedef struct svnlook_ctxt_t
{
  svn_fs_t *fs;
  svn_boolean_t is_revision;
  svn_revnum_t rev_id;
  svn_fs_txn_t *txn;
  char *txn_name;

} svnlook_ctxt_t;




/*** Helper functions. ***/
static svn_error_t *
get_property (svn_stringbuf_t **prop_value, 
              svnlook_ctxt_t *c, 
              svn_string_t *prop_name,
              apr_pool_t *pool)
{
  /* Fetch transaction property... */
  if (! c->is_revision)
    return svn_fs_txn_prop (prop_value, c->txn, prop_name, pool);

  /* ...or revision property -- it's your call. */
  return svn_fs_revision_prop (prop_value, c->fs, c->rev_id, prop_name, pool);
}


static svn_error_t *
get_root (svn_fs_root_t **root,
          svnlook_ctxt_t *c,
          apr_pool_t *pool)
{
  /* Open up the appropriate root (revision or transaction). */
  if (c->is_revision)
    {
      /* If we didn't get a valid revision number, we'll look at the
         youngest revision. */
      if (! SVN_IS_VALID_REVNUM (c->rev_id))
        SVN_ERR (svn_fs_youngest_rev (&(c->rev_id), c->fs, pool));

      SVN_ERR (svn_fs_revision_root (root, c->fs, c->rev_id, pool));
    }
  else
    {
      SVN_ERR (svn_fs_txn_root (root, c->txn, pool));
    }

  return SVN_NO_ERROR;
}




/*** Tree Routines ***/

/* Generate a generic delta tree. */
static svn_error_t *
generate_delta_tree (svn_repos_node_t **tree,
                     svn_fs_t *fs,
                     svn_fs_root_t *root, 
                     svn_revnum_t base_rev, 
                     apr_pool_t *pool)
{
  svn_fs_root_t *base_root;
  const svn_delta_edit_fns_t *editor;
  void *edit_baton;
  apr_hash_t *src_revs = apr_hash_make (pool);

  /* Get the current root. */
  apr_hash_set (src_revs, "", APR_HASH_KEY_STRING, &base_rev);

  /* Get the base root. */
  SVN_ERR (svn_fs_revision_root (&base_root, fs, base_rev, pool));

  /* Request our editor. */
  SVN_ERR (svn_repos_node_editor (&editor, &edit_baton, fs,
                                  root, base_root, pool));
  
  /* Drive our editor. */
  SVN_ERR (svn_repos_dir_delta (base_root, 
                                svn_stringbuf_create ("", pool), 
                                NULL, src_revs, root, 
                                svn_stringbuf_create ("", pool), 
                                editor, edit_baton, pool));

  /* Return the tree we just built. */
  *tree = svn_repos_node_from_baton (edit_baton);
  return SVN_NO_ERROR;
}




/*** Tree Printing Routines ***/

/* Recursively print only directory nodes that either a) have property
   mods, or b) contains files that have changed. */
static void
print_dirs_changed_tree (svn_repos_node_t *node,
                         svn_stringbuf_t *path,
                         apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  int print_me = 0;
  svn_stringbuf_t *full_path;

  if (! node)
    return;

  /* Not a directory?  We're not interested. */
  if (node->kind != svn_node_dir)
    return;

  /* Got prop mods?  Excellent. */
  if (node->prop_mod)
    print_me = 1;

  if (! print_me)
    {
      /* Fly through the list of children, checking for modified files. */
      tmp_node = node->child;
      if (tmp_node)
        {
          if ((tmp_node->kind == svn_node_file)
              || (tmp_node->text_mod)
              || (tmp_node->action == 'A')
              || (tmp_node->action == 'D'))
            {
              print_me = 1;
            }
          while (tmp_node->sibling && (! print_me ))
            {
              tmp_node = tmp_node->sibling;
              if ((tmp_node->kind == svn_node_file)
                  || (tmp_node->text_mod)
                  || (tmp_node->action == 'A')
                  || (tmp_node->action == 'D'))
                {
                  print_me = 1;
                }
            }
        }
    }
  
  /* Print the node if it qualifies. */
  if (print_me)
    {
      printf ("%s/\n", path->data);
    }

  /* Recursively handle the node's children. */
  tmp_node = node->child;
  if (! tmp_node)
    return;

  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  print_dirs_changed_tree (tmp_node, full_path, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      print_dirs_changed_tree (tmp_node, full_path, pool);
    }

  return;
}


/* Recursively print all nodes in the tree that have been modified
   (do not include directories affected only by "bubble-up"). */
static void
print_changed_tree (svn_repos_node_t *node,
                    svn_stringbuf_t *path,
                    apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  svn_stringbuf_t *full_path;
  char status[3] = "_ ";
  int print_me = 1;

  if (! node)
    return;

  /* Print the node. */
  tmp_node = node;
  if (tmp_node->action == 'A')
    status[0] = 'A';
  else if (tmp_node->action == 'D')
    status[0] = 'D';
  else if (tmp_node->action == 'R')
    {
      if ((! tmp_node->text_mod) && (! tmp_node->prop_mod))
        print_me = 0;
      if (tmp_node->text_mod)
        status[0] = 'U';
      if (tmp_node->prop_mod)
        status[1] = 'U';
    }
  else
    print_me = 0;

  /* Print this node unless told to skip it. */
  if (print_me)
    printf ("%s  %s%s\n",
            status,
            path->data,
            tmp_node->kind == svn_node_dir ? "/" : "");
  
  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return;

  /* Recursively handle the node's children. */
  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  print_changed_tree (tmp_node, full_path, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      print_changed_tree (tmp_node, full_path, pool);
    }

  return;
}


static svn_error_t *
open_writable_binary_file (apr_file_t **fh, 
                           svn_stringbuf_t *path, 
                           apr_pool_t *pool)
{
  apr_array_header_t *path_pieces;
  apr_status_t apr_err;
  int i;
  svn_stringbuf_t *full_path, *dir, *basename;
  
  /* Try the easy way to open the file. */
  apr_err = apr_file_open (fh, path->data, 
                           APR_WRITE | APR_CREATE | APR_TRUNCATE | APR_BINARY,
                           APR_OS_DEFAULT, pool);
  if (! apr_err)
    return SVN_NO_ERROR;

  svn_path_split (path, &dir, &basename, svn_path_local_style, pool);

  /* If the file path has no parent, then we've already tried to open
     it as best as we care to try above. */
  if (svn_path_is_empty (dir, svn_path_local_style))
    return svn_error_createf (apr_err, 0, NULL, pool,
                              "Error opening writable file %s", path->data);

  path_pieces = svn_path_decompose (dir,
                                    svn_path_local_style,
                                    pool);
  if (! path_pieces->nelts)
    return APR_SUCCESS;

  full_path = svn_stringbuf_create ("", pool);
  for (i = 0; i < path_pieces->nelts; i++)
    {
      enum svn_node_kind kind;
      svn_stringbuf_t *piece = ((svn_stringbuf_t **) (path_pieces->elts))[i];
      svn_path_add_component (full_path, piece, svn_path_local_style);
      SVN_ERR (svn_io_check_path (full_path, &kind, pool));

      /* Does this path component exist at all? */
      if (kind == svn_node_none)
        {
          apr_err = apr_dir_make (full_path->data, APR_OS_DEFAULT, pool);
          if (apr_err)
            return svn_error_createf (apr_err, 0, NULL, pool,
                                      "Error creating dir %s", 
                                      full_path->data);
        }
      else if (kind != svn_node_dir)
        {
          if (apr_err)
            return svn_error_createf (apr_err, 0, NULL, pool,
                                      "Error creating dir %s (path exists)", 
                                      full_path->data);
        }
    }

  /* Now that we are ensured that the parent path for this file
     exists, try once more to open it. */
  apr_err = apr_file_open (fh, path->data, 
                           APR_WRITE | APR_CREATE | APR_TRUNCATE | APR_BINARY,
                           APR_OS_DEFAULT, pool);
  if (apr_err)
    return svn_error_createf (apr_err, 0, NULL, pool,
                              "Error opening writable file %s", path->data);
    
  return SVN_NO_ERROR;
}


static svn_error_t *
dump_contents (apr_file_t *fh,
               svn_fs_root_t *root,
               svn_stringbuf_t *path,
               apr_pool_t *pool)
{
  apr_status_t apr_err;
  apr_size_t len, len2;
  svn_stream_t *stream;
  unsigned char buffer[1024];

  /* Get a stream to the current file's contents. */
  SVN_ERR (svn_fs_file_contents (&stream, root, path->data, pool));
  
  /* Now, route that data into our temporary file. */
  while (1)
    {
      len = sizeof (buffer);
      SVN_ERR (svn_stream_read (stream, buffer, &len));
      len2 = len;
      apr_err = apr_file_write (fh, buffer, &len2);
      if ((apr_err) || (len2 != len))
        return svn_error_createf 
          (apr_err ? apr_err : SVN_ERR_INCOMPLETE_DATA, 0, NULL, pool,
               "Error writing contents of %s", path->data);
      if (len != sizeof (buffer))
        break;
    }

  /* And close the file. */
  apr_file_close (fh);
  return SVN_NO_ERROR;
}


/* Recursively print all nodes in the tree that have been modified
   (do not include directories affected only by "bubble-up"). */
static svn_error_t *
print_diff_tree (svn_fs_root_t *root,
                 svn_fs_root_t *base_root,
                 svn_repos_node_t *node, 
                 svn_stringbuf_t *path,
                 apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  svn_stringbuf_t *full_path;
  svn_stringbuf_t *orig_path = NULL, *new_path = NULL;
  apr_file_t *fh1, *fh2;
      
  if (! node)
    return SVN_NO_ERROR;

  /* Print the node. */
  tmp_node = node;

  /* ### todo:  Need a plan for printing diffs here.  */
  /* First, we'll just print file content diffs. */
  if (tmp_node->kind == svn_node_file)
    {
      if ((tmp_node->action == 'R') && (tmp_node->text_mod))
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          SVN_ERR (dump_contents (fh2, base_root, path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          SVN_ERR (dump_contents (fh1, root, path, pool));
          apr_file_close (fh1);
        }
      if (tmp_node->action == 'A')
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          SVN_ERR (dump_contents (fh1, root, path, pool));
          apr_file_close (fh1);
        }
      if (tmp_node->action == 'D')
        {
          orig_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_BASE, pool);
          svn_path_add_component (orig_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh2, orig_path, pool));
          SVN_ERR (dump_contents (fh2, base_root, path, pool));
          apr_file_close (fh2);

          new_path = svn_stringbuf_create (SVNLOOK_DIFF_TMPDIR_NEW, pool);
          svn_path_add_component (new_path, path, svn_path_local_style);
          SVN_ERR (open_writable_binary_file (&fh1, new_path, pool));
          apr_file_close (fh1);
        }
    }

  if (orig_path && new_path)
    {
      const char *args[5];
      apr_file_t *outhandle;
      apr_status_t apr_err;

      printf ("%s: %s\n", 
              ((tmp_node->action == 'A') ? "Added" : 
               ((tmp_node->action == 'D') ? "Deleted" :
                ((tmp_node->action == 'R') ? "Modified" : "Index"))),
              path->data);
      printf ("===============================================================\
===============\n");
      fflush (stdout);

      args[0] = SVN_CLIENT_DIFF;
      args[1] = "-c";
      args[2] = orig_path->data;
      args[3] = new_path->data;
      args[4] = NULL;

      /* Get an apr_file_t representing stdout, which is where we'll have
         the diff program print to. */
      apr_err = apr_file_open_stdout (&outhandle, pool);
      if (apr_err)
        return svn_error_create 
          (apr_err, 0, NULL, pool,
           "print_diff_tree: can't open handle to stdout");

      SVN_ERR (svn_io_run_cmd (".", SVN_CLIENT_DIFF, args, 
                               NULL, outhandle, NULL, pool));
      printf ("\n");
      fflush (stdout);
    }
  
  /* Now, delete any temporary files. */
  if (orig_path)
    apr_file_remove (orig_path->data, pool);
  if (new_path)
    apr_file_remove (new_path->data, pool);

  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return SVN_NO_ERROR;

  /* Recursively handle the node's children. */
  full_path = svn_stringbuf_dup (path, pool);
  svn_path_add_component_nts (full_path, tmp_node->name, svn_path_repos_style);
  SVN_ERR (print_diff_tree (root, base_root, tmp_node, full_path, pool));
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      svn_stringbuf_set (full_path, path->data);
      svn_path_add_component_nts 
        (full_path, tmp_node->name, svn_path_repos_style);
      SVN_ERR (print_diff_tree (root, base_root, tmp_node, full_path, pool));
    }

  return SVN_NO_ERROR;
}


/* Recursively print all nodes in the tree.  If SHOW_IDS is non-zero,
   print the id of each node next to its name. */
static void
print_tree (svn_repos_node_t *node,
            svn_boolean_t show_ids,
            int indentation,
            apr_pool_t *pool)
{
  svn_repos_node_t *tmp_node;
  int i;

  if (! node)
    return;

  /* Print the indentation. */
  for (i = 0; i < indentation; i++)
    {
      printf (" ");
    }

  /* Print the node. */
  tmp_node = node;
  if (show_ids)
    {
      svn_stringbuf_t *unparsed_id;

      if (tmp_node->id)
        unparsed_id = svn_fs_unparse_id (tmp_node->id, pool);

      printf ("%s%s <%s>\n", 
              tmp_node->name, 
              tmp_node->kind == svn_node_dir ? "/" : "",
              tmp_node->id ? unparsed_id->data : "unknown");
    }
  else
    {
      printf ("%s%s \n", 
              tmp_node->name, 
              tmp_node->kind == svn_node_dir ? "/" : "");
    }

  /* Return here if the node has no children. */
  tmp_node = tmp_node->child;
  if (! tmp_node)
    return;

  /* Recursively handle the node's children. */
  print_tree (tmp_node, show_ids, indentation + 1, pool);
  while (tmp_node->sibling)
    {
      tmp_node = tmp_node->sibling;
      print_tree (tmp_node, show_ids, indentation + 1, pool);
    }

  return;
}





/*** Subcommand handlers. ***/

/* Print the revision's log message to stdout, followed by a newline. */
static svn_error_t *
do_log (svnlook_ctxt_t *c, svn_boolean_t print_size, apr_pool_t *pool)
{
  svn_stringbuf_t *prop_value;
  svn_string_t prop_name;
  const char *name = SVN_PROP_REVISION_LOG;

  prop_name.data = name;
  prop_name.len = strlen (name);

  SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

  if (prop_value && prop_value->data)
    {
      if (print_size)
        {
          printf ("%lu\n", (unsigned long)prop_value->len);
        }

      printf ("%s", prop_value->data);
    }
  else if (print_size)
    {
      printf ("0");
    }
  
  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print the timestamp of the commit (in the revision case) or the
   empty string (in the transaction case) to stdout, followed by a
   newline. */
static svn_error_t *
do_date (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  if (c->is_revision)
    {
      svn_stringbuf_t *prop_value;
      svn_string_t prop_name;
      const char *name = SVN_PROP_REVISION_DATE;
      
      prop_name.data = name;
      prop_name.len = strlen (name);
      
      SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

      if (prop_value && prop_value->data)
        {
          /* The date stored in the repository is in a really complex
             and precise format...and we don't care.  Let's convert
             that to just "YYYY-MM-DD hh:mm".

             ### todo: Right now, "svn dates" are not GMT, but the
             results of svn_time_from_string are.  This sucks. */
          apr_exploded_time_t extime;
          apr_time_t aprtime;
          apr_status_t apr_err;
              
          aprtime = svn_time_from_string (prop_value);
          apr_err = apr_explode_time (&extime, aprtime, 0);
          if (apr_err)
            return svn_error_create (apr_err, 0, NULL, pool,
                                     "do_date: error exploding time");
              
          printf ("%04lu-%02lu-%02lu %02lu:%02lu GMT",
                  (unsigned long)(extime.tm_year + 1900),
                  (unsigned long)(extime.tm_mon + 1),
                  (unsigned long)(extime.tm_mday),
                  (unsigned long)(extime.tm_hour),
                  (unsigned long)(extime.tm_min));
        }
    }

  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print the author of the commit to stdout, followed by a newline. */
static svn_error_t *
do_author (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_stringbuf_t *prop_value;
  svn_string_t prop_name;
  const char *name = SVN_PROP_REVISION_AUTHOR;

  prop_name.data = name;
  prop_name.len = strlen (name);

  SVN_ERR (get_property (&prop_value, c, &prop_name, pool));

  if (prop_value && prop_value->data)
    printf ("%s", prop_value->data);
  
  printf ("\n");
  return SVN_NO_ERROR;
}


/* Print a list of all directories in which files, or directory
   properties, have been modified. */
static svn_error_t *
do_dirs_changed (svnlook_ctxt_t *c, apr_pool_t *pool)
{ 
  svn_fs_root_t *root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    print_dirs_changed_tree (tree, svn_stringbuf_create ("", pool), pool);

  return SVN_NO_ERROR;
}


/* Print a list of all paths modified in a format compatible with `svn
   update'. */
static svn_error_t *
do_changed (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_fs_root_t *root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    print_changed_tree (tree, svn_stringbuf_create ("", pool), pool);

  return SVN_NO_ERROR;
}


/* Print some diff-y stuff in a TBD way. :-) */
static svn_error_t *
do_diff (svnlook_ctxt_t *c, apr_pool_t *pool)
{
  svn_fs_root_t *root, *base_root;
  svn_revnum_t base_rev_id;
  svn_repos_node_t *tree;

  SVN_ERR (get_root (&root, c, pool));
  if (c->is_revision)
    base_rev_id = c->rev_id - 1;
  else
    base_rev_id = svn_fs_txn_base_revision (c->txn);

  if (! SVN_IS_VALID_REVNUM (base_rev_id))
    return svn_error_createf 
      (SVN_ERR_FS_NO_SUCH_REVISION, 0, NULL, pool,
       "Transaction '%s' is not based on a revision.  How odd.",
       c->txn_name);
  
  SVN_ERR (generate_delta_tree (&tree, c->fs, root, base_rev_id, pool)); 
  if (tree)
    {
      SVN_ERR (svn_fs_revision_root (&base_root, c->fs, base_rev_id, pool));
      SVN_ERR (print_diff_tree 
               (root, base_root, tree, svn_stringbuf_create ("", pool), pool));
      apr_dir_remove_recursively (SVNLOOK_DIFF_TMPDIR_BASE, pool);
      apr_dir_remove_recursively (SVNLOOK_DIFF_TMPDIR_NEW, pool);
    }
{'fraction_non_alphanumeric': 0.12088704391522831, 'fraction_numerical': 0.004020100502512563, 'mean_word_length': 2.4035544318857824, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 239, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6581044', 'n_tokens_mistral': 16872, 'n_tokens_neox': 16421, 'n_words': 3757}