<commit_before>/********************************************************************
* Description: interp_o_word.cc
*
*
* Author: <NAME>
* License: GPL Version 2
* System: Linux
*    
* Copyright 2005 All rights reserved.
*
* Last change:
********************************************************************/

#include <boost/python.hpp>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include "rs274ngc.hh"
#include "rs274ngc_return.hh"
#include "interp_return.hh"
#include "interp_internal.hh"
#include "rs274ngc_interp.hh"

namespace bp = boost::python;

//========================================================================
// Functions for control stuff (O-words)
//========================================================================

/*
  Given the root of a directory tree and a file name,
  find the path to the file, if any.
*/

int Interp::findFile( // ARGUMENTS
		     char *direct,  // the directory to start looking in
		     char *target,  // the name of the file to find
		     char *foundFileDirect) // where to store the result
{
    FILE *file;
    DIR *aDir;
    struct dirent *aFile;
    char targetPath[PATH_MAX+1];

    snprintf(targetPath, PATH_MAX, "%s/%s", direct, target);

    file = fopen(targetPath, "r");

    if(file)
    {
        strncpy(foundFileDirect, direct, PATH_MAX);
        fclose(file);
        return INTERP_OK;
    }

    aDir = opendir(direct);

    if(!aDir)
    {
	ERS(NCE_FILE_NOT_OPEN);
    }

    while((aFile = readdir(aDir)))
    {
        if(aFile->d_type == DT_DIR &&
           (0 != strcmp(aFile->d_name, "..")) &&
           (0 != strcmp(aFile->d_name, ".")))
        {
            char path[PATH_MAX+1];
            snprintf(path, PATH_MAX, "%s/%s", direct, aFile->d_name);
            if(INTERP_OK == findFile(path, target, foundFileDirect))
            {
	        closedir(aDir);
                return INTERP_OK;
            }
        }
    }
    closedir(aDir);
    ERS(NCE_FILE_NOT_OPEN);
}

/************************************************************************/
/*
   In the long run, this function will use a hash table or other
   fast data structure
*/
int Interp::control_save_offset( /* ARGUMENTS                   */
 int line,                   /* (o-word) line number        */
 block_pointer block,        /* pointer to a block of RS274/NGC instructions */
 setup_pointer settings)     /* pointer to machine settings */
{
  static char name[] = "control_save_offset";
  int index;

  logOword("Entered:%s for o_name:|%s|", name, block->o_name);

  if(control_find_oword(block, settings, &index) == INTERP_OK)
  {
      // already exists
      ERS(_("File:%s line:%d redefining sub: o|%s| already defined in file:%s"),
               settings->filename, settings->sequence_number,
	       block->o_name,
               settings->oword_offset[index].filename);
      //return INTERP_OK;
  }

  CHKS((settings->oword_labels >= INTERP_OWORD_LABELS),
      NCE_TOO_MANY_OWORD_LABELS);


  index = settings->oword_labels++;
  //logOword("index: %d offset: %ld", index, block->offset);

  //  settings->oword_offset[index].o_word = line;
  settings->oword_offset[index].o_word_name = block->o_name;
  settings->oword_offset[index].type = block->o_type;
  settings->oword_offset[index].offset = block->offset;
  settings->oword_offset[index].filename = strstore(settings->filename);
  settings->oword_offset[index].repeat_count = -1;

  // the sequence number has already been bumped, so save
  // the proper value
  settings->oword_offset[index].sequence_number =
    settings->sequence_number - 1;

  logOword("control_save_offset: o_word_name=%s type=%d offset=%ld filename=%s repeat_count=%d sequence_number=%d\n",
	   settings->oword_offset[index].o_word_name = block->o_name,
	   settings->oword_offset[index].type,
	   settings->oword_offset[index].offset,
	   settings->oword_offset[index].filename,
	   settings->oword_offset[index].repeat_count,
	   settings->oword_offset[index].sequence_number);

  return INTERP_OK;
}

int Interp::control_find_oword( /* ARGUMENTS                       */
  block_pointer block,      /* pointer to block */
  setup_pointer settings,   /* pointer to machine settings      */
  int *o_index)             /* the index of o-word (returned) */
{
  static char name[] = "control_find_oword";
  int i;

  logOword("Entered:%s", name);
  for(i=0; i<settings->oword_labels; i++)
    {
      if(0 == strcmp(settings->oword_offset[i].o_word_name, block->o_name))
	{
	  *o_index = i;
	  logOword("Found oword[%d]: |%s|", i, block->o_name);
	  return INTERP_OK;
	}
    }
  logOword("Unknown oword name: |%s|", block->o_name);
  ERS(NCE_UNKNOWN_OWORD_NUMBER);
}

//
// TESTME!!! MORE THOROUGHLY !!!KL
//
// In the past, calls had to be to predefined subs
//
// Now they don't. Do things in the following sequence:
// 1 -- if o_word is already defined, just go back to it, else
// 2 -- if there is a file with the name of the o_word,
//             open it and start skipping (as in 3, below)
// 3 -- skip to the o_word (will be an error if not found)
//
int Interp::control_back_to( /* ARGUMENTS                       */
 block_pointer block, // pointer to block
 setup_pointer settings)   /* pointer to machine settings      */
{
  static char name[] = "control_back_to";
  int i,dct;
  char newFileName[PATH_MAX+1];
  char foundPlace[PATH_MAX+1];
  char tmpFileName[PATH_MAX+1];
  FILE *newFP;

  foundPlace[0] = 0;
  logOword("Entered:%s", name);
  for(i=0; i<settings->oword_labels; i++)
    {
      // if(settings->oword_offset[i].o_word == line)
      if(0 == strcmp(settings->oword_offset[i].o_word_name, block->o_name))
	{
	    if ((settings->filename[0] != 0) &
		(settings->file_pointer == NULL))  {
		ERS(NCE_FILE_NOT_OPEN);
	    }
          if(0 != strcmp(settings->filename,
                         settings->oword_offset[i].filename))
          {
              // open the new file...

              newFP = fopen(settings->oword_offset[i].filename, "r");

              // set the line number
              settings->sequence_number = 0;

              strcpy(settings->filename, settings->oword_offset[i].filename);

              if(newFP)
              {
                  // close the old file...
		  if (settings->file_pointer) // only close if it was open
		      fclose(settings->file_pointer);
                  settings->file_pointer = newFP;
              }
              else
              {
                  logOword("Unable to open file: %s", settings->filename);
                  ERS(NCE_UNABLE_TO_OPEN_FILE,settings->filename);
              }
          }
	  if (settings->file_pointer) // only seek if it was open
	      fseek(settings->file_pointer,
		    settings->oword_offset[i].offset, SEEK_SET);

	  settings->sequence_number =
	    settings->oword_offset[i].sequence_number;

	  return INTERP_OK;
	}
    }

  // NO o_word found

  // look for a new file
  logOword("settings->program_prefix:%s:", settings->program_prefix);
  sprintf(tmpFileName, "%s.ngc", block->o_name);

  // find subroutine by search: program_prefix, subroutines, wizard_root
  // use first file found

  // first look in the program_prefix place
  sprintf(newFileName, "%s/%s", settings->program_prefix, tmpFileName);

  newFP = fopen(newFileName, "r");
  logOword("fopen: |%s|", newFileName);

  // then look in the subroutines place
  if(!newFP)
  {
      for (dct=0; dct < MAX_SUB_DIRS; dct++) {
          if (!settings->subroutines[dct][0]) continue;
          sprintf(newFileName, "%s/%s", settings->subroutines[dct], tmpFileName);
          newFP = fopen(newFileName, "r");
          if (newFP) {
              logOword("fopen: |%s|", newFileName);
              break; // use first occurrence in dir search
          }
      }
  }


  // if not found, search the wizard tree
  if(!newFP)
  {
      int ret;
      ret = findFile(settings->wizard_root, tmpFileName, foundPlace);

      if(INTERP_OK == ret)
      {
	  // create the long name
          sprintf(newFileName, "%s/%s",
		  foundPlace, tmpFileName);
          newFP = fopen(newFileName, "r");
      }
  }

  if(newFP)
  {
      logOword("fopen: |%s| OK", newFileName);

      // close the old file...
      if (settings->file_pointer)
	  fclose(settings->file_pointer);
      settings->file_pointer = newFP;

      strcpy(settings->filename, newFileName);
  }
  else
  {
     char *dirname = get_current_dir_name();
     logOword("fopen: |%s| failed CWD:|%s|", newFileName,
              dirname);
     free(dirname);
     ERS(NCE_UNABLE_TO_OPEN_FILE,tmpFileName);
  }

  settings->skipping_o = block->o_name; // start skipping
  settings->skipping_to_sub = block->o_name; // start skipping
  settings->skipping_start = settings->sequence_number;
  //ERS(NCE_UNKNOWN_OWORD_NUMBER);

  return INTERP_OK;
}

/************************************************************************/
/* convert_control_functions

Returned Value: int (INTERP_OK)
Side effects:
   Changes the flow of control.

Called by: execute_block

Calls: control_skip_to
       control_back_to
       control_save_offset
*/

int Interp::convert_control_functions( /* ARGUMENTS           */
 block_pointer block,      /* pointer to a block of RS274/NGC instructions */
 setup_pointer settings)   /* pointer to machine settings                  */
{
  int status = INTERP_OK;
  int index;
  int i;
  context_pointer current_frame, new_frame, leaving_frame,  returnto_frame;
  block_pointer r_block;
  bool is_remap_handler = false; // the sub is executing on behalf of a remapped code
  bool is_py_callable = false;   // the sub name is actually Python
  bool py_exception = false;

  logOword("convert_control_functions");

  // if there is an oword, must get the block->o_number
  // !!!KL
  if(block->o_name)
    {
      control_find_oword(block, settings, &(block->o_number));
    }
  else
    {
      block->o_number = 0;
    }

  // must skip if skipping
  if(settings->skipping_o && (0 != strcmp(settings->skipping_o, block->o_name)))
  {
      logOword("skipping to line: |%s|", settings->skipping_o);
      return INTERP_OK;
  }

  if(settings->skipping_to_sub && (block->o_type != O_sub))
  {
      logOword("skipping to sub: |%s|", settings->skipping_to_sub);
      return INTERP_OK;
  }

  logOword("o_type:%d", block->o_type);
  switch(block->o_type)
    {
    case O_none:
      // not an error because we use this to signal that we
      // are not evaluating functions
      break;

    case O_sub:
      // if the level is not zero, this is a call
      // not the definition
      // if we were skipping, no longer
      if(settings->skipping_o)
      {
          logOword("sub(o_|%s|) was skipping to here", settings->skipping_o);

          // skipping to a sub means that we must define this now
	  CHP(control_save_offset(block->o_number, block, settings));
      }

      if(settings->skipping_o)
	{
	  logOword("no longer skipping to:|%s|", settings->skipping_o);
          settings->skipping_o = NULL; // this IS our block number
	}
      settings->skipping_to_sub = NULL; // this IS our block number

      if(settings->call_level != 0)
	{
	  logOword("call:%f:%f:%f",
		   settings->parameters[1],
		   settings->parameters[2],
		   settings->parameters[3]);
	}
      else
	{
	  logOword("started a subroutine defn");
	  // a definition
	  CHKS((settings->defining_sub == 1), NCE_NESTED_SUBROUTINE_DEFN);
	  CHP(control_save_offset(block->o_number, block, settings));

          settings->skipping_o = block->o_name; // start skipping
          settings->skipping_start = settings->sequence_number;
	  settings->defining_sub = 1;
	  settings->sub_name = block->o_name;
	  logOword("will now skip to: |%s| %d", settings->sub_name,
		   block->o_number);
	}
      break;

    case O_endsub:
    case O_return:

	// this case is executed only for bona-fide NGC subs
	// subs whose name is a Py callable are handled inline during O_call

      // if level is not zero, in a call
      // otherwise in a defn
      // if we were skipping, no longer
      if (settings->skipping_o) 	{
	  logOword("case O_%s -- no longer skipping to:|%s|",
		   (block->o_type == O_endsub) ? "endsub" : "return",
		   settings->skipping_o);
          settings->skipping_o = NULL;
      }
      if (settings->call_level != 0) {
	  //  some shorthands to make this readable
	  leaving_frame = &settings->sub_context[settings->call_level];
	  returnto_frame = &settings->sub_context[settings->call_level - 1];

	  // free local variables
	  free_named_parameters(leaving_frame);
	  leaving_frame->subName = NULL;

	  // drop one call level.
	  settings->call_level--;

	  // restore subroutine parameters.
	  for(i = 0; i < INTERP_SUB_PARAMS; i++) {
	      settings->parameters[i+INTERP_FIRST_SUBROUTINE_PARAM] =
		  returnto_frame->saved_params[i];
	  }

	  // file at this level was marked as closed, so dont reopen.
	  // if (settings->sub_context[settings->call_level].position == -1) {
	  if (returnto_frame->position == -1) {
	      settings->file_pointer = NULL;
	      strcpy(settings->filename, "");
	  } else {
	      logOword("seeking to: %ld", returnto_frame->position);
	      if(settings->file_pointer == NULL) {
		  ERS(NCE_FILE_NOT_OPEN);
	      }
	      //!!!KL must open the new file, if changed
	      if (0 != strcmp(settings->filename, returnto_frame->filename))  {
		  fclose(settings->file_pointer);
		  settings->file_pointer = fopen(returnto_frame->filename, "r");
		  strcpy(settings->filename, returnto_frame->filename);
	      }
	      fseek(settings->file_pointer, returnto_frame->position, SEEK_SET);
	      settings->sequence_number = returnto_frame->sequence_number;

	      // cleanups on return
	      // if we have an epilog function - execute it, passing the
	      // current call frame


	      // aquire the remapped block
	      r_block = &CONTROLLING_BLOCK(_setup);

	      if (HAS_PYTHON_EPILOG(r_block->executing_remap)) {
		  logRemap("O_call: py epilog %s for NGC remap %s",
			   r_block->executing_remap->epilog_func,
			   r_block->executing_remap->remap_ngc);
		  status = pycall(settings,r_block,
				  r_block->executing_remap->epilog_func);
	      }

	      if (HAS_BUILTIN_EPILOG(r_block->executing_remap)) {
		  logRemap("O_call: calling builtin epilogue\n");
		  status = (*this.*r_block->executing_remap->builtin_epilog)(settings, r_block);

		  if (status > INTERP_MIN_ERROR) {
		      logRemap("O_call: builtin epilogue failed: %s \n", interp_status(status));
		  }
	      }

	      // a valid previous context was marked by an M73 as auto-restore
	      if ((leaving_frame->context_status &
		   (CONTEXT_RESTORE_ON_RETURN|CONTEXT_VALID)) ==
		  (CONTEXT_RESTORE_ON_RETURN|CONTEXT_VALID)) {

		  // NB: this means an M71 invalidate context will prevent an
		  // auto-restore on return/endsub
		  restore_context(settings, settings->call_level + 1);
	      }
	      // always invalidate on leaving a context so we dont accidentially
	      // 'run into' a valid context when growing the stack upwards again
	      leaving_frame->context_status &=  ~CONTEXT_VALID;

	  }
	  settings->sub_name = 0;
	  if (returnto_frame->subName)  {
	      settings->sub_name = returnto_frame->subName;
	  } else {
	      settings->sub_name = NULL;
	  }
	}
      else
       {
	 // a definition
	 if (block->o_type == O_endsub) {
	     CHKS((settings->defining_sub != 1), NCE_NOT_IN_SUBROUTINE_DEFN);
	     // no longer skipping or defining
	     if (settings->skipping_o)
		 {
		     logOword("case O_endsub in defn -- no longer skipping to:|%s|",
			      settings->skipping_o);
		     settings->skipping_o = NULL;
		 }
	     settings->defining_sub = NULL;
	     settings->sub_name = NULL;
	 }
       }
      break;
    case O_call:
	current_frame = &settings->sub_context[settings->call_level];
	new_frame = &settings->sub_context[settings->call_level + 1];
	is_py_callable = is_pycallable(settings, block->o_name);

	// aquire the 'remap_frame' a.k.a controlling block
	r_block = &CONTROLLING_BLOCK(*settings);

	// determine if this sub is the remap body executing on behalf of a remapped code
	// we're loosing a bit of context by funneling everything through the osub call
	// interface, which needs to be reestablished here but it's more general
	is_remap_handler =
	    (r_block->executing_remap != NULL) &&
	    (((r_block->executing_remap->remap_py != NULL) &&
	     (!strcmp(r_block->executing_remap->remap_py, block->o_name))) ||
	     (((r_block->executing_remap->remap_ngc != NULL) &&
	       (!strcmp(r_block->executing_remap->remap_ngc, block->o_name)))));

	if (!is_py_callable) {
	    // standard oword call incantations:
	    // copy parameters from context
	    // save old values of parameters
	    // save current file position in context
	    // if we were skipping, no longer
	    if (settings->skipping_o) {
		logOword("case O_call -- no longer skipping to:|%s|",
			 settings->skipping_o);
		settings->skipping_o = NULL;
	    }
	    if (settings->call_level >= INTERP_SUB_ROUTINE_LEVELS) {
		ERS(NCE_TOO_MANY_SUBROUTINE_LEVELS);
	    }
	    for(i = 0; i < INTERP_SUB_PARAMS; i++)	{
		current_frame->saved_params[i] =
		    settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM];

		settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM] =
		    block->params[i];
	    }

	    // if the previous file was NULL, mark positon as -1 so as not to
	    // reopen it on return.
	    if (settings->file_pointer == NULL) {
		current_frame->position = -1;
	    } else {
		current_frame->position = ftell(settings->file_pointer);
	    }

	    // save the previous filename
	    logOword("Duping |%s|", settings->filename);
	    current_frame->filename = strstore(settings->filename);
	    current_frame->sequence_number = settings->sequence_number;
	    logOword("(in call)set params[%d] return file:%s offset:%ld",
		     settings->call_level, current_frame->filename, current_frame->position);

	} // done with NGC call incantation

	settings->call_level++;

	// set the new subName
	new_frame->subName = block->o_name;
#if 0
	// just curious: detect recursion
	if ((settings->call_level > 0) &&
	    (new_frame->subName != NULL) &&
	    (current_frame->subName  != NULL)) {
	    if (!strcmp(new_frame->subName, current_frame->subName)) {
		logOword("recursive call: '%s' call_level %d\n",
			 new_frame->subName, settings->call_level);
	    }
	}
#endif
	if (is_remap_handler && HAS_PYTHON_PROLOG(r_block->executing_remap)) {
	    logRemap("O_call: py prologue %s for NGC remap %s ",
		     r_block->executing_remap->prolog_func,
		     r_block->executing_remap->remap_ngc);
	    status = pycall(settings, r_block,
			    r_block->executing_remap->prolog_func);
	}
	if (!is_py_callable) { // handle as NGC
	    if (control_back_to(block,settings) == INTERP_ERROR) {
		settings->call_level--;
		ERS(NCE_UNABLE_TO_OPEN_FILE,block->o_name);
		return INTERP_ERROR;
	    }
	} else {
	    // the o_name is Python callable. Depending wether it's a
	    // remap handler, or a Osub replacement, arguments are
	    // passed a bit differently.

	    if (is_remap_handler) {
		// execute_handler has everything set up already (name, tupleargs, kwargs)
		CHKS(pycall(settings, r_block, block->o_name),
		     "O_call: %s: remap pycall(%s) failed, unwinding",
		     r_block->executing_remap->name,
		     block->o_name);
		// now the builtin epilog if so set
		if (HAS_BUILTIN_EPILOG(r_block->executing_remap)) {
		    CHKS((*this.*r_block->executing_remap->builtin_epilog)(settings,r_block),
			 "O_call: %s: builtin  epilogue for remap pycall(%s) failed: %s",
			 r_block->executing_remap->name,
			 block->o_name,
			 interp_status(status));
		}
	    } else {
		try {
		    // call with list of positional parameters
		    bp::list plist;

		    for(int i = 0; i < INTERP_SUB_PARAMS; i++)
			plist.append(block->params[i]);
		    block->tupleargs = bp::make_tuple(plist);
		    block->kwargs = bp::dict();
		}
		catch (bp::error_already_set) {
		    if (PyErr_Occurred()) {
			PyErr_Print();
		    }
		    bp::handle_exception();
		    PyErr_Clear();
		    py_exception = true;
		}
		CHKS(py_exception,"O_call: Py exception preparing arguments for %s",
		     block->o_name);

		CHKS((status = pycall(settings, block, block->o_name)),
		     "O_call: osub pycall(%s) failed, unwinding",block->o_name);

		// no epilogues for vanilla subs and their py replacements
	    }
	    settings->call_level--;

	}
	return INTERP_OK;
	break;<commit_msg>stepping back  - osubs working<commit_after>/********************************************************************
* Description: interp_o_word.cc
*
*
* Author: <NAME>
* License: GPL Version 2
* System: Linux
*    
* Copyright 2005 All rights reserved.
*
* Last change:
********************************************************************/

#include <boost/python.hpp>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include "rs274ngc.hh"
#include "rs274ngc_return.hh"
#include "interp_return.hh"
#include "interp_internal.hh"
#include "rs274ngc_interp.hh"

namespace bp = boost::python;

//========================================================================
// Functions for control stuff (O-words)
//========================================================================

/*
  Given the root of a directory tree and a file name,
  find the path to the file, if any.
*/

int Interp::findFile( // ARGUMENTS
		     char *direct,  // the directory to start looking in
		     char *target,  // the name of the file to find
		     char *foundFileDirect) // where to store the result
{
    FILE *file;
    DIR *aDir;
    struct dirent *aFile;
    char targetPath[PATH_MAX+1];

    snprintf(targetPath, PATH_MAX, "%s/%s", direct, target);

    file = fopen(targetPath, "r");

    if(file)
    {
        strncpy(foundFileDirect, direct, PATH_MAX);
        fclose(file);
        return INTERP_OK;
    }

    aDir = opendir(direct);

    if(!aDir)
    {
	ERS(NCE_FILE_NOT_OPEN);
    }

    while((aFile = readdir(aDir)))
    {
        if(aFile->d_type == DT_DIR &&
           (0 != strcmp(aFile->d_name, "..")) &&
           (0 != strcmp(aFile->d_name, ".")))
        {
            char path[PATH_MAX+1];
            snprintf(path, PATH_MAX, "%s/%s", direct, aFile->d_name);
            if(INTERP_OK == findFile(path, target, foundFileDirect))
            {
	        closedir(aDir);
                return INTERP_OK;
            }
        }
    }
    closedir(aDir);
    ERS(NCE_FILE_NOT_OPEN);
}

/************************************************************************/
/*
   In the long run, this function will use a hash table or other
   fast data structure
*/
int Interp::control_save_offset( /* ARGUMENTS                   */
 int line,                   /* (o-word) line number        */
 block_pointer block,        /* pointer to a block of RS274/NGC instructions */
 setup_pointer settings)     /* pointer to machine settings */
{
  static char name[] = "control_save_offset";
  int index;

  logOword("Entered:%s for o_name:|%s|", name, block->o_name);

  if(control_find_oword(block, settings, &index) == INTERP_OK)
  {
      // already exists
      ERS(_("File:%s line:%d redefining sub: o|%s| already defined in file:%s"),
               settings->filename, settings->sequence_number,
	       block->o_name,
               settings->oword_offset[index].filename);
      //return INTERP_OK;
  }

  CHKS((settings->oword_labels >= INTERP_OWORD_LABELS),
      NCE_TOO_MANY_OWORD_LABELS);


  index = settings->oword_labels++;
  //logOword("index: %d offset: %ld", index, block->offset);

  //  settings->oword_offset[index].o_word = line;
  settings->oword_offset[index].o_word_name = block->o_name;
  settings->oword_offset[index].type = block->o_type;
  settings->oword_offset[index].offset = block->offset;
  settings->oword_offset[index].filename = strstore(settings->filename);
  settings->oword_offset[index].repeat_count = -1;

  // the sequence number has already been bumped, so save
  // the proper value
  settings->oword_offset[index].sequence_number =
    settings->sequence_number - 1;

  logOword("control_save_offset: o_word_name=%s type=%d offset=%ld filename=%s repeat_count=%d sequence_number=%d\n",
	   settings->oword_offset[index].o_word_name = block->o_name,
	   settings->oword_offset[index].type,
	   settings->oword_offset[index].offset,
	   settings->oword_offset[index].filename,
	   settings->oword_offset[index].repeat_count,
	   settings->oword_offset[index].sequence_number);

  return INTERP_OK;
}

int Interp::control_find_oword( /* ARGUMENTS                       */
  block_pointer block,      /* pointer to block */
  setup_pointer settings,   /* pointer to machine settings      */
  int *o_index)             /* the index of o-word (returned) */
{
  static char name[] = "control_find_oword";
  int i;

  logOword("Entered:%s", name);
  for(i=0; i<settings->oword_labels; i++)
    {
      if(0 == strcmp(settings->oword_offset[i].o_word_name, block->o_name))
	{
	  *o_index = i;
	  logOword("Found oword[%d]: |%s|", i, block->o_name);
	  return INTERP_OK;
	}
    }
  logOword("Unknown oword name: |%s|", block->o_name);
  ERS(NCE_UNKNOWN_OWORD_NUMBER);
}

//
// TESTME!!! MORE THOROUGHLY !!!KL
//
// In the past, calls had to be to predefined subs
//
// Now they don't. Do things in the following sequence:
// 1 -- if o_word is already defined, just go back to it, else
// 2 -- if there is a file with the name of the o_word,
//             open it and start skipping (as in 3, below)
// 3 -- skip to the o_word (will be an error if not found)
//
int Interp::control_back_to( /* ARGUMENTS                       */
 block_pointer block, // pointer to block
 setup_pointer settings)   /* pointer to machine settings      */
{
  static char name[] = "control_back_to";
  int i,dct;
  char newFileName[PATH_MAX+1];
  char foundPlace[PATH_MAX+1];
  char tmpFileName[PATH_MAX+1];
  FILE *newFP;

  foundPlace[0] = 0;
  logOword("Entered:%s", name);
  for(i=0; i<settings->oword_labels; i++)
    {
      // if(settings->oword_offset[i].o_word == line)
      if(0 == strcmp(settings->oword_offset[i].o_word_name, block->o_name))
	{
	    if ((settings->filename[0] != 0) &
		(settings->file_pointer == NULL))  {
		ERS(NCE_FILE_NOT_OPEN);
	    }
          if(0 != strcmp(settings->filename,
                         settings->oword_offset[i].filename))
          {
              // open the new file...

              newFP = fopen(settings->oword_offset[i].filename, "r");

              // set the line number
              settings->sequence_number = 0;

              strcpy(settings->filename, settings->oword_offset[i].filename);

              if(newFP)
              {
                  // close the old file...
		  if (settings->file_pointer) // only close if it was open
		      fclose(settings->file_pointer);
                  settings->file_pointer = newFP;
              }
              else
              {
                  logOword("Unable to open file: %s", settings->filename);
                  ERS(NCE_UNABLE_TO_OPEN_FILE,settings->filename);
              }
          }
	  if (settings->file_pointer) // only seek if it was open
	      fseek(settings->file_pointer,
		    settings->oword_offset[i].offset, SEEK_SET);

	  settings->sequence_number =
	    settings->oword_offset[i].sequence_number;

	  return INTERP_OK;
	}
    }

  // NO o_word found

  // look for a new file
  logOword("settings->program_prefix:%s:", settings->program_prefix);
  sprintf(tmpFileName, "%s.ngc", block->o_name);

  // find subroutine by search: program_prefix, subroutines, wizard_root
  // use first file found

  // first look in the program_prefix place
  sprintf(newFileName, "%s/%s", settings->program_prefix, tmpFileName);

  newFP = fopen(newFileName, "r");
  logOword("fopen: |%s|", newFileName);

  // then look in the subroutines place
  if(!newFP)
  {
      for (dct=0; dct < MAX_SUB_DIRS; dct++) {
          if (!settings->subroutines[dct][0]) continue;
          sprintf(newFileName, "%s/%s", settings->subroutines[dct], tmpFileName);
          newFP = fopen(newFileName, "r");
          if (newFP) {
              logOword("fopen: |%s|", newFileName);
              break; // use first occurrence in dir search
          }
      }
  }


  // if not found, search the wizard tree
  if(!newFP)
  {
      int ret;
      ret = findFile(settings->wizard_root, tmpFileName, foundPlace);

      if(INTERP_OK == ret)
      {
	  // create the long name
          sprintf(newFileName, "%s/%s",
		  foundPlace, tmpFileName);
          newFP = fopen(newFileName, "r");
      }
  }

  if(newFP)
  {
      logOword("fopen: |%s| OK", newFileName);

      // close the old file...
      if (settings->file_pointer)
	  fclose(settings->file_pointer);
      settings->file_pointer = newFP;

      strcpy(settings->filename, newFileName);
  }
  else
  {
     char *dirname = get_current_dir_name();
     logOword("fopen: |%s| failed CWD:|%s|", newFileName,
              dirname);
     free(dirname);
     ERS(NCE_UNABLE_TO_OPEN_FILE,tmpFileName);
  }

  settings->skipping_o = block->o_name; // start skipping
  settings->skipping_to_sub = block->o_name; // start skipping
  settings->skipping_start = settings->sequence_number;
  //ERS(NCE_UNKNOWN_OWORD_NUMBER);

  return INTERP_OK;
}

/************************************************************************/
/* convert_control_functions

Returned Value: int (INTERP_OK)
Side effects:
   Changes the flow of control.

Called by: execute_block

Calls: control_skip_to
       control_back_to
       control_save_offset
*/

int Interp::convert_control_functions( /* ARGUMENTS           */
 block_pointer block,      /* pointer to a block of RS274/NGC instructions */
 setup_pointer settings)   /* pointer to machine settings                  */
{
  int status = INTERP_OK;
  int index;
  int i;
  context_pointer current_frame, new_frame, leaving_frame,  returnto_frame;
  block_pointer r_block;
  bool is_remap_handler = false; // the sub is executing on behalf of a remapped code
  bool is_py_callable = false;   // the sub name is actually Python
  bool py_exception = false;

  logOword("convert_control_functions");

  // if there is an oword, must get the block->o_number
  // !!!KL
  if(block->o_name)
    {
      control_find_oword(block, settings, &(block->o_number));
    }
  else
    {
      block->o_number = 0;
    }

  // must skip if skipping
  if(settings->skipping_o && (0 != strcmp(settings->skipping_o, block->o_name)))
  {
      logOword("skipping to line: |%s|", settings->skipping_o);
      return INTERP_OK;
  }

  if(settings->skipping_to_sub && (block->o_type != O_sub))
  {
      logOword("skipping to sub: |%s|", settings->skipping_to_sub);
      return INTERP_OK;
  }

  logOword("o_type:%d", block->o_type);
  switch(block->o_type)
    {
    case O_none:
      // not an error because we use this to signal that we
      // are not evaluating functions
      break;

    case O_sub:
      // if the level is not zero, this is a call
      // not the definition
      // if we were skipping, no longer
      if(settings->skipping_o)
      {
          logOword("sub(o_|%s|) was skipping to here", settings->skipping_o);

          // skipping to a sub means that we must define this now
	  CHP(control_save_offset(block->o_number, block, settings));
      }

      if(settings->skipping_o)
	{
	  logOword("no longer skipping to:|%s|", settings->skipping_o);
          settings->skipping_o = NULL; // this IS our block number
	}
      settings->skipping_to_sub = NULL; // this IS our block number

      if(settings->call_level != 0)
	{
	  logOword("call:%f:%f:%f",
		   settings->parameters[1],
		   settings->parameters[2],
		   settings->parameters[3]);
	}
      else
	{
	  logOword("started a subroutine defn");
	  // a definition
	  CHKS((settings->defining_sub == 1), NCE_NESTED_SUBROUTINE_DEFN);
	  CHP(control_save_offset(block->o_number, block, settings));

          settings->skipping_o = block->o_name; // start skipping
          settings->skipping_start = settings->sequence_number;
	  settings->defining_sub = 1;
	  settings->sub_name = block->o_name;
	  logOword("will now skip to: |%s| %d", settings->sub_name,
		   block->o_number);
	}
      break;

    case O_endsub:
    case O_return:

	// this case is executed only for bona-fide NGC subs
	// subs whose name is a Py callable are handled inline during O_call

      // if level is not zero, in a call
      // otherwise in a defn
      // if we were skipping, no longer
      if (settings->skipping_o) 	{
	  logOword("case O_%s -- no longer skipping to:|%s|",
		   (block->o_type == O_endsub) ? "endsub" : "return",
		   settings->skipping_o);
          settings->skipping_o = NULL;
      }
      if (settings->call_level != 0) {
	  //  some shorthands to make this readable
	  leaving_frame = &settings->sub_context[settings->call_level];
	  returnto_frame = &settings->sub_context[settings->call_level - 1];

	  // free local variables
	  free_named_parameters(leaving_frame);
	  leaving_frame->subName = NULL;

	  // drop one call level.
	  settings->call_level--;

	  // restore subroutine parameters.
	  for(i = 0; i < INTERP_SUB_PARAMS; i++) {
	      settings->parameters[i+INTERP_FIRST_SUBROUTINE_PARAM] =
		  returnto_frame->saved_params[i];
	  }

	  // file at this level was marked as closed, so dont reopen.
	  // if (settings->sub_context[settings->call_level].position == -1) {
	  if (returnto_frame->position == -1) {
	      settings->file_pointer = NULL;
	      strcpy(settings->filename, "");
	  } else {
	      logOword("seeking to: %ld", returnto_frame->position);
	      if(settings->file_pointer == NULL) {
		  ERS(NCE_FILE_NOT_OPEN);
	      }
	      //!!!KL must open the new file, if changed
	      if (0 != strcmp(settings->filename, returnto_frame->filename))  {
		  fclose(settings->file_pointer);
		  settings->file_pointer = fopen(returnto_frame->filename, "r");
		  strcpy(settings->filename, returnto_frame->filename);
	      }
	      fseek(settings->file_pointer, returnto_frame->position, SEEK_SET);
	      settings->sequence_number = returnto_frame->sequence_number;

	      // cleanups on return
	      // if we have an epilog function - execute it, passing the
	      // current call frame


	      // aquire the remapped block
	      r_block = &CONTROLLING_BLOCK(_setup);

	      if (HAS_PYTHON_EPILOG(r_block->executing_remap)) {
		  logRemap("O_call: py epilog %s for NGC remap %s",
			   r_block->executing_remap->epilog_func,
			   r_block->executing_remap->remap_ngc);
		  status = pycall(settings,r_block,
				  r_block->executing_remap->epilog_func);
	      }

	      if (HAS_BUILTIN_EPILOG(r_block->executing_remap)) {
		  logRemap("O_call: calling builtin epilogue\n");
		  status = (*this.*r_block->executing_remap->builtin_epilog)(settings, r_block);

		  if (status > INTERP_MIN_ERROR) {
		      logRemap("O_call: builtin epilogue failed: %s \n", interp_status(status));
		  }
	      }

	      // a valid previous context was marked by an M73 as auto-restore
	      if ((leaving_frame->context_status &
		   (CONTEXT_RESTORE_ON_RETURN|CONTEXT_VALID)) ==
		  (CONTEXT_RESTORE_ON_RETURN|CONTEXT_VALID)) {

		  // NB: this means an M71 invalidate context will prevent an
		  // auto-restore on return/endsub
		  restore_context(settings, settings->call_level + 1);
	      }
	      // always invalidate on leaving a context so we dont accidentially
	      // 'run into' a valid context when growing the stack upwards again
	      leaving_frame->context_status &=  ~CONTEXT_VALID;

	  }
	  settings->sub_name = 0;
	  if (returnto_frame->subName)  {
	      settings->sub_name = returnto_frame->subName;
	  } else {
	      settings->sub_name = NULL;
	  }
	}
      else
       {
	 // a definition
	 if (block->o_type == O_endsub) {
	     CHKS((settings->defining_sub != 1), NCE_NOT_IN_SUBROUTINE_DEFN);
	     // no longer skipping or defining
	     if (settings->skipping_o)
		 {
		     logOword("case O_endsub in defn -- no longer skipping to:|%s|",
			      settings->skipping_o);
		     settings->skipping_o = NULL;
		 }
	     settings->defining_sub = NULL;
	     settings->sub_name = NULL;
	 }
       }
      break;

    case O_call:
	current_frame = &settings->sub_context[settings->call_level];
	new_frame = &settings->sub_context[settings->call_level + 1];

	if (!is_pycallable(settings, block->o_name)) {
	    // copy parameters from context
	    // save old values of parameters
	    // save current file position in context
	    // if we were skipping, no longer
	    if (settings->skipping_o) {
		logOword("case O_call -- no longer skipping to:|%s|",
			 settings->skipping_o);
		settings->skipping_o = NULL;
	    }
	    if (settings->call_level >= INTERP_SUB_ROUTINE_LEVELS) {
		ERS(NCE_TOO_MANY_SUBROUTINE_LEVELS);
	    }
	    for(i = 0; i < INTERP_SUB_PARAMS; i++)	{
		current_frame->saved_params[i] =
		    settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM];

		settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM] =
		    block->params[i];
	    }

	    // if the previous file was NULL, mark positon as -1 so as not to
	    // reopen it on return.
	    if (settings->file_pointer == NULL) {
		current_frame->position = -1;
	    } else {
		current_frame->position = ftell(settings->file_pointer);
	    }

	    // save the previous filename
	    logOword("Duping |%s|", settings->filename);
	    current_frame->filename = strstore(settings->filename);
	    current_frame->sequence_number = settings->sequence_number;
	    logOword("(in call)set params[%d] return file:%s offset:%ld",
		     settings->call_level, current_frame->filename, current_frame->position);

	    settings->call_level++;

	    // aquire the remap_frame
	    r_block = &CONTROLLING_BLOCK(*settings);

	    // set the new subName
	    new_frame->subName = block->o_name;
#if 0
	    // just curious: detect recursion
	    if ((settings->call_level > 0) &&
		(new_frame->subName != NULL) &&
		(current_frame->subName  != NULL)) {
		if (!strcmp(new_frame->subName, current_frame->subName)) {
		    logOword("recursive call: '%s' call_level %d\n",
			     new_frame->subName, settings->call_level);
		}
	    }
#endif
	    if (HAS_PYTHON_PROLOG(r_block->executing_remap)) {
		logRemap("O_call: py prologue %s for NGC remap %s ",
			 r_block->executing_remap->prolog_func,
			 r_block->executing_remap->remap_ngc);
		status = pycall(settings, r_block,
				r_block->executing_remap->prolog_func);
	    }

	    if (control_back_to(block,settings) == INTERP_ERROR) {
		settings->call_level--;
		ERS(NCE_UNABLE_TO_OPEN_FILE,block->o_name);
		return INTERP_ERROR;
	    }
	} else {

	    // the subname in 'o<subname> call' turned out to be a Python callable.

	    // we establish a new call frame with all relevant information, and
	    // pass that frame to Python to handle.
	    settings->call_level++;

	    new_frame->subName = block->o_name;

	    // aquire the remap_frame
	    r_block = &CONTROLLING_BLOCK(_setup);


	    // The 'O word sub' is actually a python callable function.
	    // call a prolog function if applicable
	    //   NB: add_parameters will prepare a word dict if called with pydict = true,
	    //   stored in settings->kwargs
	    //   pycall will add this as kwarg type dict to the parameters
	    // call python callable, and save return_value
	    // call epilogue of applicable

	    // a user-defined Python prolog to a user-defined Python function
	    // doesnt make a whole lot of sense.
	    // for now, the only builtin prolog we have is the kwargs-dict
	    // generator add_parameters()

	    if (HAS_PYTHON_PROLOG(r_block->executing_remap)) {
		logRemap("O_call: py prologue %s for py remap %s - useless.. but doing it",
			 r_block->executing_remap->prolog_func,
			 r_block->executing_remap->remap_py);
		status = pycall(settings, r_block,
				r_block->executing_remap->prolog_func);
	    }

	    status =  pycall(settings, r_block,
			     r_block->executing_remap->remap_py);

	    if (status >  INTERP_MIN_ERROR) {
		// end any remappings in progress
		logRemap("O_call: pycall failed, unwinding\n");
		settings->call_level--;
		settings->remap_level = 0;  // FIXME mah dubious
		return status;
	    }
	    logRemap("O_call(%s) return value=%f\n",
		    block->o_name, settings->return_value);


	    if (HAS_PYTHON_EPILOG(r_block->executing_remap)) {
		logRemap("O_call: py epilog %s for py remap %s - useless.. but doing it",
			 r_block->executing_remap->epilog_func,
			 r_block->executing_remap->remap_py);
		status = pycall(settings, r_block,
				  r_block->executing_remap->epilog_func);
	    }
	    settings->call_level--;

	    // now the epilogue
	    if (HAS_BUILTIN_EPILOG(r_block->executing_remap)) {
		logRemap("O_call: calling builtin epilogue for pyremap %s",
			 r_block->executing_remap->name );
		status = (*this.*r_block->executing_remap->builtin_epilog)(settings,r_block);

		// epilog is exepected to set an appropriate error string
		if (status > INTERP_MIN_ERROR) {
		    logRemap("O_call: builtin  epilogue for pyremap %s failed: %s",
			     r_block->executing_remap->name,
			     interp_status(status));
		}
	    }
	    // fall through -  return status;
	}
	break;



    // case O_call:

    // 	// all O_call code is executed in the context of the old frame.
    // 	if (settings->call_level >= INTERP_SUB_ROUTINE_LEVELS) {
    // 	    ERS(NCE_TOO_MANY_SUBROUTINE_LEVELS);
    // 	}

    // 	current_frame = &settings->sub_context[settings->call_level];
    // 	new_frame = &settings->sub_context[settings->call_level + 1];
    // 	is_py_callable = is_pycallable(settings, block->o_name);
    // 	// aquire the 'remap_frame' a.k.a controlling block
    // 	r_block = &CONTROLLING_BLOCK(*settings);

    // 	// determine if this sub is the remap body executing on behalf of a remapped code
    // 	// we're loosing a bit of context by funneling everything through the osub call
    // 	// interface, which needs to be reestablished here but it's more general
    // 	is_remap_handler =
    // 	    (r_block->executing_remap != NULL) &&
    // 	    (((r_block->executing_remap->remap_py != NULL) &&
    // 	     (!strcmp(r_block->executing_remap->remap_py, block->o_name))) ||
    // 	     (((r_block->executing_remap->remap_ngc != NULL) &&
    // 	       (!strcmp(r_block->executing_remap->remap_ngc, block->o_name)))));

    // 	// standard oword call incantations:
    // 	// copy parameters from context
    // 	// save old values of parameters
    // 	// save current file position in context
    // 	// if we were skipping, no longer
    // 	if (settings->skipping_o) {
    // 	    logOword("case O_call -- no longer skipping to:|%s|",
    // 		     settings->skipping_o);
    // 		settings->skipping_o = NULL;
    // 	}
    // 	if (!is_py_callable) {

    // 	    for(i = 0; i < INTERP_SUB_PARAMS; i++)	{
    // 		current_frame->saved_params[i] =
    // 		    settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM];
    // 		settings->parameters[i + INTERP_FIRST_SUBROUTINE_PARAM] =
    // 		    block->params[i];
    // 	    }
    // 	}
    // 	// if the previous file was NULL, mark positon as -1 so as not to
    // 	// reopen it on return.
    // 	if (settings->file_pointer == NULL) {
    // 	    current_frame->position = -1;
    // 	} else {
    // 	    current_frame->position = ftell(settings->file_pointer);
    // 	}
    // 	// save the previous filename
    // 	logOword("Duping |%s|", settings->filename);
    // 	current_frame->filename = strstore(settings->filename);
    // 	current_frame->sequence_number = settings->sequence_number;
    // 	logOword("(in call)set params[%d] return file:%s offset:%ld",
    // 		 settings->call_level, current_frame->filename, current_frame->position);

    // 	// done with NGC call incantation


    // 	// set the new subName
    // 	new_frame->subName = block->o_name;

    // 	if (is_remap_handler && HAS_PYTHON_PROLOG(r_block->executing_remap)) {
    // 	    logRemap("O_call: py prologue %s for NGC remap %s ",
    // 		     r_block->executing_remap->prolog_func,
    // 		     r_block->executing_remap->remap_ngc);
    // 	    status = pycall(settings, r_block,
    // 			    r_block->executing_remap->prolog_func);
    // 	}
    // 	if (control_back_to(block,settings) == INTERP_ERROR) {
    // 	    ERS(NCE_UNABLE_TO_OPEN_FILE,block->o_name);
    // 	    return INTERP_ERROR;
    // 	}
    // 	settings->call_level++;
    // 	//} else {
    // 	// the o_name is Python callable. Depending wether it's a
    // 	// remap handler, or a Osub replacement, arguments are
    // 	// passed a bit differently.

    // 	if (is_remap_handler) {

    // 	    // execute_handler has everything set up already (name, tupleargs, kwargs)
    // 	    // FIXME mah fix execute_handler -> named params!!
    // 	    r_block->tupleargs = bp::make_tuple(r_block->user_data);

    // 	    status = pycall(settings, r_block, block->o_name);
    // 	    if (status > INTERP_MIN_ERROR) {
    // 		ERS("O_call: %s: remap pycall(%s) returned %d, userdata=%d",
    // 		    r_block->executing_remap->name,
    // 			     block->o_name, status, r_block->user_data);

    // 		return INTERP_ERROR;
    // 	    }
    // 		// settings->call_level++;
    // 	    if (status == INTERP_EXECUTE_FINISH) {
    // 		_setup.mdi_interrupt = true; //settings->call_level++;


    // 		r_block->user_data = r_block->py_returned_userdata;
    // 		logRemap("O_call: %s: remap pycall(%s) returned %s, userdata=%d",
    // 			 r_block->executing_remap->name,
    // 			 block->o_name, interp_status(status),
    // 			 r_block->user_data);

    // 		return status;
    // 	    }

    // 	    if (status == INTERP_OK) {
    // 		//settings->call_level--;

    // 		block->o_name = NULL;
    // 		r_block->user_data = 0;
    // 		if (HAS_BUILTIN_EPILOG(r_block->executing_remap)) {
    // 		    status = (*this.*r_block->executing_remap->builtin_epilog)(settings,r_block);
    // 		    if (status > INTERP_MIN_ERROR) {
    // 			ERS("O_call: %s: builtin  epilogue for remap pycall(%s) failed: %s",
    // 			    r_block->executing_remap->name,
    // 			    block->o_name,   // FIXME IS NULL
    // 			    interp_status(status));
    // 			return INTERP_ERROR;
    // 		    }
    // 		}
    // 		return status;
    // 		    //CHP(control_back_to(block,settings));
    // 		    // 	== INTERP_ERROR) {
    // 		    // 	ERS("ERROR RETURNING FROM PYSUB %s", block->o_name);
    // 		    // 	return INTERP_ERROR;
    // 		    // }
    // 		}

    // 	} else {
    // 	    logDebug("O_call: vanilla osub pycall(%s)", block->o_name);
    // 		try {
    // 		    // call with list of positional parameters
    // 		    bp::list plist;

    // 		    for(int i = 0; i < INTERP_SUB_PARAMS; i++)
    // 			plist.append(block->params[i]);
    // 		    block->tupleargs = bp::make_tuple(plist);
    // 		    block->kwargs = bp::dict();
    // 		}
    // 		catch (bp::error_already_set) {
    // 		    if (PyErr_Occurred()) {
    // 			PyErr_Print();
    // 		    }
    // 		    bp::handle_exception();
    // 		    PyErr_Clear();
    // 		    py_exception = true;
    // 		}
    // 		if (py_exception) {
    // 		    CHKS(py_exception,"O_call: Py exception preparing arguments for %s",
    // 			 block->o_name);
    // 		}
    // 		status = pycall(settings, block, block->o_name);
    // 		if (status > INTERP_MIN_ERROR) {
    // 		    ERS("O_call: vanilla osub pycall(%s) failed, unwinding",block->o_name);
    // 		    settings->call_level--;  // drop back
    // 		    return INTERP_ERROR;
    // 		}
    // 		CHP(control_back_to(block,settings));
    // 		// if (control_back_to(block,settings) == INTERP_ERROR) {
    // 		//     ERS("ERROR RETURNING FROM PYSUB %s", block->o_name);
    // 		//     return INTERP_ERROR;
    // 		// }
    // 		settings->call_level++;  // drop back

    // 		return status;
    // 	    }

    // 	return status;
    // 	break;
{'fraction_non_alphanumeric': 0.14767231896390692, 'fraction_numerical': 0.003154770552707499, 'mean_word_length': 2.7434542770569497, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 603, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1059786', 'n_tokens_mistral': 17149, 'n_tokens_neox': 15845, 'n_words': 3532}