<commit_before>/*
 * buildandrun.cpp
 *
 * buildandrun(argc, argv)  builds the layers, connections, and
 * (to a limited extent) probes from the params file and then calls the hypercol's run method.
 *
 * build(argc, argv) builds the hypercol but does not run it.  That way additional objects
 * can be created and added by hand if they are not yet supported by build().
 *
 *  Created on: May 27, 2011
 *      Author: peteschultz
 */

#include "buildandrun.hpp"

using namespace PV;

int buildandrun(int argc, char * argv[], int (*custominit)(HyPerCol *, int, char **), int (*customexit)(HyPerCol *, int, char **), void * (*customgroups)(const char *, const char *, HyPerCol *)) {

   //Parse param file
   char * param_file = NULL;
   pv_getopt_str(argc, argv, "-p", &param_file);
   InterColComm * icComm = new InterColComm(&argc, &argv);
   PVParams * params = new PVParams(param_file, 2*(INITIAL_LAYER_ARRAY_SIZE+INITIAL_CONNECTION_ARRAY_SIZE), icComm);
   free(param_file);

   int numSweepValues = params->getSweepSize();

   int status = PV_SUCCESS;
   if (numSweepValues) {
      for (int k=0; k<numSweepValues; k++) {
         if (icComm->commRank()==0) {
            printf("Parameter sweep: starting run %d of %d\n", k+1, numSweepValues);
         }
         params->setSweepValues(k);
         status = buildandrun1paramset(argc, argv, custominit, customexit, customgroups, params) == PV_SUCCESS ? status : PV_FAILURE;
      }
   }
   else {
      status = buildandrun1paramset(argc, argv, custominit, customexit, customgroups, params);
   }

   delete params;
   delete icComm;
   return status;
}

int buildandrun1paramset(int argc, char * argv[],
                         int (*custominit)(HyPerCol *, int, char **),
                         int (*customexit)(HyPerCol *, int, char **),
                         void * (*customgroups)(const char *, const char *, HyPerCol *),
                         PVParams * params) {
   HyPerCol * hc = build(argc, argv, customgroups, params);
   if( hc == NULL ) return PV_FAILURE;  // build() prints error message

   int status = PV_SUCCESS;
   if( custominit != NULL ) {
      status = (*custominit)(hc, argc, argv);
      if(status != PV_SUCCESS) {
         fprintf(stderr, "custominit function failed with return value %d\n", status);
      }
   }

   if( status==PV_SUCCESS && hc->getInitialStep() < hc->getFinalStep() ) {
      status = hc->run();
      if( status != PV_SUCCESS ) {
         fprintf(stderr, "HyPerCol::run() returned with error code %d\n", status);
      }
   }
   if( status==PV_SUCCESS && customexit != NULL ) {
      status = (*customexit)(hc, argc, argv);
      if( status != PV_SUCCESS) {
         fprintf(stderr, "customexit function failed with return value %d\n", status);
      }
   }
   delete hc; /* HyPerCol's destructor takes care of deleting layers and connections */
   return status;
}

HyPerCol * build(int argc, char * argv[], void * (*customgroups)(const char *, const char *, HyPerCol *), PVParams * params) {
   HyPerCol * hc = new HyPerCol("column", argc, argv, params);
   if( hc == NULL ) {
      fprintf(stderr, "Unable to create HyPerCol\n");
      return NULL;
   }
   HyPerCol * addedHyPerCol;
   HyPerConn * addedHyPerConn;
   HyPerLayer * addedHyPerLayer;
   ColProbe * addedColProbe;
   LayerProbe * addedLayerProbe;
   BaseConnectionProbe * addedBaseConnectionProbe;

   const char * allowedkeywordarray[] = { // indentation indicates derived class hierarchy
           "_Start_HyPerCols_",
             "HyPerCol",
           "_Stop_HyPerCols_",
           "_Start_HyPerLayers_",
             "HyPerLayer",
             "ANNLayer",
               "AccumulateLayer",
               "ANNSquaredLayer",
               "ANNWhitenedLayer",
               "ANNDivInhLayer",
               "CliqueLayer",
               "GenerativeLayer",
                 "LogLatWTAGenLayer",
                 "PursuitLayer",
               "IncrementLayer",
               "LeakyIntegrator",
               "MatchingPursuitResidual",
               "PoolingANNLayer",
               "PtwiseProductLayer",
               "TrainingLayer",
               "MaxPooling",
               "HyPerLCALayer",
               "ANNErrorLayer",
               "MLPErrorLayer",
               "MLPForwardLayer",
               "MLPOutputLayer",
               "LabelErrorLayer",
               "ANNLabelLayer",
// Leaving ANNTriggerUpdateOnNewImageLayer in for now, to provide a meaningful error message if someone tries to use it
               "ANNTriggerUpdateOnNewImageLayer",
               "ConstantLayer",
             "CloneVLayer",
               "BIDSCloneLayer",
               "GapLayer",
               "RescaleLayer",
               "SigmoidLayer",
             "BinningLayer",
             "LabelLayer",
             "TextStream",
#ifdef PV_USE_SNDFILE
             "SoundStream",
#endif
             "Image",
               "CreateMovies",
               "Movie",
               "Patterns",
             "LIF",
                "LIFGap",
                "BIDSLayer",
                "LCALIFLayer",
             "MatchingPursuitLayer",
             "Retina",
             "ShuffleLayer",
             "BIDSMovieCloneMap",
             "BIDSSensorLayer",
           "_Stop_HyPerLayers_",
           "_Start_HyPerConns_",
             "HyPerConn",
               "BIDSConn",
               "KernelConn",
                 "MapReduceKernelConn",
                 "CliqueConn",
                 "CloneKernelConn",
                 "IdentConn",
                 "ImprintConn",
                 "GapConn",
                 "GenerativeConn",
                   "PoolingGenConn",
#ifdef OBSOLETE  // Marked obsolete Sept 16, 2013.  Learning rule for LCA is the same in KernelConn, so no need to subclass
                 "LCAConn",
#endif // OBSOLETE
                 "LCALIFLateralKernelConn",
                 "NoSelfKernelConn",
                   "SiblingConn",
                 "OjaKernelConn",
                 "ReciprocalConn",
                 "TransposeConn",
                   "FeedbackConn",
               "LCALIFLateralConn",
               "OjaSTDPConn",
               "InhibSTDPConn",
               "STDPConn",
               "STDP3Conn",
           "_Stop_HyPerConns_",
           "_Start_ColProbes_",
             "ColProbe",
               "GenColProbe",
           "_Stop_ColProbes_",
           "_Start_LayerProbes_",
             "LayerProbe",
               "PointProbe",
               "TextStreamProbe",
	       "LCAProbe",
                  "PointLIFProbe",
                    "PointLCALIFProbe",
               "StatsProbe",
               "SparsityLayerProbe",
               "LayerFunctionProbe",
                 "L2NormProbe",
                 "SparsityTermProbe",
                 "LogLatWTAProbe",
               "RequireAllZeroActivityProbe",
           "_Stop_LayerProbes_",
           "_Start_BaseConnectionProbes_",
             "KernelProbe",
             "OjaConnProbe",
             "OjaKernelSpikeRateProbe",
             "LCALIFLateralProbe",
             "PatchProbe",
             "ReciprocalEnergyProbe",
           "_Stop_BaseConnectionProbes_",
           "_Start_ConnectionProbes_",
           "_Stop_ConnectionProbes_",
           "_End_allowedkeywordarray" // Don't delete this; it provides a for-loop test that doesn't require you to keep track of the total number of keywords.
   };
   int first_hypercol_index = -1;
   int last_hypercol_index = -1;
   int first_hyperlayer_index = -1;
   int last_hyperlayer_index = -1;
   int first_hyperconn_index = -1;
   int last_hyperconn_index = -1;
   int first_colprobe_index = -1;
   int last_colprobe_index = -1;
   int first_baseconnectionprobe_index = -1;
   int last_baseconnectionprobe_index = -1;
   int first_connectionprobe_index = -1;
   int last_connectionprobe_index = -1;
   int first_layerprobe_index = -1;
   int last_layerprobe_index = -1;

   int j;
   for( j=0; strcmp(allowedkeywordarray[j],"_End_allowedkeywordarray"); j++ ) {
       const char * kw = allowedkeywordarray[j];
       if( !strcmp(kw,"_Start_HyPerCols_") ) { first_hypercol_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerCols_") )  { last_hypercol_index = j; continue;}
       if( !strcmp(kw,"_Start_HyPerLayers_") ) { first_hyperlayer_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerLayers_") ) { last_hyperlayer_index = j; continue;}
       if( !strcmp(kw,"_Start_HyPerConns_") ) { first_hyperconn_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerConns_") ) { last_hyperconn_index = j; continue;}
       if( !strcmp(kw,"_Start_ColProbes_") ) { first_colprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_ColProbes_") ) { last_colprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_BaseConnectionProbes_") ) { first_baseconnectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_BaseConnectionProbes_") ) { last_baseconnectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_ConnectionProbes_") ) { first_connectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_ConnectionProbes_") ) { last_connectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_LayerProbes_") ) { first_layerprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_LayerProbes_") ) { last_layerprobe_index = j; continue;}
   }

   int numclasskeywords = j;

   assert( first_hypercol_index >= 0 && first_hypercol_index < numclasskeywords );
   assert( last_hypercol_index >= 0 && last_hypercol_index < numclasskeywords );
   assert( first_hyperconn_index >= 0 && first_hyperconn_index < numclasskeywords );
   assert( last_hyperconn_index >= 0 && last_hyperconn_index < numclasskeywords );
   assert( first_hyperlayer_index >= 0 && first_hyperlayer_index < numclasskeywords );
   assert( last_hyperlayer_index >= 0 && last_hyperlayer_index < numclasskeywords );
   assert( first_colprobe_index >= 0 && first_colprobe_index < numclasskeywords );
   assert( last_colprobe_index >= 0 && last_colprobe_index < numclasskeywords );
   assert( first_connectionprobe_index >= 0 && first_connectionprobe_index < numclasskeywords );
   assert( last_connectionprobe_index >= 0 && last_connectionprobe_index < numclasskeywords );
   assert( first_layerprobe_index >= 0 && first_layerprobe_index < numclasskeywords );
   assert( last_layerprobe_index >= 0 && last_layerprobe_index < numclasskeywords );

   PVParams * hcparams = hc->parameters();
   int numGroups = hcparams->numberOfGroups();

   // Make sure first group defines a column
   if( strcmp(hcparams->groupKeywordFromIndex(0), "HyPerCol") ) {
      fprintf(stderr, "First group of params file did not define a HyPerCol.\n");
      delete hc;
      return NULL;
   }

   for( int k=0; k<numGroups; k++ ) {
      const char * kw = hcparams->groupKeywordFromIndex(k);
      const char * name = hcparams->groupNameFromIndex(k);
      bool didAddObject = false;

      int matchedkeyword = -1;
      for( j=0; j<numclasskeywords; j++ ) {
         if( !strcmp( kw, allowedkeywordarray[j]) ) {
            matchedkeyword = j;
            break;
         }
      }
      if( matchedkeyword < 0 && customgroups != NULL ) {
         void * addedCustomObject = customgroups(kw, name, hc);
         didAddObject = addedCustomObject != NULL;
      }
      else if( j > first_hypercol_index && j < last_hypercol_index ) {
         addedHyPerCol = addHyPerColToColumn(kw, name, hc);
         didAddObject = addedHyPerCol != NULL;
      }
      else if( j > first_hyperconn_index && j < last_hyperconn_index ) {
         addedHyPerConn = addConnToColumn(kw, name, hc);
         didAddObject = addedHyPerConn != NULL;
      }
      else if( j > first_hyperlayer_index && j < last_hyperlayer_index ) {
         addedHyPerLayer = addLayerToColumn(kw, name, hc);
         didAddObject = addedHyPerLayer != NULL;
      }
      else if( j > first_colprobe_index && j < last_colprobe_index ) {
         addedColProbe = addColProbeToColumn(kw, name, hc);
         didAddObject = addedColProbe != NULL;
      }
      else if( j > first_baseconnectionprobe_index && j < last_baseconnectionprobe_index ) {
         addedBaseConnectionProbe = addBaseConnectionProbeToColumn(kw, name, hc);
         didAddObject = addedBaseConnectionProbe != NULL;
      }
     else if( j > first_layerprobe_index && j < last_layerprobe_index ) {
         addedLayerProbe = addLayerProbeToColumn(kw, name, hc);
         didAddObject = addedLayerProbe != NULL;
      }
      else {
         fprintf(stderr,"%s \"%s\" in params: Keyword %s is unrecognized.\n", kw, name, kw);
         exit(EXIT_FAILURE);
      }

      if( !didAddObject && hc->icCommunicator()->commRank()==0 ) {
         fprintf(stderr, "Parameter group \"%s\": %s could not be created.\n", name, kw);
      }
   }

   if( hc->numberOfLayers() == 0 ) {
      fprintf(stderr, "HyPerCol \"%s\" does not have any layers.\n", hc->getName());
      delete hc;
      return NULL;
   }
   // if( hc->numberOfConnections() == 0 ) {
   //    fprintf(stderr, "HyPerCol \"%s\" does not have any connections.\n", hc->getName());
   //    delete hc;
   //    return NULL;
   // }
   return hc;
}

HyPerCol * addHyPerColToColumn(const char * classkeyword, const char * name, HyPerCol * hc) {
   return hc;
}

HyPerLayer * addLayerToColumn(const char * classkeyword, const char * name, HyPerCol * hc) {
   bool keywordMatched;
   HyPerLayer * addedLayer;
   assert( hc != NULL );
   if( !strcmp(classkeyword, "HyPerLayer") ) {
      keywordMatched = true;
      fprintf(stderr, "Group \"%s\": abstract class HyPerLayer cannot be instantiated.\n", name);
      addedLayer = NULL;
   }
   int status = PV_SUCCESS;
   if( !strcmp(classkeyword, "ANNLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNLayer(name, hc);
   }
   if( !strcmp(classkeyword, "AccumulateLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new AccumulateLayer(name, hc);
   }
   if( !strcmp(classkeyword, "BIDSMovieCloneMap") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BIDSMovieCloneMap(name, hc);
   }
   if( !strcmp(classkeyword, "BIDSSensorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BIDSSensorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNSquaredLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNSquaredLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNWhitenedLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNWhitenedLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNDivInhLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNDivInh(name, hc);
   }
   if( !strcmp(classkeyword, "CliqueLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CliqueLayer(name, hc);
   }
   if( !strcmp(classkeyword, "GenerativeLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new GenerativeLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LogLatWTAGenLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LogLatWTAGenLayer(name, hc);
   }
   if (!strcmp(classkeyword, "PursuitLayer")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PursuitLayer(name, hc);
   }
   if( !strcmp(classkeyword, "IncrementLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new IncrementLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LeakyIntegrator") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LeakyIntegrator(name, hc);
   }
   if (!strcmp(classkeyword, "MatchingPursuitResidual")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MatchingPursuitResidual(name, hc);
   }
   if( !strcmp(classkeyword, "PoolingANNLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PoolingANNLayer(name, hc);
   }
   if( !strcmp(classkeyword, "PtwiseProductLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PtwiseProductLayer(name, hc);
   }
   if( !strcmp(classkeyword, "TrainingLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new TrainingLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MaxPooling") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MaxPooling(name, hc);
   }
   if( !strcmp(classkeyword, "CloneVLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CloneVLayer(name, hc);
   }
   if( !strcmp(classkeyword, "BinningLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BinningLayer(name, hc);
   }
   if( !strcmp(classkeyword, "TextStream") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new TextStream(name, hc);
   }
#ifdef PV_USE_SNDFILE
   if( !strcmp(classkeyword, "SoundStream") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new SoundStream(name, hc);
   }
#endif
   if( !strcmp(classkeyword, "Image") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Image(name, hc);
   }
   if( !strcmp(classkeyword, "CreateMovies") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CreateMovies(name, hc);
   }
   if( !strcmp(classkeyword, "Movie") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Movie(name, hc);
   }
   if ( !strcmp(classkeyword, "LabelLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LabelLayer(name,hc);
   }
   if( !strcmp(classkeyword, "Patterns") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Patterns(name, hc);
   }
   if( !strcmp(classkeyword, "LIF") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LIF(name, hc);
   }
   if( !strcmp(classkeyword, "LIFGap") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LIFGap(name, hc);
   }
   if( !strcmp(classkeyword, "LCALIFLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LCALIFLayer(name, hc);
   }
   if( !strcmp(classkeyword, "GapLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new GapLayer(name, hc);
   }
   if( !strcmp(classkeyword, "HyPerLCALayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new HyPerLCALayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPForwardLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPForwardLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPOutputLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPOutputLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LabelErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LabelErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNLabelLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNLabelLayer(name, hc);
   }
// Use ANNLayer with triggerFlag set to true and triggerLayerName for the triggering layer
   if( !strcmp(classkeyword, "ANNTriggerUpdateOnNewImageLayer") ) {
      keywordMatched = false; // true;
      // addedLayer = (HyPerLayer *) new ANNTriggerUpdateOnNewImageLayer(name, hc);
      if (hc->columnId()==0) {
         fprintf(stderr, "Error: ANNTriggerUpdateOnNewImageLayer is obsolete.\n");
         fprintf(stderr, "    Use ANNLayer with parameter triggerFlag set to true\n");
         fprintf(stderr, "    and triggerLayerName set to the triggering layer.\n");
      }
      MPI_Barrier(hc->icCommunicator()->communicator());
      exit(EXIT_FAILURE);
   }
   if( !strcmp(classkeyword, "ConstantLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ConstantLayer(name, hc);
   }
   if( !strcmp(classkeyword, "SigmoidLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new SigmoidLayer(name, hc);
   }
   if( !strcmp(classkeyword, "RescaleLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new RescaleLayer(name, hc);
   }
   if (!strcmp(classkeyword, "MatchingPursuitLayer")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MatchingPursuitLayer(name, hc);
   }
   if( !strcmp(classkeyword, "Retina") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Retina(name, hc);
   }<commit_msg>Added #if PV_USE_MPI guards so that PV can compile without MPI compiler.<commit_after>/*
 * buildandrun.cpp
 *
 * buildandrun(argc, argv)  builds the layers, connections, and
 * (to a limited extent) probes from the params file and then calls the hypercol's run method.
 *
 * build(argc, argv) builds the hypercol but does not run it.  That way additional objects
 * can be created and added by hand if they are not yet supported by build().
 *
 *  Created on: May 27, 2011
 *      Author: peteschultz
 */

#include "buildandrun.hpp"

using namespace PV;

int buildandrun(int argc, char * argv[], int (*custominit)(HyPerCol *, int, char **), int (*customexit)(HyPerCol *, int, char **), void * (*customgroups)(const char *, const char *, HyPerCol *)) {

   //Parse param file
   char * param_file = NULL;
   pv_getopt_str(argc, argv, "-p", &param_file);
   InterColComm * icComm = new InterColComm(&argc, &argv);
   PVParams * params = new PVParams(param_file, 2*(INITIAL_LAYER_ARRAY_SIZE+INITIAL_CONNECTION_ARRAY_SIZE), icComm);
   free(param_file);

   int numSweepValues = params->getSweepSize();

   int status = PV_SUCCESS;
   if (numSweepValues) {
      for (int k=0; k<numSweepValues; k++) {
         if (icComm->commRank()==0) {
            printf("Parameter sweep: starting run %d of %d\n", k+1, numSweepValues);
         }
         params->setSweepValues(k);
         status = buildandrun1paramset(argc, argv, custominit, customexit, customgroups, params) == PV_SUCCESS ? status : PV_FAILURE;
      }
   }
   else {
      status = buildandrun1paramset(argc, argv, custominit, customexit, customgroups, params);
   }

   delete params;
   delete icComm;
   return status;
}

int buildandrun1paramset(int argc, char * argv[],
                         int (*custominit)(HyPerCol *, int, char **),
                         int (*customexit)(HyPerCol *, int, char **),
                         void * (*customgroups)(const char *, const char *, HyPerCol *),
                         PVParams * params) {
   HyPerCol * hc = build(argc, argv, customgroups, params);
   if( hc == NULL ) return PV_FAILURE;  // build() prints error message

   int status = PV_SUCCESS;
   if( custominit != NULL ) {
      status = (*custominit)(hc, argc, argv);
      if(status != PV_SUCCESS) {
         fprintf(stderr, "custominit function failed with return value %d\n", status);
      }
   }

   if( status==PV_SUCCESS && hc->getInitialStep() < hc->getFinalStep() ) {
      status = hc->run();
      if( status != PV_SUCCESS ) {
         fprintf(stderr, "HyPerCol::run() returned with error code %d\n", status);
      }
   }
   if( status==PV_SUCCESS && customexit != NULL ) {
      status = (*customexit)(hc, argc, argv);
      if( status != PV_SUCCESS) {
         fprintf(stderr, "customexit function failed with return value %d\n", status);
      }
   }
   delete hc; /* HyPerCol's destructor takes care of deleting layers and connections */
   return status;
}

HyPerCol * build(int argc, char * argv[], void * (*customgroups)(const char *, const char *, HyPerCol *), PVParams * params) {
   HyPerCol * hc = new HyPerCol("column", argc, argv, params);
   if( hc == NULL ) {
      fprintf(stderr, "Unable to create HyPerCol\n");
      return NULL;
   }
   HyPerCol * addedHyPerCol;
   HyPerConn * addedHyPerConn;
   HyPerLayer * addedHyPerLayer;
   ColProbe * addedColProbe;
   LayerProbe * addedLayerProbe;
   BaseConnectionProbe * addedBaseConnectionProbe;

   const char * allowedkeywordarray[] = { // indentation indicates derived class hierarchy
           "_Start_HyPerCols_",
             "HyPerCol",
           "_Stop_HyPerCols_",
           "_Start_HyPerLayers_",
             "HyPerLayer",
             "ANNLayer",
               "AccumulateLayer",
               "ANNSquaredLayer",
               "ANNWhitenedLayer",
               "ANNDivInhLayer",
               "CliqueLayer",
               "GenerativeLayer",
                 "LogLatWTAGenLayer",
                 "PursuitLayer",
               "IncrementLayer",
               "LeakyIntegrator",
               "MatchingPursuitResidual",
               "PoolingANNLayer",
               "PtwiseProductLayer",
               "TrainingLayer",
               "MaxPooling",
               "HyPerLCALayer",
               "ANNErrorLayer",
               "MLPErrorLayer",
               "MLPForwardLayer",
               "MLPOutputLayer",
               "LabelErrorLayer",
               "ANNLabelLayer",
// Leaving ANNTriggerUpdateOnNewImageLayer in for now, to provide a meaningful error message if someone tries to use it
               "ANNTriggerUpdateOnNewImageLayer",
               "ConstantLayer",
             "CloneVLayer",
               "BIDSCloneLayer",
               "GapLayer",
               "RescaleLayer",
               "SigmoidLayer",
             "BinningLayer",
             "LabelLayer",
             "TextStream",
#ifdef PV_USE_SNDFILE
             "SoundStream",
#endif
             "Image",
               "CreateMovies",
               "Movie",
               "Patterns",
             "LIF",
                "LIFGap",
                "BIDSLayer",
                "LCALIFLayer",
             "MatchingPursuitLayer",
             "Retina",
             "ShuffleLayer",
             "BIDSMovieCloneMap",
             "BIDSSensorLayer",
           "_Stop_HyPerLayers_",
           "_Start_HyPerConns_",
             "HyPerConn",
               "BIDSConn",
               "KernelConn",
                 "MapReduceKernelConn",
                 "CliqueConn",
                 "CloneKernelConn",
                 "IdentConn",
                 "ImprintConn",
                 "GapConn",
                 "GenerativeConn",
                   "PoolingGenConn",
#ifdef OBSOLETE  // Marked obsolete Sept 16, 2013.  Learning rule for LCA is the same in KernelConn, so no need to subclass
                 "LCAConn",
#endif // OBSOLETE
                 "LCALIFLateralKernelConn",
                 "NoSelfKernelConn",
                   "SiblingConn",
                 "OjaKernelConn",
                 "ReciprocalConn",
                 "TransposeConn",
                   "FeedbackConn",
               "LCALIFLateralConn",
               "OjaSTDPConn",
               "InhibSTDPConn",
               "STDPConn",
               "STDP3Conn",
           "_Stop_HyPerConns_",
           "_Start_ColProbes_",
             "ColProbe",
               "GenColProbe",
           "_Stop_ColProbes_",
           "_Start_LayerProbes_",
             "LayerProbe",
               "PointProbe",
               "TextStreamProbe",
	       "LCAProbe",
                  "PointLIFProbe",
                    "PointLCALIFProbe",
               "StatsProbe",
               "SparsityLayerProbe",
               "LayerFunctionProbe",
                 "L2NormProbe",
                 "SparsityTermProbe",
                 "LogLatWTAProbe",
               "RequireAllZeroActivityProbe",
           "_Stop_LayerProbes_",
           "_Start_BaseConnectionProbes_",
             "KernelProbe",
             "OjaConnProbe",
             "OjaKernelSpikeRateProbe",
             "LCALIFLateralProbe",
             "PatchProbe",
             "ReciprocalEnergyProbe",
           "_Stop_BaseConnectionProbes_",
           "_Start_ConnectionProbes_",
           "_Stop_ConnectionProbes_",
           "_End_allowedkeywordarray" // Don't delete this; it provides a for-loop test that doesn't require you to keep track of the total number of keywords.
   };
   int first_hypercol_index = -1;
   int last_hypercol_index = -1;
   int first_hyperlayer_index = -1;
   int last_hyperlayer_index = -1;
   int first_hyperconn_index = -1;
   int last_hyperconn_index = -1;
   int first_colprobe_index = -1;
   int last_colprobe_index = -1;
   int first_baseconnectionprobe_index = -1;
   int last_baseconnectionprobe_index = -1;
   int first_connectionprobe_index = -1;
   int last_connectionprobe_index = -1;
   int first_layerprobe_index = -1;
   int last_layerprobe_index = -1;

   int j;
   for( j=0; strcmp(allowedkeywordarray[j],"_End_allowedkeywordarray"); j++ ) {
       const char * kw = allowedkeywordarray[j];
       if( !strcmp(kw,"_Start_HyPerCols_") ) { first_hypercol_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerCols_") )  { last_hypercol_index = j; continue;}
       if( !strcmp(kw,"_Start_HyPerLayers_") ) { first_hyperlayer_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerLayers_") ) { last_hyperlayer_index = j; continue;}
       if( !strcmp(kw,"_Start_HyPerConns_") ) { first_hyperconn_index = j; continue;}
       if( !strcmp(kw,"_Stop_HyPerConns_") ) { last_hyperconn_index = j; continue;}
       if( !strcmp(kw,"_Start_ColProbes_") ) { first_colprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_ColProbes_") ) { last_colprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_BaseConnectionProbes_") ) { first_baseconnectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_BaseConnectionProbes_") ) { last_baseconnectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_ConnectionProbes_") ) { first_connectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_ConnectionProbes_") ) { last_connectionprobe_index = j; continue;}
       if( !strcmp(kw,"_Start_LayerProbes_") ) { first_layerprobe_index = j; continue;}
       if( !strcmp(kw,"_Stop_LayerProbes_") ) { last_layerprobe_index = j; continue;}
   }

   int numclasskeywords = j;

   assert( first_hypercol_index >= 0 && first_hypercol_index < numclasskeywords );
   assert( last_hypercol_index >= 0 && last_hypercol_index < numclasskeywords );
   assert( first_hyperconn_index >= 0 && first_hyperconn_index < numclasskeywords );
   assert( last_hyperconn_index >= 0 && last_hyperconn_index < numclasskeywords );
   assert( first_hyperlayer_index >= 0 && first_hyperlayer_index < numclasskeywords );
   assert( last_hyperlayer_index >= 0 && last_hyperlayer_index < numclasskeywords );
   assert( first_colprobe_index >= 0 && first_colprobe_index < numclasskeywords );
   assert( last_colprobe_index >= 0 && last_colprobe_index < numclasskeywords );
   assert( first_connectionprobe_index >= 0 && first_connectionprobe_index < numclasskeywords );
   assert( last_connectionprobe_index >= 0 && last_connectionprobe_index < numclasskeywords );
   assert( first_layerprobe_index >= 0 && first_layerprobe_index < numclasskeywords );
   assert( last_layerprobe_index >= 0 && last_layerprobe_index < numclasskeywords );

   PVParams * hcparams = hc->parameters();
   int numGroups = hcparams->numberOfGroups();

   // Make sure first group defines a column
   if( strcmp(hcparams->groupKeywordFromIndex(0), "HyPerCol") ) {
      fprintf(stderr, "First group of params file did not define a HyPerCol.\n");
      delete hc;
      return NULL;
   }

   for( int k=0; k<numGroups; k++ ) {
      const char * kw = hcparams->groupKeywordFromIndex(k);
      const char * name = hcparams->groupNameFromIndex(k);
      bool didAddObject = false;

      int matchedkeyword = -1;
      for( j=0; j<numclasskeywords; j++ ) {
         if( !strcmp( kw, allowedkeywordarray[j]) ) {
            matchedkeyword = j;
            break;
         }
      }
      if( matchedkeyword < 0 && customgroups != NULL ) {
         void * addedCustomObject = customgroups(kw, name, hc);
         didAddObject = addedCustomObject != NULL;
      }
      else if( j > first_hypercol_index && j < last_hypercol_index ) {
         addedHyPerCol = addHyPerColToColumn(kw, name, hc);
         didAddObject = addedHyPerCol != NULL;
      }
      else if( j > first_hyperconn_index && j < last_hyperconn_index ) {
         addedHyPerConn = addConnToColumn(kw, name, hc);
         didAddObject = addedHyPerConn != NULL;
      }
      else if( j > first_hyperlayer_index && j < last_hyperlayer_index ) {
         addedHyPerLayer = addLayerToColumn(kw, name, hc);
         didAddObject = addedHyPerLayer != NULL;
      }
      else if( j > first_colprobe_index && j < last_colprobe_index ) {
         addedColProbe = addColProbeToColumn(kw, name, hc);
         didAddObject = addedColProbe != NULL;
      }
      else if( j > first_baseconnectionprobe_index && j < last_baseconnectionprobe_index ) {
         addedBaseConnectionProbe = addBaseConnectionProbeToColumn(kw, name, hc);
         didAddObject = addedBaseConnectionProbe != NULL;
      }
     else if( j > first_layerprobe_index && j < last_layerprobe_index ) {
         addedLayerProbe = addLayerProbeToColumn(kw, name, hc);
         didAddObject = addedLayerProbe != NULL;
      }
      else {
         fprintf(stderr,"%s \"%s\" in params: Keyword %s is unrecognized.\n", kw, name, kw);
         exit(EXIT_FAILURE);
      }

      if( !didAddObject && hc->icCommunicator()->commRank()==0 ) {
         fprintf(stderr, "Parameter group \"%s\": %s could not be created.\n", name, kw);
      }
   }

   if( hc->numberOfLayers() == 0 ) {
      fprintf(stderr, "HyPerCol \"%s\" does not have any layers.\n", hc->getName());
      delete hc;
      return NULL;
   }
   // if( hc->numberOfConnections() == 0 ) {
   //    fprintf(stderr, "HyPerCol \"%s\" does not have any connections.\n", hc->getName());
   //    delete hc;
   //    return NULL;
   // }
   return hc;
}

HyPerCol * addHyPerColToColumn(const char * classkeyword, const char * name, HyPerCol * hc) {
   return hc;
}

HyPerLayer * addLayerToColumn(const char * classkeyword, const char * name, HyPerCol * hc) {
   bool keywordMatched;
   HyPerLayer * addedLayer;
   assert( hc != NULL );
   if( !strcmp(classkeyword, "HyPerLayer") ) {
      keywordMatched = true;
      fprintf(stderr, "Group \"%s\": abstract class HyPerLayer cannot be instantiated.\n", name);
      addedLayer = NULL;
   }
   int status = PV_SUCCESS;
   if( !strcmp(classkeyword, "ANNLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNLayer(name, hc);
   }
   if( !strcmp(classkeyword, "AccumulateLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new AccumulateLayer(name, hc);
   }
   if( !strcmp(classkeyword, "BIDSMovieCloneMap") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BIDSMovieCloneMap(name, hc);
   }
   if( !strcmp(classkeyword, "BIDSSensorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BIDSSensorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNSquaredLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNSquaredLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNWhitenedLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNWhitenedLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNDivInhLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNDivInh(name, hc);
   }
   if( !strcmp(classkeyword, "CliqueLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CliqueLayer(name, hc);
   }
   if( !strcmp(classkeyword, "GenerativeLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new GenerativeLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LogLatWTAGenLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LogLatWTAGenLayer(name, hc);
   }
   if (!strcmp(classkeyword, "PursuitLayer")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PursuitLayer(name, hc);
   }
   if( !strcmp(classkeyword, "IncrementLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new IncrementLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LeakyIntegrator") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LeakyIntegrator(name, hc);
   }
   if (!strcmp(classkeyword, "MatchingPursuitResidual")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MatchingPursuitResidual(name, hc);
   }
   if( !strcmp(classkeyword, "PoolingANNLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PoolingANNLayer(name, hc);
   }
   if( !strcmp(classkeyword, "PtwiseProductLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new PtwiseProductLayer(name, hc);
   }
   if( !strcmp(classkeyword, "TrainingLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new TrainingLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MaxPooling") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MaxPooling(name, hc);
   }
   if( !strcmp(classkeyword, "CloneVLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CloneVLayer(name, hc);
   }
   if( !strcmp(classkeyword, "BinningLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new BinningLayer(name, hc);
   }
   if( !strcmp(classkeyword, "TextStream") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new TextStream(name, hc);
   }
#ifdef PV_USE_SNDFILE
   if( !strcmp(classkeyword, "SoundStream") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new SoundStream(name, hc);
   }
#endif
   if( !strcmp(classkeyword, "Image") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Image(name, hc);
   }
   if( !strcmp(classkeyword, "CreateMovies") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new CreateMovies(name, hc);
   }
   if( !strcmp(classkeyword, "Movie") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Movie(name, hc);
   }
   if ( !strcmp(classkeyword, "LabelLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LabelLayer(name,hc);
   }
   if( !strcmp(classkeyword, "Patterns") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Patterns(name, hc);
   }
   if( !strcmp(classkeyword, "LIF") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LIF(name, hc);
   }
   if( !strcmp(classkeyword, "LIFGap") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LIFGap(name, hc);
   }
   if( !strcmp(classkeyword, "LCALIFLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LCALIFLayer(name, hc);
   }
   if( !strcmp(classkeyword, "GapLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new GapLayer(name, hc);
   }
   if( !strcmp(classkeyword, "HyPerLCALayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new HyPerLCALayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPForwardLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPForwardLayer(name, hc);
   }
   if( !strcmp(classkeyword, "MLPOutputLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MLPOutputLayer(name, hc);
   }
   if( !strcmp(classkeyword, "LabelErrorLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new LabelErrorLayer(name, hc);
   }
   if( !strcmp(classkeyword, "ANNLabelLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ANNLabelLayer(name, hc);
   }
// Use ANNLayer with triggerFlag set to true and triggerLayerName for the triggering layer
   if( !strcmp(classkeyword, "ANNTriggerUpdateOnNewImageLayer") ) {
      keywordMatched = false; // true;
      // addedLayer = (HyPerLayer *) new ANNTriggerUpdateOnNewImageLayer(name, hc);
      if (hc->columnId()==0) {
         fprintf(stderr, "Error: ANNTriggerUpdateOnNewImageLayer is obsolete.\n");
         fprintf(stderr, "    Use ANNLayer with parameter triggerFlag set to true\n");
         fprintf(stderr, "    and triggerLayerName set to the triggering layer.\n");
      }
#if PV_USE_MPI
      MPI_Barrier(hc->icCommunicator()->communicator());
#endif
      exit(EXIT_FAILURE);
   }
   if( !strcmp(classkeyword, "ConstantLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new ConstantLayer(name, hc);
   }
   if( !strcmp(classkeyword, "SigmoidLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new SigmoidLayer(name, hc);
   }
   if( !strcmp(classkeyword, "RescaleLayer") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new RescaleLayer(name, hc);
   }
   if (!strcmp(classkeyword, "MatchingPursuitLayer")) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new MatchingPursuitLayer(name, hc);
   }
   if( !strcmp(classkeyword, "Retina") ) {
      keywordMatched = true;
      addedLayer = (HyPerLayer *) new Retina(name, hc);
   }
{'fraction_non_alphanumeric': 0.11977466695679505, 'fraction_numerical': 0.0027562196271850295, 'mean_word_length': 2.6519512625816706, 'pattern_counts': {'":': 4, '<': 49, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5295936', 'n_tokens_mistral': 13064, 'n_tokens_neox': 12978, 'n_words': 2842}