<commit_before>/*
 *  Copyright 2007-2012 The OpenMx Project
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */


#include <R.h>
#include <Rinternals.h>
#include <Rdefines.h>
#include <R_ext/Rdynload.h>
#include <R_ext/BLAS.h>
#include <R_ext/Lapack.h>
#include "omxDefines.h"
#include "omxAlgebraFunctions.h"
#include "omxSymbolTable.h"
#include "omxData.h"
#include "omxFIMLObjective.h"
#include "omxSadmvnWrapper.h"


/**
 * The localobj reference is used to access read-only variables,
 * or variables that can be modified but whose state cannot be
 * accessed from other threads.
 *
 * The sharedobj reference is used to access write-only variables,
 * where the memory writes of any two threads are non-overlapping.
 * No synchronization mechanisms are employed to maintain consistency
 * of sharedobj references.
 *
 *
 * Because (1) these functions may be invoked with arbitrary 
 * rowbegin and rowcount values, and (2) the log-likelihood
 * values for all data rows must be calculated (even in cases
 * of errors), this function is forbidden from return()-ing early.
 *
 * As another consequence of (1) and (2), if "rowbegin" is in
 * the middle of a sequence of identical rows, then defer
 * move "rowbegin" to after the sequence of identical rows.
 * Grep for "[[Comment 4]]" in source code.
 */
void omxFIMLSingleIterationJoint(omxObjective *localobj, omxObjective *sharedobj, int rowbegin, int rowcount) {

    omxFIMLObjective* ofo = ((omxFIMLObjective*) localobj->argStruct);
    omxFIMLObjective* shared_ofo = ((omxFIMLObjective*) sharedobj->argStruct);

	double Q = 0.0;
	int numDefs;
	int numOrdRemoves = 0, numContRemoves=0;
	int returnRowLikelihoods = 0;
    int numIdenticalDefs = 0, numIdenticalOrdinalMissingness = 0, numIdenticalOrdinalRows = 0, numOrdinal = 1,
	                    numIdenticalContinuousMissingness = 0, numIdenticalContinuousRows = 0, numContinuous = 1;

	omxMatrix *cov, *means, *smallRow, *smallCov, *smallMeans, *RCX, *dataColumns;
	omxMatrix *rowLikelihoods, *rowLogLikelihoods;
    omxMatrix *ordMeans, *ordCov, *ordRow, *contRow;
    omxMatrix *halfCov, *reduceCov, *ordContCov;
	omxThresholdColumn *thresholdCols;
	omxData* data;
	double *lThresh, *uThresh, *corList, *weights, *oldDefs;
	int *Infin;
	omxDefinitionVar* defVars;
	
	omxObjective* subObjective;
	

	// Locals, for readability.  Compiler should cut through this.
	cov 		= ofo->cov;
	means		= ofo->means;
	smallRow 	= ofo->smallRow;
	smallCov 	= ofo->smallCov;
	smallMeans	= ofo->smallMeans;
    ordMeans    = ofo->ordMeans;
    ordCov      = ofo->ordCov;
    ordRow      = ofo->ordRow;
    contRow     = ofo->contRow;
    halfCov     = ofo->halfCov;
    reduceCov   = ofo->reduceCov;
    ordContCov  = ofo->ordContCov;
	RCX 		= ofo->RCX;

	data		= ofo->data;
	dataColumns	= ofo->dataColumns;
	defVars		= ofo->defVars;
	oldDefs		= ofo->oldDefs;
	numDefs		= ofo->numDefs;

	corList 	= ofo->corList;
	weights		= ofo->weights;
	lThresh		= ofo->lThresh;
	uThresh		= ofo->uThresh;
	thresholdCols = ofo->thresholdCols;
	returnRowLikelihoods = ofo->returnRowLikelihoods;
	rowLikelihoods = shared_ofo->rowLikelihoods;		// write-only
	rowLogLikelihoods = shared_ofo->rowLogLikelihoods;  // write-only

    Infin            = ofo->Infin;
	subObjective = localobj->subObjective;

    int ordRemove[cov->cols], contRemove[cov->cols];
    int zeros[cov->cols];
    char u = 'U', l = 'L';
    int info;
    double determinant;
    double oned = 1.0, zerod = 0.0, minusoned = -1.0;
    int onei = 1;
    double likelihood;
	int inform;

	int firstRow = 1;
    int row = rowbegin;

    if(numDefs == 0) {
        numIdenticalDefs = data->rows;
    }

    resetDefinitionVariables(oldDefs, numDefs);

	// [[Comment 4]] moving row starting position
	if (row > 0) {
		int prevIdentical = omxDataNumIdenticalRows(data, row - 1);
		row += (prevIdentical - 1);
	}

	while(row < data->rows && (row - rowbegin) < rowcount) {
        localobj->matrix->currentState->currentRow = row;		// Set to a new row.
        int numIdentical = omxDataNumIdenticalRows(data, row);
        if(numIdentical == 0) numIdentical = 1; 
        // N.B.: numIdentical == 0 means an error occurred and was not properly handled;
        // it should never be the case.
        
        omxDataRow(data, row, dataColumns, smallRow);                               // Populate data row
        
        if(OMX_DEBUG_ROWS(row)) {
            Rprintf("Identicality check. Is %sfirst. Total: %d rows identical, %d defs, %d missingness: Continuous: %d rows, %d missingness; Ordinal: %d rows, %d missingness.\n", 
                    (firstRow?"":"not "), numIdentical, numIdenticalDefs, omxDataNumIdenticalRows(data, row), 
                    numIdenticalContinuousRows, numIdenticalContinuousMissingness, 
                    numIdenticalOrdinalRows, numIdenticalOrdinalMissingness);
        }

        if(numIdenticalDefs <= 0 || numIdenticalContinuousMissingness <= 0 || numIdenticalOrdinalMissingness <= 0 || firstRow ) {  // If we're keeping covariance from the previous row, do not populate 
            // Handle Definition Variables.
            if((numDefs != 0 && numIdenticalDefs > 0) || firstRow) {
				int numVarsFilled = 0;
				if(OMX_DEBUG_ROWS(row)) { Rprintf("Handling Definition Vars.\n"); }
				numVarsFilled = handleDefinitionVarList(data, localobj->matrix->currentState, row, defVars, oldDefs, numDefs);
				if (numVarsFilled < 0) {
					for(int nid = 0; nid < numIdentical; nid++) {
						if(returnRowLikelihoods) omxSetMatrixElement(sharedobj->matrix, omxDataIndex(data, row+nid), 0, 0.0);
						omxSetMatrixElement(rowLikelihoods, omxDataIndex(data, row+nid), 0, 0.0);
					}
					if(keepCov <= 0) keepCov = omxDataNumIdenticalDefs(data, row);
					if(keepInverse  <= 0) keepInverse = omxDataNumIdenticalMissingness(data, row);
					// Rprintf("Incrementing Row."); //:::DEBUG:::
					row += numIdentical;
					keepCov -= numIdentical;
					keepInverse -= numIdentical;
					continue;
				} else if (numVarsFilled || firstRow) { 
					// Use firstrow instead of rows == 0 for the case where the first row is all NAs
					// N.B. handling of definition var lists always happens, regardless of firstRow.
					// Recalculate means and covariances.
					if(!(subObjective == NULL)) {
						omxObjectiveCompute(subObjective);
					} else {
						omxRecompute(cov);
						omxRecompute(means);
					}
				}
			}
            // Filter down correlation matrix and calculate thresholds.
            // TODO: If identical ordinal or continuous missingness, ignore only the appropriate columns.
            numOrdRemoves = 0;
            numContRemoves = 0;
			for(int j = 0; j < dataColumns->cols; j++) {
                zeros[j] = 0;
    			int var = omxVectorElement(dataColumns, j);
    			int value = omxIntDataElement(data, row, var);// Indexing correction means this is the index of the upper bound +1.
    			// TODO: Might save time by preseparating ordinal from continuous.
    			if(isnan(value) || value == NA_INTEGER) {  // Value is NA, therefore filter.
    				numOrdRemoves++;
                    numContRemoves++;
                    ordRemove[j] = 1;
                    contRemove[j] = 1;
    				Infin[j] = -1;
                    if(OMX_DEBUG_ROWS(row)) { 
    				    Rprintf("Row %d, column %d, value %d.  NA.\n", row, j, value);<commit_msg>Fix compilation error that I introduced in NA def.var handling<commit_after>/*
 *  Copyright 2007-2012 The OpenMx Project
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */


#include <R.h>
#include <Rinternals.h>
#include <Rdefines.h>
#include <R_ext/Rdynload.h>
#include <R_ext/BLAS.h>
#include <R_ext/Lapack.h>
#include "omxDefines.h"
#include "omxAlgebraFunctions.h"
#include "omxSymbolTable.h"
#include "omxData.h"
#include "omxFIMLObjective.h"
#include "omxSadmvnWrapper.h"


/**
 * The localobj reference is used to access read-only variables,
 * or variables that can be modified but whose state cannot be
 * accessed from other threads.
 *
 * The sharedobj reference is used to access write-only variables,
 * where the memory writes of any two threads are non-overlapping.
 * No synchronization mechanisms are employed to maintain consistency
 * of sharedobj references.
 *
 *
 * Because (1) these functions may be invoked with arbitrary 
 * rowbegin and rowcount values, and (2) the log-likelihood
 * values for all data rows must be calculated (even in cases
 * of errors), this function is forbidden from return()-ing early.
 *
 * As another consequence of (1) and (2), if "rowbegin" is in
 * the middle of a sequence of identical rows, then defer
 * move "rowbegin" to after the sequence of identical rows.
 * Grep for "[[Comment 4]]" in source code.
 */
void omxFIMLSingleIterationJoint(omxObjective *localobj, omxObjective *sharedobj, int rowbegin, int rowcount) {

    omxFIMLObjective* ofo = ((omxFIMLObjective*) localobj->argStruct);
    omxFIMLObjective* shared_ofo = ((omxFIMLObjective*) sharedobj->argStruct);

	double Q = 0.0;
	int numDefs;
	int numOrdRemoves = 0, numContRemoves=0;
	int returnRowLikelihoods = 0;
    int numIdenticalDefs = 0, numIdenticalOrdinalMissingness = 0, numIdenticalOrdinalRows = 0, numOrdinal = 1,
	                    numIdenticalContinuousMissingness = 0, numIdenticalContinuousRows = 0, numContinuous = 1;

	omxMatrix *cov, *means, *smallRow, *smallCov, *smallMeans, *RCX, *dataColumns;
	omxMatrix *rowLikelihoods, *rowLogLikelihoods;
    omxMatrix *ordMeans, *ordCov, *ordRow, *contRow;
    omxMatrix *halfCov, *reduceCov, *ordContCov;
	omxThresholdColumn *thresholdCols;
	omxData* data;
	double *lThresh, *uThresh, *corList, *weights, *oldDefs;
	int *Infin;
	omxDefinitionVar* defVars;
	
	omxObjective* subObjective;
	

	// Locals, for readability.  Compiler should cut through this.
	cov 		= ofo->cov;
	means		= ofo->means;
	smallRow 	= ofo->smallRow;
	smallCov 	= ofo->smallCov;
	smallMeans	= ofo->smallMeans;
    ordMeans    = ofo->ordMeans;
    ordCov      = ofo->ordCov;
    ordRow      = ofo->ordRow;
    contRow     = ofo->contRow;
    halfCov     = ofo->halfCov;
    reduceCov   = ofo->reduceCov;
    ordContCov  = ofo->ordContCov;
	RCX 		= ofo->RCX;

	data		= ofo->data;
	dataColumns	= ofo->dataColumns;
	defVars		= ofo->defVars;
	oldDefs		= ofo->oldDefs;
	numDefs		= ofo->numDefs;

	corList 	= ofo->corList;
	weights		= ofo->weights;
	lThresh		= ofo->lThresh;
	uThresh		= ofo->uThresh;
	thresholdCols = ofo->thresholdCols;
	returnRowLikelihoods = ofo->returnRowLikelihoods;
	rowLikelihoods = shared_ofo->rowLikelihoods;		// write-only
	rowLogLikelihoods = shared_ofo->rowLogLikelihoods;  // write-only

    Infin            = ofo->Infin;
	subObjective = localobj->subObjective;

    int ordRemove[cov->cols], contRemove[cov->cols];
    int zeros[cov->cols];
    char u = 'U', l = 'L';
    int info;
    double determinant;
    double oned = 1.0, zerod = 0.0, minusoned = -1.0;
    int onei = 1;
    double likelihood;
	int inform;

	int firstRow = 1;
    int row = rowbegin;

    if(numDefs == 0) {
        numIdenticalDefs = data->rows;
    }

    resetDefinitionVariables(oldDefs, numDefs);

	// [[Comment 4]] moving row starting position
	if (row > 0) {
		int prevIdentical = omxDataNumIdenticalRows(data, row - 1);
		row += (prevIdentical - 1);
	}

	while(row < data->rows && (row - rowbegin) < rowcount) {
        localobj->matrix->currentState->currentRow = row;		// Set to a new row.
        int numIdentical = omxDataNumIdenticalRows(data, row);
        if(numIdentical == 0) numIdentical = 1; 
        // N.B.: numIdentical == 0 means an error occurred and was not properly handled;
        // it should never be the case.
        
        omxDataRow(data, row, dataColumns, smallRow);                               // Populate data row
        
        if(OMX_DEBUG_ROWS(row)) {
            Rprintf("Identicality check. Is %sfirst. Total: %d rows identical, %d defs, %d missingness: Continuous: %d rows, %d missingness; Ordinal: %d rows, %d missingness.\n", 
                    (firstRow?"":"not "), numIdentical, numIdenticalDefs, omxDataNumIdenticalRows(data, row), 
                    numIdenticalContinuousRows, numIdenticalContinuousMissingness, 
                    numIdenticalOrdinalRows, numIdenticalOrdinalMissingness);
        }

        if(numIdenticalDefs <= 0 || numIdenticalContinuousMissingness <= 0 || numIdenticalOrdinalMissingness <= 0 || firstRow ) {  // If we're keeping covariance from the previous row, do not populate 
            // Handle Definition Variables.
            if((numDefs != 0 && numIdenticalDefs > 0) || firstRow) {
				int numVarsFilled = 0;
				if(OMX_DEBUG_ROWS(row)) { Rprintf("Handling Definition Vars.\n"); }
				numVarsFilled = handleDefinitionVarList(data, localobj->matrix->currentState, row, defVars, oldDefs, numDefs);
				if (numVarsFilled < 0) {
					for(int nid = 0; nid < numIdentical; nid++) {
						if(returnRowLikelihoods) omxSetMatrixElement(sharedobj->matrix, omxDataIndex(data, row+nid), 0, 0.0);
						omxSetMatrixElement(rowLikelihoods, omxDataIndex(data, row+nid), 0, 0.0);
					}
                    if(numDefs != 0 && numIdenticalDefs <= 0) numIdenticalDefs = omxDataNumIdenticalDefs(data, row);
                    if(numIdenticalContinuousMissingness <= 0) numIdenticalContinuousMissingness =
                                                        omxDataNumIdenticalContinuousMissingness(data, row);
                    if(numIdenticalOrdinalMissingness <= 0) numIdenticalOrdinalMissingness = 
                                                        omxDataNumIdenticalOrdinalMissingness(data, row);
                    if(numIdenticalContinuousRows <= 0) numIdenticalContinuousRows = 
                                                        omxDataNumIdenticalContinuousRows(data, row);
                    if(numIdenticalOrdinalRows <= 0) numIdenticalOrdinalRows = 
                                                        omxDataNumIdenticalOrdinalRows(data, row);
                    row += numIdentical;
                    numIdenticalDefs -= numIdentical;
                    numIdenticalContinuousMissingness -= numIdentical;
                    numIdenticalContinuousRows -= numIdentical;
                    numIdenticalOrdinalMissingness -= numIdentical;
                    numIdenticalOrdinalRows -= numIdentical;
					continue;
				} else if (numVarsFilled || firstRow) { 
					// Use firstrow instead of rows == 0 for the case where the first row is all NAs
					// N.B. handling of definition var lists always happens, regardless of firstRow.
					// Recalculate means and covariances.
					if(!(subObjective == NULL)) {
						omxObjectiveCompute(subObjective);
					} else {
						omxRecompute(cov);
						omxRecompute(means);
					}
				}
			}
            // Filter down correlation matrix and calculate thresholds.
            // TODO: If identical ordinal or continuous missingness, ignore only the appropriate columns.
            numOrdRemoves = 0;
            numContRemoves = 0;
			for(int j = 0; j < dataColumns->cols; j++) {
                zeros[j] = 0;
    			int var = omxVectorElement(dataColumns, j);
    			int value = omxIntDataElement(data, row, var);// Indexing correction means this is the index of the upper bound +1.
    			// TODO: Might save time by preseparating ordinal from continuous.
    			if(isnan(value) || value == NA_INTEGER) {  // Value is NA, therefore filter.
    				numOrdRemoves++;
                    numContRemoves++;
                    ordRemove[j] = 1;
                    contRemove[j] = 1;
    				Infin[j] = -1;
                    if(OMX_DEBUG_ROWS(row)) { 
    				    Rprintf("Row %d, column %d, value %d.  NA.\n", row, j, value);
{'fraction_non_alphanumeric': 0.1038751345532831, 'fraction_numerical': 0.008611410118406888, 'mean_word_length': 2.8719611021069693, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7122675', 'n_tokens_mistral': 5739, 'n_tokens_neox': 5302, 'n_words': 1480}