<commit_before>//
//  meanie3D-detect
//  cf-algorithms
//
//  Created by <NAME> on 5/3/12.
//  Copyright (c) 2012 <NAME>. All rights reserved.
//

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/progress.hpp>
#include <boost/date_time.hpp>
#include <boost/date_time/local_time/local_time.hpp>
#include <sstream>

#include <meanie3D/meanie3D.h>

#include <map>
#include <vector>
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>
#include <exception>
#include <locale>
#include <limits>
#include <stdlib.h>
#include <netcdf>
#include <map>
#include <time.h>

using namespace std;
using namespace boost;
using namespace netCDF;
using namespace m3D;

#pragma mark -
#pragma mark Definitions

/** Feature-space data type 
 */
typedef double T;

/** Filename formats 
 */
typedef enum {
    TimestampFormatRadolan,
    TimestampFormatOASE2D,
    TimestampFormatOASE3D,
    TimestampFormatHErZTB4,
    TimestampFormatAutomatic,

} TimestampFormat;


#pragma mark -
#pragma mark Command line parsing

void parse_commmandline(program_options::variables_map vm,
        string &filename,
        bool &force) {
    if (vm.count("file") == 0) {
        cerr << "Missing 'file' argument" << endl;

        exit(1);
    }

    filename = vm["file"].as<string>();

    force = (vm.count("force") > 0);
}

#pragma mark -
#pragma mark Helper Methods

/** Allocates an array of type T with n objects.
 * 
 * @param n
 * @return 
 */
template <typename T>
T* allocate(size_t n)
{
    T *array = (T*) malloc(sizeof (T) * n);
    if (array == NULL)
    {
        cerr << "ERROR:out of memory" << endl;
        exit(EXIT_FAILURE);
    }
    return array;
}

template <typename T>
void get_limits(NcVar variable, T& min, T& max) 
{
    min = std::numeric_limits<T>::max();
    max = std::numeric_limits<T>::min();

    T fill_value = 0.0;

    bool have_fill_value = false;

    try 
    {
        NcVarAtt fillValue = variable.getAtt("_FillValue");

        if (!fillValue.isNull()) 
        {
            fillValue.getValues(&fill_value);

            have_fill_value = true;
        }
    }    
    catch (::netCDF::exceptions::NcException &e) {}

    T *values = NULL;
    size_t numElements = 0;
    
    if (variable.getDimCount() == 1) 
    {
        // 1D

        numElements = variable.getDim(0).getSize();
        values = allocate<T>(numElements);
    } 
    else if (variable.getDimCount() == 2) 
    {
        // 2D

        size_t N = variable.getDim(0).getSize();
        size_t M = variable.getDim(1).getSize();
        numElements = N * M; 

        values = allocate<T>(numElements);
        variable.getVar(values);
        
    }
    else if (variable.getDimCount() == 3) 
    {
        // 3D

        size_t N = variable.getDim(0).getSize();
        size_t M = variable.getDim(1).getSize();
        size_t K = variable.getDim(2).getSize();
        numElements = N * M * K;

        values = allocate<T>(numElements);

        vector<size_t> count(3, 1);
        count[0] = 1;
        count[1] = M;
        count[2] = K;

        vector<size_t> start(3, 0);
        start[0] = 0;
        start[1] = 0;
        start[2] = 0;

        variable.getVar(start, count, values);
    }
    else 
    {
        cerr << "ERROR:Variables with " << variable.getDimCount() 
                << " dimensions are not currently handled" << endl;
        return;
    }
    
    #if WITH_OPENMP
    #pragma omp parallel for 
    #endif
    for (size_t j = 0; j < numElements; j++) 
    {
        T value = values[j];

        if (have_fill_value && value == fill_value) continue;

        if (value < min) 
        {
            #if WITH_OPENMP
            #pragma omp critical
            #endif
            min = value;
        }

        if (value > max) 
        {
            #if WITH_OPENMP
            #pragma omp critical<commit_msg>Fixed the 1D bug in minmax tool<commit_after>//
//  meanie3D-detect
//  cf-algorithms
//
//  Created by <NAME> on 5/3/12.
//  Copyright (c) 2012 <NAME>. All rights reserved.
//

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/progress.hpp>
#include <boost/date_time.hpp>
#include <boost/date_time/local_time/local_time.hpp>
#include <sstream>

#include <meanie3D/meanie3D.h>

#include <map>
#include <vector>
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>
#include <exception>
#include <locale>
#include <limits>
#include <stdlib.h>
#include <netcdf>
#include <map>
#include <time.h>

using namespace std;
using namespace boost;
using namespace netCDF;
using namespace m3D;

#pragma mark -
#pragma mark Definitions

/** Feature-space data type 
 */
typedef double T;

/** Filename formats 
 */
typedef enum {
    TimestampFormatRadolan,
    TimestampFormatOASE2D,
    TimestampFormatOASE3D,
    TimestampFormatHErZTB4,
    TimestampFormatAutomatic,

} TimestampFormat;


#pragma mark -
#pragma mark Command line parsing

void parse_commmandline(program_options::variables_map vm,
        string &filename,
        bool &force) {
    if (vm.count("file") == 0) {
        cerr << "Missing 'file' argument" << endl;

        exit(1);
    }

    filename = vm["file"].as<string>();

    force = (vm.count("force") > 0);
}

#pragma mark -
#pragma mark Helper Methods

/** Allocates an array of type T with n objects.
 * 
 * @param n
 * @return 
 */
template <typename T>
T* allocate(size_t n)
{
    T *array = (T*) malloc(sizeof (T) * n);
    if (array == NULL)
    {
        cerr << "ERROR:out of memory" << endl;
        exit(EXIT_FAILURE);
    }
    return array;
}

template <typename T>
void get_limits(NcVar variable, T& min, T& max) 
{
    min = std::numeric_limits<T>::max();
    max = std::numeric_limits<T>::min();

    T fill_value = 0.0;

    bool have_fill_value = false;

    try 
    {
        NcVarAtt fillValue = variable.getAtt("_FillValue");

        if (!fillValue.isNull()) 
        {
            fillValue.getValues(&fill_value);

            have_fill_value = true;
        }
    }    
    catch (::netCDF::exceptions::NcException &e) {}

    T *values = NULL;
    size_t numElements = 0;
    
    if (variable.getDimCount() == 1) 
    {
        // 1D

        numElements = variable.getDim(0).getSize();
        values = allocate<T>(numElements);
        variable.getVar(values);
    } 
    else if (variable.getDimCount() == 2) 
    {
        // 2D

        size_t N = variable.getDim(0).getSize();
        size_t M = variable.getDim(1).getSize();
        numElements = N * M; 

        values = allocate<T>(numElements);
        variable.getVar(values);
        
    }
    else if (variable.getDimCount() == 3) 
    {
        // 3D

        size_t N = variable.getDim(0).getSize();
        size_t M = variable.getDim(1).getSize();
        size_t K = variable.getDim(2).getSize();
        numElements = N * M * K;

        values = allocate<T>(numElements);

        vector<size_t> count(3, 1);
        count[0] = 1;
        count[1] = M;
        count[2] = K;

        vector<size_t> start(3, 0);
        start[0] = 0;
        start[1] = 0;
        start[2] = 0;

        variable.getVar(start, count, values);
    }
    else 
    {
        cerr << "ERROR:Variables with " << variable.getDimCount() 
                << " dimensions are not currently handled" << endl;
        return;
    }
    
    #if WITH_OPENMP
    #pragma omp parallel for 
    #endif
    for (size_t j = 0; j < numElements; j++) 
    {
        T value = values[j];

        if (have_fill_value && value == fill_value) continue;

        if (value < min) 
        {
            #if WITH_OPENMP
            #pragma omp critical
            #endif
            min = value;
        }
        
        if (value > max) 
        {
            #if WITH_OPENMP
            #pragma omp critical
{'fraction_non_alphanumeric': 0.13462766636862947, 'fraction_numerical': 0.012389832673393792, 'mean_word_length': 2.2235487855084397, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1348251', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2579, 'n_words': 634}