<commit_before>// main.cc
//
// Copyright (c) 2016 <NAME> (<EMAIL>)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Expect command is : ./<this_cmd> --input <folder_name> --output <folder_name> --band <band> --maxzerror <max_z_error>

#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>

#include <string>

#include "lerc_util.h"

void create_directory(const char* directory) {
  struct stat st = {0};
  if (stat(directory, &st) == -1) {
    mkdir(directory, 0700);
  }
}

const char *get_filename_ext(const char *filename) {
  const char *dot = strrchr(filename, '.');
  if(!dot || dot == filename) return "";
  return dot + 1;
}

void list_files_do_stuff(const char* name, int level, const std::string& input_path,
                         const std::string& output_path, double max_z_error, int band,
                         bool signed_type) {
  gago::Logger::LogD("DEBUG - working %s", name);
  gago::Logger::LogD("DT_DIR is %d, DT_UNKOWN is %d", DT_DIR, DT_UNKNOWN);
  
  DIR *dir;
  struct dirent *entry;
  
  if (!(dir = opendir(name)))
    return;
  if (!(entry = readdir(dir)))
    return;
  
  do {
    gago::Logger::LogD("!!! - %s is %d", entry->d_name, entry->d_type);
    
    if (entry->d_type == DT_DIR) {
      gago::Logger::LogD("DEBUG - step2/ DIR - working %s", entry->d_name);
      
      char path[1024];
      int len = snprintf(path, sizeof(path) - 1, "%s/%s", name, entry->d_name);
      path[len] = 0;
      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
        continue;
      
      // create directory in dest folder.
      std::string spec_output_folder = name;
      spec_output_folder += "/";
      spec_output_folder += entry->d_name;
      spec_output_folder.replace(spec_output_folder.begin(), spec_output_folder.begin() + input_path.size(), output_path);
      create_directory(spec_output_folder.c_str());
      
      // continue
      list_files_do_stuff(path, level + 1, input_path, output_path, max_z_error, band, signed_type);
    } else {<commit_msg>Use stat replacing dir.<commit_after>// main.cc
//
// Copyright (c) 2016 <NAME> (<EMAIL>)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Expect command is : ./<this_cmd> --input <folder_name> --output <folder_name> --band <band> --maxzerror <max_z_error>

#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>

#include <string>

#include "lerc_util.h"

void create_directory(const char* directory) {
  struct stat st = {0};
  if (stat(directory, &st) == -1) {
    mkdir(directory, 0700);
  }
}

const char *get_filename_ext(const char *filename) {
  const char *dot = strrchr(filename, '.');
  if(!dot || dot == filename) return "";
  return dot + 1;
}

void list_files_do_stuff(const char* name, int level, const std::string& input_path,
                         const std::string& output_path, double max_z_error, int band,
                         bool signed_type) {
  gago::Logger::LogD("DEBUG - working %s", name);
  gago::Logger::LogD("DT_DIR is %d, DT_UNKOWN is %d", DT_DIR, DT_UNKNOWN);
  
  DIR *dir;
  struct dirent *entry;
  
  if (!(dir = opendir(name)))
    return;
  if (!(entry = readdir(dir)))
    return;
  
  do {
    bool isDir = false;
    if (entry->d_type == DT_UNKNOWN) {
      struct stat st;
      if (stat(name, &st) == 0) {
        isDir = S_ISDIR(st.st_mode);
      }
    } else if (entry->d_type == DT_DIR) {
      isDir = true;
    }
    
    if (isDir) {
      gago::Logger::LogD("DEBUG - step2/ DIR - working %s", entry->d_name);
      char path[1024];
      int len = snprintf(path, sizeof(path) - 1, "%s/%s", name, entry->d_name);
      path[len] = 0;
      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
        continue;
      
      // create directory in dest folder.
      std::string spec_output_folder = name;
      spec_output_folder += "/";
      spec_output_folder += entry->d_name;
      spec_output_folder.replace(spec_output_folder.begin(), spec_output_folder.begin() + input_path.size(), output_path);
      create_directory(spec_output_folder.c_str());
      
      // continue
      list_files_do_stuff(path, level + 1, input_path, output_path, max_z_error, band, signed_type);
    } else {
{'fraction_non_alphanumeric': 0.12947615324472245, 'fraction_numerical': 0.006724003127443315, 'mean_word_length': 3.2498338870431893, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '572525', 'n_tokens_mistral': 2056, 'n_tokens_neox': 1977, 'n_words': 677}