<commit_before>#include "dynet/init.h"
#include "dynet/aligned-mem-pool.h"
#include "dynet/dynet.h"
#include "dynet/weight-decay.h"

#include <iostream>
#include <random>
#include <cmath>

#if HAVE_CUDA
#include "dynet/cuda.h"
#include <device_launch_parameters.h>
#endif

using namespace std;

namespace dynet {

static void remove_args(int& argc, char**& argv, int& argi, int n) {
  for (int i = argi + n; i < argc; ++i)
    argv[i - n] = argv[i];
  argc -= n;
  assert(argc >= 0);
}

DynetParams extract_dynet_params(int& argc, char**& argv, bool shared_parameters) {
  DynetParams params;

  int argi = 1;

#if HAVE_CUDA
  params.gpu_mask = std::vector<int>(MAX_GPUS, 0);
#endif


  while (argi < argc) {
    string arg = argv[argi];

    // Memory
    if (arg == "--dynet-mem" || arg == "--dynet_mem") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-mem expects an argument (the memory, in megabytes, to reserve)\n";
        abort();
      } else {
        params.mem_descriptor = argv[argi + 1];
        remove_args(argc, argv, argi, 2);
      }
    }

    // Weight decay
    if (arg == "--dynet-l2" || arg == "--dynet_l2") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-l2 requires an argument (the weight decay per update)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        istringstream d(a2); d >> params.weight_decay;
        remove_args(argc, argv, argi, 2);
      }
    }

    // Random seed
    if (arg == "--dynet-seed" || arg == "--dynet_seed") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-seed expects an argument (the random number seed)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        istringstream c(a2); c >> params.random_seed;
        remove_args(argc, argv, argi, 2);
      }
    }

#if HAVE_CUDA
    // Number of GPUs
    if (arg == "--dynet_gpus" || arg == "--dynet-gpus") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-gpus expects an argument (number of GPUs to use)\n";
        abort();
      } else {
        if (params.ngpus_requested) {
          cerr << "Multiple instances of --dynet-gpus" << endl; abort();
        }
        params.ngpus_requested = true;
        string a2 = argv[argi + 1];
        istringstream c(a2); c >> params.requested_gpus;
        remove_args(argc, argv, argi, 2);
      }
    }

    // GPU ids
    if (arg == "--dynet_gpu_ids" || arg == "--dynet-gpu-ids") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-gpu-ids expects an argument (comma separated list of physical GPU ids to use)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        if (params.ids_requested) {
          cerr << "Multiple instances of --dynet-gpu-ids" << endl; abort();
        }
        params.ids_requested = true;
        if (a2.size() % 2 != 1) {
          cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
        }
        for (unsigned i = 0; i < a2.size(); ++i) {
          if ((i % 2 == 0 && (a2[i] < '0' || a2[i] > '9')) ||
              (i % 2 == 1 && a2[i] != ',')) {
            cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
          }
          if (i % 2 == 0) {
            int gpu_id = a2[i] - '0';
            if (gpu_id >= MAX_GPUS) { cerr << "Raise MAX_GPUS\n"; abort(); }
            params.gpu_mask[gpu_id]++;
            params.requested_gpus++;
            if (params.gpu_mask[gpu_id] != 1) {
              cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
            }
          }
        }
        remove_args(argc, argv, argi, 2);
      }
    }
#endif
    // Go to next argument
    argi++;

  }

#if HAVE_CUDA
  // Check for conflict between the two ways of requesting GPUs
  if (params.ids_requested && params.ngpus_requested) {
    cerr << "Use only --dynet_gpus or --dynet_gpu_ids, not both\n";
    abort();
  }
#endif

  return params;
}<commit_msg>Fixed bug where some arguments were overlooked.<commit_after>#include "dynet/init.h"
#include "dynet/aligned-mem-pool.h"
#include "dynet/dynet.h"
#include "dynet/weight-decay.h"

#include <iostream>
#include <random>
#include <cmath>

#if HAVE_CUDA
#include "dynet/cuda.h"
#include <device_launch_parameters.h>
#endif

using namespace std;

namespace dynet {

static void remove_args(int& argc, char**& argv, int& argi, int n) {
  for (int i = argi + n; i < argc; ++i)
    argv[i - n] = argv[i];
  argc -= n;
  assert(argc >= 0);
}

DynetParams extract_dynet_params(int& argc, char**& argv, bool shared_parameters) {
  DynetParams params;

  int argi = 1;

#if HAVE_CUDA
  params.gpu_mask = std::vector<int>(MAX_GPUS, 0);
#endif


  while (argi < argc) {
    string arg = argv[argi];

    // Memory
    if (arg == "--dynet-mem" || arg == "--dynet_mem") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-mem expects an argument (the memory, in megabytes, to reserve)\n";
        abort();
      } else {
        params.mem_descriptor = argv[argi + 1];
        remove_args(argc, argv, argi, 2);
      }
    }

    // Weight decay
    else if (arg == "--dynet-l2" || arg == "--dynet_l2") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-l2 requires an argument (the weight decay per update)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        istringstream d(a2); d >> params.weight_decay;
        remove_args(argc, argv, argi, 2);
      }
    }

    // Random seed
    else if (arg == "--dynet-seed" || arg == "--dynet_seed") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-seed expects an argument (the random number seed)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        istringstream c(a2); c >> params.random_seed;
        remove_args(argc, argv, argi, 2);
      }
    }

#if HAVE_CUDA
    // Number of GPUs
    else if (arg == "--dynet_gpus" || arg == "--dynet-gpus") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-gpus expects an argument (number of GPUs to use)\n";
        abort();
      } else {
        if (params.ngpus_requested) {
          cerr << "Multiple instances of --dynet-gpus" << endl; abort();
        }
        params.ngpus_requested = true;
        string a2 = argv[argi + 1];
        istringstream c(a2); c >> params.requested_gpus;
        remove_args(argc, argv, argi, 2);
      }
    }

    // GPU ids
    else if (arg == "--dynet_gpu_ids" || arg == "--dynet-gpu-ids") {
      if ((argi + 1) > argc) {
        cerr << "[dynet] --dynet-gpu-ids expects an argument (comma separated list of physical GPU ids to use)\n";
        abort();
      } else {
        string a2 = argv[argi + 1];
        if (params.ids_requested) {
          cerr << "Multiple instances of --dynet-gpu-ids" << endl; abort();
        }
        params.ids_requested = true;
        if (a2.size() % 2 != 1) {
          cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
        }
        for (unsigned i = 0; i < a2.size(); ++i) {
          if ((i % 2 == 0 && (a2[i] < '0' || a2[i] > '9')) ||
              (i % 2 == 1 && a2[i] != ',')) {
            cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
          }
          if (i % 2 == 0) {
            int gpu_id = a2[i] - '0';
            if (gpu_id >= MAX_GPUS) { cerr << "Raise MAX_GPUS\n"; abort(); }
            params.gpu_mask[gpu_id]++;
            params.requested_gpus++;
            if (params.gpu_mask[gpu_id] != 1) {
              cerr << "Bad argument to --dynet-gpu-ids: " << a2 << endl; abort();
            }
          }
        }
        remove_args(argc, argv, argi, 2);
      }
    }
#endif

    // Go to next argument
    else {
      argi++;
    }


  }

#if HAVE_CUDA
  // Check for conflict between the two ways of requesting GPUs
  if (params.ids_requested && params.ngpus_requested) {
    cerr << "Use only --dynet_gpus or --dynet_gpu_ids, not both\n";
    abort();
  }
#endif

  return params;
}
{'fraction_non_alphanumeric': 0.16919223544145273, 'fraction_numerical': 0.012523481527864746, 'mean_word_length': 2.0101771579344136, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2677341', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2870, 'n_words': 759}