<commit_before>#!/usr/bin/env python
# Copyright 2017 Calico LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

from optparse import OptionParser, OptionGroup
import os
import pdb
import pickle
import shutil
import subprocess
import sys

import slurm

"""
basenji_train_reps.py

Train Basenji model replicates using given parameters and data.
"""

################################################################################
# main
################################################################################
def main():
  usage = 'usage: %prog [options] <params_file> <data_dir>'
  parser = OptionParser(usage)

  # train
  train_options = OptionGroup(parser, 'basenji_train.py options')
  train_options.add_option('-o', dest='out_dir',
      default='train_out',
      help='Output directory for test statistics [Default: %default]')
  train_options.add_option('--restore', dest='restore',
      help='Restore model and continue training [Default: %default]')
  train_options.add_option('--trunk', dest='trunk',
      default=False, action='store_true',
      help='Restore only model trunk [Default: %default]')
  train_options.add_option('--tfr_train', dest='tfr_train_pattern',
      default='train-*.tfr',
      help='Training TFRecord pattern string appended to data_dir [Default: %default]')
  train_options.add_option('--tfr_eval', dest='tfr_eval_pattern',
      default='valid-*.tfr',
      help='Evaluation TFRecord pattern string appended to data_dir [Default: %default]')
  parser.add_option_group(train_options)

  # multi
  rep_options = OptionGroup(parser, 'replication options')
  rep_options.add_option('--name', dest='name',
      default='train', help='SLURM name prefix [Default: %default]')
  rep_options.add_option('-p', dest='processes',
      default=None, type='int',
      help='Number of processes, passed by multi script')
  rep_options.add_option('-q', dest='queue',
      default='gtx1080ti',
      help='SLURM queue on which to run the jobs [Default: %default]')
  parser.add_option_group(rep_options)

  (options, args) = parser.parse_args()

  if len(args) != 2:
    parser.error('Must provide parameters and data directory.')
  else:
    params_file = os.path.abspath(args[0])
    data_dir = os.path.abspath(args[1])

  #######################################################
  # prep work

  if os.path.isdir(options.out_dir):
    shutil.rmtree(options.out_dir)
  os.mkdir(options.out_dir)

  # pickle options
  # options_pkl_file = '%s/options.pkl' % options.out_dir
  # options_pkl = open(options_pkl_file, 'wb')
  # pickle.dump(options, options_pkl)
  # options_pkl.close()

  #######################################################
  # launch worker threads
  jobs = []
  for pi in range(options.processes):
    rep_dir = '%s/rep%d' % (options.out_dir, pi)
    os.mkdir(rep_dir)

    cmd = '. /home/drk/anaconda3/etc/profile.d/conda.sh;'
    cmd += ' conda activate tf1.14-gpu;'
    cmd += ' echo $HOSTNAME;'

    cmd += ' basenji_train.py' 
    cmd += ' %s' % options_string(options, train_options, rep_dir)
    cmd += ' %s %s' % (params_file, data_dir)

    name = '%s_p%d' % (options.name, pi)
    sbf = os.path.abspath('%s/train.sb' % rep_dir)
    outf = os.path.abspath('%s/train.out' % rep_dir)
    errf = os.path.abspath('%s/train.err' % rep_dir)

    j = slurm.Job(cmd, name,
        outf, errf, sbf,<commit_msg>temp path<commit_after>#!/usr/bin/env python
# Copyright 2019 Calico LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

from optparse import OptionParser, OptionGroup
import os
import pdb
import pickle
import shutil
import subprocess
import sys

import slurm

"""
basenji_train_reps.py

Train Basenji model replicates using given parameters and data.
"""

################################################################################
# main
################################################################################
def main():
  usage = 'usage: %prog [options] <params_file> <data_dir>'
  parser = OptionParser(usage)

  # train
  train_options = OptionGroup(parser, 'basenji_train.py options')
  train_options.add_option('-o', dest='out_dir',
      default='train_out',
      help='Output directory for test statistics [Default: %default]')
  train_options.add_option('--restore', dest='restore',
      help='Restore model and continue training [Default: %default]')
  train_options.add_option('--trunk', dest='trunk',
      default=False, action='store_true',
      help='Restore only model trunk [Default: %default]')
  train_options.add_option('--tfr_train', dest='tfr_train_pattern',
      default='train-*.tfr',
      help='Training TFRecord pattern string appended to data_dir [Default: %default]')
  train_options.add_option('--tfr_eval', dest='tfr_eval_pattern',
      default='valid-*.tfr',
      help='Evaluation TFRecord pattern string appended to data_dir [Default: %default]')
  parser.add_option_group(train_options)

  # multi
  rep_options = OptionGroup(parser, 'replication options')
  rep_options.add_option('--name', dest='name',
      default='train', help='SLURM name prefix [Default: %default]')
  rep_options.add_option('-p', dest='processes',
      default=None, type='int',
      help='Number of processes, passed by multi script')
  rep_options.add_option('-q', dest='queue',
      default='gtx1080ti',
      help='SLURM queue on which to run the jobs [Default: %default]')
  parser.add_option_group(rep_options)

  (options, args) = parser.parse_args()

  if len(args) != 2:
    parser.error('Must provide parameters and data directory.')
  else:
    params_file = os.path.abspath(args[0])
    data_dir = os.path.abspath(args[1])

  #######################################################
  # prep work

  if os.path.isdir(options.out_dir):
    shutil.rmtree(options.out_dir)
  os.mkdir(options.out_dir)

  # pickle options
  # options_pkl_file = '%s/options.pkl' % options.out_dir
  # options_pkl = open(options_pkl_file, 'wb')
  # pickle.dump(options, options_pkl)
  # options_pkl.close()

  #######################################################
  # launch worker threads
  jobs = []
  for pi in range(options.processes):
    rep_dir = '%s/rep%d' % (options.out_dir, pi)
    os.mkdir(rep_dir)

    cmd = '. /home/drk/anaconda3/etc/profile.d/conda.sh;'
    cmd += ' conda activate tf1.15-gpu;'
    cmd += ' echo $HOSTNAME;'

    # TEMP path!
    cmd += ' /home/drk/code/basenji2/bin/basenji_train.py' 
    cmd += ' %s' % options_string(options, train_options, rep_dir)
    cmd += ' %s %s' % (params_file, data_dir)

    name = '%s_p%d' % (options.name, pi)
    sbf = os.path.abspath('%s/train.sb' % rep_dir)
    outf = os.path.abspath('%s/train.out' % rep_dir)
    errf = os.path.abspath('%s/train.err' % rep_dir)

    j = slurm.Job(cmd, name,
        outf, errf, sbf,
{'fraction_non_alphanumeric': 0.20694006309148266, 'fraction_numerical': 0.004921135646687697, 'mean_word_length': 4.477539737387699, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6057080', 'n_tokens_mistral': 2514, 'n_tokens_neox': 2429, 'n_words': 618}