def gen(args):
    tokenizer = BertTokenizer.from_pretrained(modelconfig.MODEL_ARCHIVE_MAP[args.bert_model])
    train_examples = data_utils.read_squad_examples(os.path.join(args.input_dir, 'train.json'), is_training=True)
    train_features = data_utils.convert_examples_to_features(train_examples, tokenizer, args.max_seq_length, args.doc_stride, args.max_query_length, is_training=True)
    logger.info('***** Running training *****')
    logger.info('  Num orig examples = %d', len(train_examples))
    logger.info('  Num split examples = %d', len(train_features))
    input_ids_np = np.array([f.input_ids for f in train_features], dtype=np.int16)
    segment_ids_np = np.array([f.segment_ids for f in train_features], dtype=np.int16)
    input_mask_np = np.array([f.input_mask for f in train_features], dtype=np.int16)
    start_positions_np = np.array([f.start_position for f in train_features], dtype=np.int16)
    end_positions_np = np.array([f.end_position for f in train_features], dtype=np.int16)
    np.savez_compressed(os.path.join(args.output_dir, 'data.npz'), input_ids=input_ids_np, segment_ids=segment_ids_np, input_mask=input_mask_np, start_positions=start_positions_np, end_positions=end_positions_np)
    valid_examples = data_utils.read_squad_examples(os.path.join(args.input_dir, 'dev.json'), is_training=True)
    valid_features = data_utils.convert_examples_to_features(valid_examples, tokenizer, args.max_seq_length, args.doc_stride, args.max_query_length, is_training=True)
    logger.info('  Num orig examples = %d', len(valid_examples))
    logger.info('  Num split examples = %d', len(valid_features))
    valid_input_ids_np = np.array([f.input_ids for f in valid_features], dtype=np.int16)
    valid_segment_ids_np = np.array([f.segment_ids for f in valid_features], dtype=np.int16)
    valid_input_mask_np = np.array([f.input_mask for f in valid_features], dtype=np.int16)
    valid_start_positions_np = np.array([f.start_position for f in valid_features], dtype=np.int16)
    valid_end_positions_np = np.array([f.end_position for f in valid_features], dtype=np.int16)
    np.savez_compressed(os.path.join(args.output_dir, 'dev.npz'), input_ids=valid_input_ids_np, segment_ids=valid_segment_ids_np, input_mask=valid_input_mask_np, start_positions=valid_start_positions_np, end_positions=valid_end_positions_np)
def gen(args):
    tokenizer = BertTokenizer.from_pretrained(modelconfig.MODEL_ARCHIVE_MAP[args.bert_model])
    train_examples = data_utils.read_squad_examples(os.path.join(args.input_dir, 'train.json'), is_training=True)
    train_features = data_utils.convert_examples_to_features(train_examples, tokenizer, args.max_seq_length, args.doc_stride, args.max_query_length, is_training=True)
    logger.info('***** Running training *****')
    logger.info('  Num orig examples = %d', len(train_examples))
    logger.info('  Num split examples = %d', len(train_features))
    input_ids_np = np.array([f.input_ids for f in train_features], dtype=np.int16)
    segment_ids_np = np.array([f.segment_ids for f in train_features], dtype=np.int16)
    input_mask_np = np.array([f.input_mask for f in train_features], dtype=np.int16)
    start_positions_np = np.array([f.start_position for f in train_features], dtype=np.int16)
    end_positions_np = np.array([f.end_position for f in train_features], dtype=np.int16)
    np.savez_compressed(os.path.join(args.output_dir, 'data.npz'), input_ids=input_ids_np, segment_ids=segment_ids_np, input_mask=input_mask_np, start_positions=start_positions_np, end_positions=end_positions_np)
    valid_examples = data_utils.read_squad_examples(os.path.join(args.input_dir, 'dev.json'), is_training=True)
    valid_features = data_utils.convert_examples_to_features(valid_examples, tokenizer, args.max_seq_length, args.doc_stride, args.max_query_length, is_training=True)
    logger.info('  Num orig examples = %d', len(valid_examples))
    logger.info('  Num split examples = %d', len(valid_features))
    valid_input_ids_np = np.array([f.input_ids for f in valid_features], dtype=np.int16)
    valid_segment_ids_np = np.array([f.segment_ids for f in valid_features], dtype=np.int16)
    valid_input_mask_np = np.array([f.input_mask for f in valid_features], dtype=np.int16)
    valid_start_positions_np = np.array([f.start_position for f in valid_features], dtype=np.int16)
    valid_end_positions_np = np.array([f.end_position for f in valid_features], dtype=np.int16)
    np.savez_compressed(os.path.join(args.output_dir, 'dev.npz'), input_ids=valid_input_ids_np, segment_ids=valid_segment_ids_np, input_mask=valid_input_mask_np, start_positions=valid_start_positions_np, end_positions=valid_end_positions_np)