def prepare_vocabulary(options):
    vocab_prefix = os.path.join(options.working_dir, 'vocab')
    extract_vocab_options = extract_vocab.create_parser().parse_args(['--input', options.corpus_stem, '--output', vocab_prefix])
    extract_vocab.main(extract_vocab_options)
    if (options.input_words_file is None):
        options.input_words_file = (vocab_prefix + '.input')
        orig = (vocab_prefix + '.all')
        filtered_vocab = open(orig).readlines()
        if options.input_vocab_size:
            filtered_vocab = filtered_vocab[:options.input_vocab_size]
        open(options.input_words_file, 'w').writelines(filtered_vocab)
    if (options.output_words_file is None):
        options.output_words_file = (vocab_prefix + '.output')
        if (options.mode == 'label'):
            blacklist = ['<null', '<root', '<start_head', '<dummy', '<head_head', '<stop_head']
            orig = (vocab_prefix + '.special')
            filtered_vocab = open(orig).readlines()
            orig = (vocab_prefix + '.nonterminals')
            filtered_vocab += open(orig).readlines()
            filtered_vocab = [word for word in filtered_vocab if (not any((word.startswith(prefix) for prefix in blacklist)))]
            if options.output_vocab_size:
                filtered_vocab = filtered_vocab[:options.output_vocab_size]
        else:
            orig = (vocab_prefix + '.all')
            filtered_vocab = open(orig).readlines()[:options.output_vocab_size]
        open(options.output_words_file, 'w').writelines(filtered_vocab)
def prepare_vocabulary(options):
    vocab_prefix = os.path.join(options.working_dir, 'vocab')
    extract_vocab_options = extract_vocab.create_parser().parse_args(['--input', options.corpus_stem, '--output', vocab_prefix])
    extract_vocab.main(extract_vocab_options)
    if (options.input_words_file is None):
        options.input_words_file = (vocab_prefix + '.input')
        orig = (vocab_prefix + '.all')
        filtered_vocab = open(orig).readlines()
        if options.input_vocab_size:
            filtered_vocab = filtered_vocab[:options.input_vocab_size]
        open(options.input_words_file, 'w').writelines(filtered_vocab)
    if (options.output_words_file is None):
        options.output_words_file = (vocab_prefix + '.output')
        if (options.mode == 'label'):
            blacklist = ['<null', '<root', '<start_head', '<dummy', '<head_head', '<stop_head']
            orig = (vocab_prefix + '.special')
            filtered_vocab = open(orig).readlines()
            orig = (vocab_prefix + '.nonterminals')
            filtered_vocab += open(orig).readlines()
            filtered_vocab = [word for word in filtered_vocab if (not any((word.startswith(prefix) for prefix in blacklist)))]
            if options.output_vocab_size:
                filtered_vocab = filtered_vocab[:options.output_vocab_size]
        else:
            orig = (vocab_prefix + '.all')
            filtered_vocab = open(orig).readlines()[:options.output_vocab_size]
        open(options.output_words_file, 'w').writelines(filtered_vocab)