<commit_before>#!/usr/bin/env python3.4

import argparse, os, sys
import logging
from intent.utils.argpasser import add_args_to_namespace

# Start the logger and set it up. ----------------------------------------------
logging.basicConfig(format=logging.BASIC_FORMAT)
MAIN_LOG = logging.getLogger('INTENT')

#===============================================================================
# Check for dependencies...
#===============================================================================

import_errors = False

try:
	import nltk
except ImportError:
	MAIN_LOG.critical('NLTK module not installed')
	import_errors = True
	
try:
	import lxml
except ImportError:
	MAIN_LOG.critical('lxml not installed.')
	import_errors = True
	
if import_errors:
	MAIN_LOG.critical('Necessary python modules were not found. Please install and try again.')
	sys.exit(2)

	
#===============================================================================
# Set up the environment...
#===============================================================================

from intent.utils.env import c
from intent.utils.argutils import DefaultHelpParser, existsfile,\
	PathArgException, configfile

#===============================================================================
# Now, intialize the subcommands.
#===============================================================================



main = DefaultHelpParser(description="This is the main module for the INTENT package.",
								formatter_class=argparse.ArgumentDefaultsHelpFormatter,
								fromfile_prefix_chars='@')

subparsers = main.add_subparsers(help = 'Valid subcommands', dest='subcommand')

#===============================================================================
# Enrich subcommand
#===============================================================================
enrich = subparsers.add_parser('enrich', help='Enrich igt data.', 
							description='Ingest a XIGT document and add information, such as alignment, or POS tags.',
							formatter_class=argparse.ArgumentDefaultsHelpFormatter)

# Positional arguments ---------------------------------------------------------
enrich.add_argument('IN_FILE', type=existsfile, help='Input XIGT file.')
enrich.add_argument('OUT_FILE', help='Path to output XIGT file.')

# Optional arguments -----------------------------------------------------------
enrich.add_argument('--alignment', choices=['giza','heur', 'none'], default='none',
					help="Add alignment between the translation and gloss lines using the specified method. (Required for projecting POS from translation to language lines.)")
enrich.add_argument('--pos-trans', choices=[0, 1], default=1, type=int, help='POS tag the translation line (required for projecting POS to language line.)')

enrich.add_argument('--pos-lang', choices=['proj', 'class', 'none'], default='none',
				 help='POS tag the language line using either projection (which requires a POS tagged translation line and alignment between trans and gloss)')

enrich.add_argument('--parse-trans', choices=[0,1], default=0, type=int, help='Parse the translation line for phrase structure and dependencies.')
enrich.add_argument('--project-pt', choices=[0,1], default=0, type=int, help='Project the phrase structure tree from translation to language line.')

<commit_msg>Also accidentally removed the "subparsers.required = True" line.<commit_after>#!/usr/bin/env python3.4

import argparse, os, sys
import logging
from intent.utils.argpasser import add_args_to_namespace

# Start the logger and set it up. ----------------------------------------------
logging.basicConfig(format=logging.BASIC_FORMAT)
MAIN_LOG = logging.getLogger('INTENT')

#===============================================================================
# Check for dependencies...
#===============================================================================

import_errors = False

try:
	import nltk
except ImportError:
	MAIN_LOG.critical('NLTK module not installed')
	import_errors = True
	
try:
	import lxml
except ImportError:
	MAIN_LOG.critical('lxml not installed.')
	import_errors = True
	
if import_errors:
	MAIN_LOG.critical('Necessary python modules were not found. Please install and try again.')
	sys.exit(2)

	
#===============================================================================
# Set up the environment...
#===============================================================================

from intent.utils.env import c
from intent.utils.argutils import DefaultHelpParser, existsfile,\
	PathArgException, configfile

#===============================================================================
# Now, intialize the subcommands.
#===============================================================================



main = DefaultHelpParser(description="This is the main module for the INTENT package.",
								formatter_class=argparse.ArgumentDefaultsHelpFormatter,
								fromfile_prefix_chars='@')

subparsers = main.add_subparsers(help = 'Valid subcommands', dest='subcommand')
subparsers.required = True

#===============================================================================
# Enrich subcommand
#===============================================================================
enrich = subparsers.add_parser('enrich', help='Enrich igt data.', 
							description='Ingest a XIGT document and add information, such as alignment, or POS tags.',
							formatter_class=argparse.ArgumentDefaultsHelpFormatter)

# Positional arguments ---------------------------------------------------------
enrich.add_argument('IN_FILE', type=existsfile, help='Input XIGT file.')
enrich.add_argument('OUT_FILE', help='Path to output XIGT file.')

# Optional arguments -----------------------------------------------------------
enrich.add_argument('--alignment', choices=['giza','heur', 'none'], default='none',
					help="Add alignment between the translation and gloss lines using the specified method. (Required for projecting POS from translation to language lines.)")
enrich.add_argument('--pos-trans', choices=[0, 1], default=1, type=int, help='POS tag the translation line (required for projecting POS to language line.)')

enrich.add_argument('--pos-lang', choices=['proj', 'class', 'none'], default='none',
				 help='POS tag the language line using either projection (which requires a POS tagged translation line and alignment between trans and gloss)')

enrich.add_argument('--parse-trans', choices=[0,1], default=0, type=int, help='Parse the translation line for phrase structure and dependencies.')
enrich.add_argument('--project-pt', choices=[0,1], default=0, type=int, help='Project the phrase structure tree from translation to language line.')


{'fraction_non_alphanumeric': 0.3226336219767095, 'fraction_numerical': 0.0035831591519856674, 'mean_word_length': 4.959964412811388, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1480950', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1498, 'n_words': 466}