<commit_before>#
# sfi.py - basic SFA command-line client
# this module is also used in sfascan
#

import sys
sys.path.append('.')

import os, os.path
import socket
import re
import datetime
import codecs
import pickle
import json
import shutil
from lxml import etree
from StringIO import StringIO
from optparse import OptionParser
from pprint import PrettyPrinter
from tempfile import mkstemp

from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.gid import GID
from sfa.trust.credential import Credential
from sfa.trust.sfaticket import SfaTicket

from sfa.util.faults import SfaInvalidArgument
from sfa.util.sfalogging import sfi_logger
from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, Xrn
from sfa.util.config import Config
from sfa.util.version import version_core
from sfa.util.cache import Cache

from sfa.storage.record import Record

from sfa.rspecs.rspec import RSpec
from sfa.rspecs.rspec_converter import RSpecConverter
from sfa.rspecs.version_manager import VersionManager

from sfa.client.sfaclientlib import SfaClientBootstrap
from sfa.client.sfaserverproxy import SfaServerProxy, ServerException
from sfa.client.client_helper import pg_users_arg, sfa_users_arg
from sfa.client.return_value import ReturnValue
from sfa.client.candidates import Candidates

CM_PORT=12346

# utility methods here
def optparse_listvalue_callback(option, option_string, value, parser):
    setattr(parser.values, option.dest, value.split(','))

# a code fragment that could be helpful for argparse which unfortunately is 
# available with 2.7 only, so this feels like too strong a requirement for the client side
#class ExtraArgAction  (argparse.Action):
#    def __call__ (self, parser, namespace, values, option_string=None):
# would need a try/except of course
#        (k,v)=values.split('=')
#        d=getattr(namespace,self.dest)
#        d[k]=v
#####
#parser.add_argument ("-X","--extra",dest='extras', default={}, action=ExtraArgAction,
#                     help="set extra flags, testbed dependent, e.g. --extra enabled=true")
    
def optparse_dictvalue_callback (option, option_string, value, parser):
    try:
        (k,v)=value.split('=',1)
        d=getattr(parser.values, option.dest)
        d[k]=v
    except:
        parser.print_help()
        sys.exit(1)

# display methods
def display_rspec(rspec, format='rspec'):
    if format in ['dns']:
        tree = etree.parse(StringIO(rspec))
        root = tree.getroot()
        result = root.xpath("./network/site/node/hostname/text()")
    elif format in ['ip']:
        # The IP address is not yet part of the new RSpec
        # so this doesn't do anything yet.
        tree = etree.parse(StringIO(rspec))
        root = tree.getroot()
        result = root.xpath("./network/site/node/ipv4/text()")
    else:
        result = rspec

    print result
    return

def display_list(results):
    for result in results:
        print result

def display_records(recordList, dump=False):
    ''' Print all fields in the record'''
    for record in recordList:
        display_record(record, dump)

def display_record(record, dump=False):
    if dump:
        record.dump(sort=True)
    else:
        info = record.getdict()
        print "%s (%s)" % (info['hrn'], info['type'])
    return


def filter_records(type, records):
    filtered_records = []
    for record in records:
        if (record['type'] == type) or (type == "all"):
            filtered_records.append(record)
    return filtered_records


def credential_printable (credential_string):
    credential=Credential(string=credential_string)
    result=""
    result += credential.get_summary_tostring()
    result += "\n"
    rights = credential.get_privileges()
    result += "rights=%s"%rights
    result += "\n"
    return result

def show_credentials (cred_s):
    if not isinstance (cred_s,list): cred_s = [cred_s]
    for cred in cred_s:
        print "Using Credential %s"%credential_printable(cred)

# save methods
def save_raw_to_file(var, filename, format="text", banner=None):
    if filename == "-":
        # if filename is "-", send it to stdout
        f = sys.stdout
    else:
        f = open(filename, "w")
    if banner:
        f.write(banner+"\n")
    if format == "text":
        f.write(str(var))
    elif format == "pickled":
        f.write(pickle.dumps(var))
    elif format == "json":
        if hasattr(json, "dumps"):
            f.write(json.dumps(var))   # python 2.6
        else:
            f.write(json.write(var))   # python 2.5
    else:
        # this should never happen
        print "unknown output format", format
    if banner:
        f.write('\n'+banner+"\n")

def save_rspec_to_file(rspec, filename):
    if not filename.endswith(".rspec"):
        filename = filename + ".rspec"
    f = open(filename, 'w')
    f.write(rspec)
    f.close()
    return

def save_records_to_file(filename, record_dicts, format="xml"):
    if format == "xml":
        index = 0
        for record_dict in record_dicts:
            if index > 0:
                save_record_to_file(filename + "." + str(index), record_dict)
            else:
                save_record_to_file(filename, record_dict)
            index = index + 1
    elif format == "xmllist":
        f = open(filename, "w")
        f.write("<recordlist>\n")
        for record_dict in record_dicts:
            record_obj=Record(dict=record_dict)
            f.write('<record hrn="' + record_obj.hrn + '" type="' + record_obj.type + '" />\n')
        f.write("</recordlist>\n")
        f.close()
    elif format == "hrnlist":
        f = open(filename, "w")
        for record_dict in record_dicts:
            record_obj=Record(dict=record_dict)
            f.write(record_obj.hrn + "\n")
        f.close()
    else:
        # this should never happen
        print "unknown output format", format

def save_record_to_file(filename, record_dict):
    record = Record(dict=record_dict)
    xml = record.save_as_xml()
    f=codecs.open(filename, encoding='utf-8',mode="w")
    f.write(xml)
    f.close()
    return

# used in sfi list
def terminal_render (records,options):
    # sort records by type
    grouped_by_type={}
    for record in records:
        type=record['type']
        if type not in grouped_by_type: grouped_by_type[type]=[]
        grouped_by_type[type].append(record)
    group_types=grouped_by_type.keys()
    group_types.sort()
    for type in group_types:
        group=grouped_by_type[type]
#        print 20 * '-', type
        try:    renderer=eval('terminal_render_'+type)
        except: renderer=terminal_render_default
        for record in group: renderer(record,options)

def render_plural (how_many, name,names=None):
    if not names: names="%ss"%name
    if how_many<=0: return "No %s"%name
    elif how_many==1: return "1 %s"%name
    else: return "%d %s"%(how_many,names)

def terminal_render_default (record,options):
    print "%s (%s)" % (record['hrn'], record['type'])
def terminal_render_user (record, options):
    print "%s (User)"%record['hrn'],
    if record.get('reg-pi-authorities',None): print " [PI at %s]"%(" and ".join(record['reg-pi-authorities'])),
    if record.get('reg-slices',None): print " [IN slices %s]"%(" and ".join(record['reg-slices'])),
    user_keys=record.get('reg-keys',[])
    if not options.verbose:
        print " [has %s]"%(render_plural(len(user_keys),"key"))
    else:
        print ""
        for key in user_keys: print 8*' ',key.strip("\n")
        
def terminal_render_slice (record, options):
    print "%s (Slice)"%record['hrn'],
    if record.get('reg-researchers',None): print " [USERS %s]"%(" and ".join(record['reg-researchers'])),
#    print record.keys()
    print ""
def terminal_render_authority (record, options):
    print "%s (Authority)"%record['hrn'],
    if record.get('reg-pis',None): print " [PIS %s]"%(" and ".join(record['reg-pis'])),
    print ""
def terminal_render_node (record, options):
    print "%s (Node)"%record['hrn']

# minimally check a key argument
def check_ssh_key (key):
    good_ssh_key = r'^.*(?:ssh-dss|ssh-rsa)[ ]+[A-Za-z0-9+/=]+(?: .*)?$'
    return re.match(good_ssh_key, key, re.IGNORECASE)

# load methods
def load_record_from_opts(options):
    record_dict = {}
    if hasattr(options, 'xrn') and options.xrn:
        if hasattr(options, 'type') and options.type:
            xrn = Xrn(options.xrn, options.type)
        else:
            xrn = Xrn(options.xrn)
        record_dict['urn'] = xrn.get_urn()
        record_dict['hrn'] = xrn.get_hrn()
        record_dict['type'] = xrn.get_type()
    if hasattr(options, 'key') and options.key:
        try:
            pubkey = open(options.key, 'r').read()
        except IOError:
            pubkey = options.key
        if not check_ssh_key (pubkey):
            raise SfaInvalidArgument(name='key',msg="Could not find file, or wrong key format")
        record_dict['keys'] = [pubkey]
    if hasattr(options, 'slices') and options.slices:
        record_dict['slices'] = options.slices
    if hasattr(options, 'researchers') and options.researchers:
        record_dict['researcher'] = options.researchers
    if hasattr(options, 'email') and options.email:
        record_dict['email'] = options.email
    if hasattr(options, 'pis') and options.pis:
        record_dict['pi'] = options.pis

    # handle extra settings
    record_dict.update(options.extras)
    
    return Record(dict=record_dict)

def load_record_from_file(filename):
    f=codecs.open(filename, encoding="utf-8", mode="r")
    xml_string = f.read()
    f.close()
    return Record(xml=xml_string)


import uuid
def unique_call_id(): return uuid.uuid4().urn

class Sfi:
    
    # dirty hack to make this class usable from the outside
    required_options=['verbose',  'debug',  'registry',  'sm',  'auth',  'user', 'user_private_key']

    @staticmethod
    def default_sfi_dir ():
        if os.path.isfile("./sfi_config"): 
            return os.getcwd()
        else:
            return os.path.expanduser("~/.sfi/")

    # dummy to meet Sfi's expectations for its 'options' field
    # i.e. s/t we can do setattr on
    class DummyOptions:
        pass

    def __init__ (self,options=None):
        if options is None: options=Sfi.DummyOptions()
        for opt in Sfi.required_options:
            if not hasattr(options,opt): setattr(options,opt,None)
        if not hasattr(options,'sfi_dir'): options.sfi_dir=Sfi.default_sfi_dir()
        self.options = options
        self.user = None
        self.authority = None
        self.logger = sfi_logger
        self.logger.enable_console()
        self.available_names = [ tuple[0] for tuple in Sfi.available ]
        self.available_dict = dict (Sfi.available)
   
    # tuples command-name expected-args in the order in which they should appear in the help
    available = [ 
        ("version", ""),  
        ("list", "authority"),
        ("show", "name"),
        ("add", "record"),
        ("update", "record"),
        ("remove", "name"),
        ("slices", ""),
        ("resources", "[slice_hrn]"),
        ("create", "slice_hrn rspec"),
        ("delete", "slice_hrn"),
        ("status", "slice_hrn"),
        ("start", "slice_hrn"),
        ("stop", "slice_hrn"),
        ("reset", "slice_hrn"),
        ("renew", "slice_hrn time"),
        ("shutdown", "slice_hrn"),
        ("get_ticket", "slice_hrn rspec"),
        ("redeem_ticket", "ticket"),
        ("delegate", "to_hrn"),
        ("gid", "[name]"),
        ("trusted", "cred"),
        ("config", ""),
        ]

    def print_command_help (self, options):
        verbose=getattr(options,'verbose')
        format3="%18s %-15s %s"
        line=80*'-'
        if not verbose:
            print format3%("command","cmd_args","description")
            print line
        else:
            print line
            self.create_parser().print_help()
        for command in self.available_names:
            args=self.available_dict[command]
            method=getattr(self,command,None)
            doc=""
            if method: doc=getattr(method,'__doc__',"")
            if not doc: doc="*** no doc found ***"
            doc=doc.strip(" \t\n")
            doc=doc.replace("\n","\n"+35*' ')
            if verbose:
                print line
            print format3%(command,args,doc)
            if verbose:
                self.create_command_parser(command).print_help()

    def create_command_parser(self, command):
        if command not in self.available_dict:
            msg="Invalid command\n"
            msg+="Commands: "
            msg += ','.join(self.available_names)            
            self.logger.critical(msg)
            sys.exit(2)

        parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \
                                     % (command, self.available_dict[command]))

        if command in ("add", "update"):
            parser.add_option('-x', '--xrn', dest='xrn', metavar='<xrn>', help='object hrn/urn (mandatory)')
            parser.add_option('-t', '--type', dest='type', metavar='<type>', help='object type', default=None)
            parser.add_option('-e', '--email', dest='email', default="",  help="email (mandatory for users)") 
# use --extra instead
#            parser.add_option('-u', '--url', dest='url', metavar='<url>', default=None, help="URL, useful for slices") 
#            parser.add_option('-d', '--description', dest='description', metavar='<description>', 
#                              help='Description, useful for slices', default=None)
            parser.add_option('-k', '--key', dest='key', metavar='<key>', help='public key string or file', 
                              default=None)
            parser.add_option('-s', '--slices', dest='slices', metavar='<slices>', help='slice xrns',
                              default='', type="str", action='callback', callback=optparse_listvalue_callback)
            parser.add_option('-r', '--researchers', dest='researchers', metavar='<researchers>', 
                              help='slice researchers', default='', type="str", action='callback', 
                              callback=optparse_listvalue_callback)
            parser.add_option('-p', '--pis', dest='pis', metavar='<PIs>', help='Principal Investigators/Project Managers',
                              default='', type="str", action='callback', callback=optparse_listvalue_callback)
# use --extra instead
#            parser.add_option('-f', '--firstname', dest='firstname', metavar='<firstname>', help='user first name')
#            parser.add_option('-l', '--lastname', dest='lastname', metavar='<lastname>', help='user last name')
            parser.add_option ('-X','--extra',dest='extras',default={},type='str',metavar="<EXTRA_ASSIGNS>",
                               action="callback", callback=optparse_dictvalue_callback, nargs=1,
                               help="set extra/testbed-dependent flags, e.g. --extra enabled=true")

        # show_credential option
        if command in ("list","resources","create","add","update","remove","slices","delete","status","renew"):
            parser.add_option("-C","--credential",dest='show_credential',action='store_true',default=False,
                              help="show credential(s) used in human-readable form")
        # registy filter option
        if command in ("list", "show", "remove"):
            parser.add_option("-t", "--type", dest="type", type="choice",
                            help="type filter ([all]|user|slice|authority|node|aggregate)",
                            choices=("all", "user", "slice", "authority", "node", "aggregate"),
                            default="all")
        if command in ("show"):
            parser.add_option("-k","--key",dest="keys",action="append",default=[],
                              help="specify specific keys to be displayed from record")
        if command in ("resources"):
            # rspec version
            parser.add_option("-r", "--rspec-version", dest="rspec_version", default="SFA 1",
                              help="schema type and version of resulting RSpec")
            # disable/enable cached rspecs
            parser.add_option("-c", "--current", dest="current", default=False,
                              action="store_true",  
                              help="Request the current rspec bypassing the cache. Cached rspecs are returned by default")
            # display formats
            parser.add_option("-f", "--format", dest="format", type="choice",
                             help="display format ([xml]|dns|ip)", default="xml",
                             choices=("xml", "dns", "ip"))
            #panos: a new option to define the type of information about resources a user is interested in
            parser.add_option("-i", "--info", dest="info",
                                help="optional component information", default=None)
            # a new option to retreive or not reservation-oriented RSpecs (leases)
            parser.add_option("-l", "--list_leases", dest="list_leases", type="choice",
                                help="Retreive or not reservation-oriented RSpecs ([resources]|leases|all )",
                                choices=("all", "resources", "leases"), default="resources")


        # 'create' does return the new rspec, makes sense to save that too
        if command in ("resources", "show", "list", "gid", 'create'):
           parser.add_option("-o", "--output", dest="file",
                            help="output XML to file", metavar="FILE", default=None)

        if command in ("show", "list"):
           parser.add_option("-f", "--format", dest="format", type="choice",
                             help="display format ([text]|xml)", default="text",
                             choices=("text", "xml"))

           parser.add_option("-F", "--fileformat", dest="fileformat", type="choice",
                             help="output file format ([xml]|xmllist|hrnlist)", default="xml",
                             choices=("xml", "xmllist", "hrnlist"))
        if command == 'list':
           parser.add_option("-r", "--recursive", dest="recursive", action='store_true',
                             help="list all child records", default=False)
           parser.add_option("-v", "--verbose", dest="verbose", action='store_true',
                             help="gives details, like user keys", default=False)
        if command in ("delegate"):
           parser.add_option("-u", "--user",
                             action="store_true", dest="delegate_user", default=False,
                             help="delegate your own credentials; default if no other option is provided")
           parser.add_option("-s", "--slice", dest="delegate_slices",action='append',default=[],
                             metavar="slice_hrn", help="delegate cred. for slice HRN")
           parser.add_option("-a", "--auths", dest='delegate_auths',action='append',default=[],
                             metavar='auth_hrn', help="delegate cred for auth HRN")
           # this primarily is a shorthand for -a my_hrn^
           parser.add_option("-p", "--pi", dest='delegate_pi', default=None, action='store_true',
                             help="delegate your PI credentials, so s.t. like -a your_hrn^")
           parser.add_option("-A","--to-authority",dest='delegate_to_authority',action='store_true',default=False,
                             help="""by default the mandatory argument is expected to be a user, 
use this if you mean an authority instead""")
        
        if command in ("version"):
            parser.add_option("-R","--registry-version",
                              action="store_true", dest="version_registry", default=False,
                              help="probe registry version instead of sliceapi")
            parser.add_option("-l","--local",
                              action="store_true", dest="version_local", default=False,
                              help="display version of the local client")

        return parser

        
    def create_parser(self):

        # Generate command line parser
        parser = OptionParser(usage="sfi [sfi_options] command [cmd_options] [cmd_args]",
                             description="Commands: %s"%(" ".join(self.available_names)))
        parser.add_option("-r", "--registry", dest="registry",
                         help="root registry", metavar="URL", default=None)
        parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL",
                         help="slice API - in general a SM URL, but can be used to talk to an aggregate")
        parser.add_option("-R", "--raw", dest="raw", default=None,
                          help="Save raw, unparsed server response to a file")
        parser.add_option("", "--rawformat", dest="rawformat", type="choice",
                          help="raw file format ([text]|pickled|json)", default="text",
                          choices=("text","pickled","json"))
        parser.add_option("", "--rawbanner", dest="rawbanner", default=None,
                          help="text string to write before and after raw output")
        parser.add_option("-d", "--dir", dest="sfi_dir",
                         help="config & working directory - default is %default",
                         metavar="PATH", default=Sfi.default_sfi_dir())
        parser.add_option("-u", "--user", dest="user",
                         help="user name", metavar="HRN", default=None)
        parser.add_option("-a", "--auth", dest="auth",
                         help="authority name", metavar="HRN", default=None)
        parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0,
                         help="verbose mode - cumulative")
        parser.add_option("-D", "--debug",
                          action="store_true", dest="debug", default=False,
                          help="Debug (xml-rpc) protocol messages")
        # would it make sense to use ~/.ssh/id_rsa as a default here ?
        parser.add_option("-k", "--private-key",
                         action="store", dest="user_private_key", default=None,
                         help="point to the private key file to use if not yet installed in sfi_dir")
        parser.add_option("-t", "--timeout", dest="timeout", default=None,
                         help="Amout of time to wait before timing out the request")
        parser.add_option("-?", "--commands", 
                         action="store_true", dest="command_help", default=False,
                         help="one page summary on commands & exit")
        parser.disable_interspersed_args()

        return parser
        

    def print_help (self):
        print "==================== Generic sfi usage"
        self.sfi_parser.print_help()
        print "==================== Specific command usage"
        self.command_parser.print_help()

    #
    # Main: parse arguments and dispatch to command
    #
    def dispatch(self, command, command_options, command_args):
        method=getattr(self, command,None)
        if not method:
            print "Unknown command %s"%command
            return
        return method(command_options, command_args)

    def main(self):
        self.sfi_parser = self.create_parser()
        (options, args) = self.sfi_parser.parse_args()
        if options.command_help: 
            self.print_command_help(options)
            sys.exit(1)
        self.options = options

        self.logger.setLevelFromOptVerbose(self.options.verbose)

        if len(args) <= 0:
            self.logger.critical("No command given. Use -h for help.")
            self.print_command_help(options)
            return -1
    
        # complete / find unique match with command set
        command_candidates = Candidates (self.available_names)
        input = args[0]
        command = command_candidates.only_match(input)
        if not command:
            self.print_command_help(options)
            sys.exit(1)
        # second pass options parsing
        self.command_parser = self.create_command_parser(command)
        (command_options, command_args) = self.command_parser.parse_args(args[1:])
        self.command_options = command_options

        self.read_config () 
        self.bootstrap ()
        self.logger.debug("Command=%s" % command)

        try:
            self.dispatch(command, command_options, command_args)
        except:
            self.logger.log_exc ("sfi command %s failed"%command)
            sys.exit(1)

        return
    
    ####################
    def read_config(self):
        config_file = os.path.join(self.options.sfi_dir,"sfi_config")
        shell_config_file  = os.path.join(self.options.sfi_dir,"sfi_config.sh")
        try:
            if Config.is_ini(config_file):
                config = Config (config_file)
            else:
                # try upgrading from shell config format
                fp, fn = mkstemp(suffix='sfi_config', text=True)  
                config = Config(fn)
                # we need to preload the sections we want parsed 
                # from the shell config
                config.add_section('sfi')
                config.add_section('sface')
                config.load(config_file)
                # back up old config
                shutil.move(config_file, shell_config_file)
                # write new config
                config.save(config_file)
                 
        except:
            self.logger.critical("Failed to read configuration file %s"%config_file)
            self.logger.info("Make sure to remove the export clauses and to add quotes")
            if self.options.verbose==0:
                self.logger.info("Re-run with -v for more details")
            else:
                self.logger.log_exc("Could not read config file %s"%config_file)
            sys.exit(1)
     
        errors = 0
        # Set SliceMgr URL
        if (self.options.sm is not None):
           self.sm_url = self.options.sm
        elif hasattr(config, "SFI_SM"):
           self.sm_url = config.SFI_SM
        else:
           self.logger.error("You need to set e.g. SFI_SM='http://your.slicemanager.url:12347/' in %s" % config_file)
           errors += 1 

        # Set Registry URL
        if (self.options.registry is not None):
           self.reg_url = self.options.registry
        elif hasattr(config, "SFI_REGISTRY"):
           self.reg_url = config.SFI_REGISTRY
        else:
           self.logger.error("You need to set e.g. SFI_REGISTRY='http://your.registry.url:12345/' in %s" % config_file)
           errors += 1 

        # Set user HRN
        if (self.options.user is not None):
           self.user = self.options.user
        elif hasattr(config, "SFI_USER"):
           self.user = config.SFI_USER
        else:
           self.logger.error("You need to set e.g. SFI_USER='plc.princeton.username' in %s" % config_file)
           errors += 1 

        # Set authority HRN
        if (self.options.auth is not None):
           self.authority = self.options.auth
        elif hasattr(config, "SFI_AUTH"):
           self.authority = config.SFI_AUTH
        else:
           self.logger.error("You need to set e.g. SFI_AUTH='plc.princeton' in %s" % config_file)
           errors += 1 

        self.config_file=config_file
        if errors:
           sys.exit(1)

    def show_config (self):
        print "From configuration file %s"%self.config_file
        flags=[ 
            ('SFI_USER','user'),
            ('SFI_AUTH','authority'),
            ('SFI_SM','sm_url'),
            ('SFI_REGISTRY','reg_url'),
            ]
        for (external_name, internal_name) in flags:
            print "%s='%s'"%(external_name,getattr(self,internal_name))

    #
    # Get various credential and spec files
    #
    # Establishes limiting conventions
    #   - conflates MAs and SAs
    #   - assumes last token in slice name is unique
    #
    # Bootstraps credentials
    #   - bootstrap user credential from self-signed certificate
    #   - bootstrap authority credential from user credential
    #   - bootstrap slice credential from user credential
    #
    
    # init self-signed cert, user credentials and gid
    def bootstrap (self):
        client_bootstrap = SfaClientBootstrap (self.user, self.reg_url, self.options.sfi_dir,
                                               logger=self.logger)
        # if -k is provided, use this to initialize private key
        if self.options.user_private_key:
            client_bootstrap.init_private_key_if_missing (self.options.user_private_key)
        else:
            # trigger legacy compat code if needed 
            # the name has changed from just <leaf>.pkey to <hrn>.pkey
            if not os.path.isfile(client_bootstrap.private_key_filename()):
                self.logger.info ("private key not found, trying legacy name")
                try:
                    legacy_private_key = os.path.join (self.options.sfi_dir, "%s.pkey"%Xrn.unescape(get_leaf(self.user)))
                    self.logger.debug("legacy_private_key=%s"%legacy_private_key)
                    client_bootstrap.init_private_key_if_missing (legacy_private_key)
                    self.logger.info("Copied private key from legacy location %s"%legacy_private_key)
                except:
                    self.logger.log_exc("Can't find private key ")
                    sys.exit(1)
            
        # make it bootstrap
        client_bootstrap.bootstrap_my_gid()
        # extract what's needed
        self.private_key = client_bootstrap.private_key()
        self.my_credential_string = client_bootstrap.my_credential_string ()
        self.my_gid = client_bootstrap.my_gid ()
        self.client_bootstrap = client_bootstrap


    def my_authority_credential_string(self):
        if not self.authority:
            self.logger.critical("no authority specified. Use -a or set SF_AUTH")
            sys.exit(-1)
        return self.client_bootstrap.authority_credential_string (self.authority)

    def authority_credential_string(self, auth_hrn):
        return self.client_bootstrap.authority_credential_string (auth_hrn)

    def slice_credential_string(self, name):
        return self.client_bootstrap.slice_credential_string (name)

    #
    # Management of the servers
    # 

    def registry (self):
        # cache the result
        if not hasattr (self, 'registry_proxy'):
            self.logger.info("Contacting Registry at: %s"%self.reg_url)
            self.registry_proxy = SfaServerProxy(self.reg_url, self.private_key, self.my_gid, 
                                                 timeout=self.options.timeout, verbose=self.options.debug)  
        return self.registry_proxy

    def sliceapi (self):
        # cache the result
        if not hasattr (self, 'sliceapi_proxy'):
            # if the command exposes the --component option, figure it's hostname and connect at CM_PORT
            if hasattr(self.command_options,'component') and self.command_options.component:
                # resolve the hrn at the registry
                node_hrn = self.command_options.component
                records = self.registry().Resolve(node_hrn, self.my_credential_string)
                records = filter_records('node', records)
                if not records:
                    self.logger.warning("No such component:%r"% opts.component)
                record = records[0]
                cm_url = "http://%s:%d/"%(record['hostname'],CM_PORT)
                self.sliceapi_proxy=SfaServerProxy(cm_url, self.private_key, self.my_gid)
            else:
                # otherwise use what was provided as --sliceapi, or SFI_SM in the config
                if not self.sm_url.startswith('http://') or self.sm_url.startswith('https://'):
                    self.sm_url = 'http://' + self.sm_url
                self.logger.info("Contacting Slice Manager at: %s"%self.sm_url)
                self.sliceapi_proxy = SfaServerProxy(self.sm_url, self.private_key, self.my_gid, 
                                                     timeout=self.options.timeout, verbose=self.options.debug)  
        return self.sliceapi_proxy

    def get_cached_server_version(self, server):
        # check local cache first
        cache = None
        version = None 
        cache_file = os.path.join(self.options.sfi_dir,'sfi_cache.dat')
        cache_key = server.url + "-version"
        try:
            cache = Cache(cache_file)
        except IOError:
            cache = Cache()
            self.logger.info("Local cache not found at: %s" % cache_file)

        if cache:
            version = cache.get(cache_key)

        if not version: 
            result = server.GetVersion()
            version= ReturnValue.get_value(result)
            # cache version for 20 minutes
            cache.add(cache_key, version, ttl= 60*20)
            self.logger.info("Updating cache file %s" % cache_file)
            cache.save_to_file(cache_file)

        return version   
        
    ### resurrect this temporarily so we can support V1 aggregates for a while
    def server_supports_options_arg(self, server):
        """
        Returns true if server support the optional call_id arg, false otherwise. 
        """
        server_version = self.get_cached_server_version(server)
        result = False
        # xxx need to rewrite this 
        if int(server_version.get('geni_api')) >= 2:
            result = True
        return result

    def server_supports_call_id_arg(self, server):
        server_version = self.get_cached_server_version(server)
        result = False      
        if 'sfa' in server_version and 'code_tag' in server_version:
            code_tag = server_version['code_tag']
            code_tag_parts = code_tag.split("-")
            version_parts = code_tag_parts[0].split(".")
            major, minor = version_parts[0], version_parts[1]
            rev = code_tag_parts[1]
            if int(major) == 1 and minor == 0 and build >= 22:
                result = True
        return result                 

    ### ois = options if supported
    # to be used in something like serverproxy.Method (arg1, arg2, *self.ois(api_options))
    def ois (self, server, option_dict):
        if self.server_supports_options_arg (server): 
            return [option_dict]
        elif self.server_supports_call_id_arg (server):
            return [ unique_call_id () ]
        else: 
            return []

    ### cis = call_id if supported - like ois
    def cis (self, server):
        if self.server_supports_call_id_arg (server):
            return [ unique_call_id ]
        else:
            return []

    ######################################## miscell utilities
    def get_rspec_file(self, rspec):
       if (os.path.isabs(rspec)):
          file = rspec
       else:
          file = os.path.join(self.options.sfi_dir, rspec)
       if (os.path.isfile(file)):
          return file
       else:
          self.logger.critical("No such rspec file %s"%rspec)
          sys.exit(1)
    
    def get_record_file(self, record):
       if (os.path.isabs(record)):
          file = record
       else:
          file = os.path.join(self.options.sfi_dir, record)
       if (os.path.isfile(file)):
          return file
       else:
          self.logger.critical("No such registry record file %s"%record)
          sys.exit(1)


    #==========================================================================
    # Following functions implement the commands
    #
    # Registry-related commands
    #==========================================================================

    def version(self, options, args):
        """
        display an SFA server version (GetVersion)
or version information about sfi itself
        """
        if options.version_local:
            version=version_core()
        else:
            if options.version_registry:
                server=self.registry()
            else:
                server = self.sliceapi()
            result = server.GetVersion()
            version = ReturnValue.get_value(result)
        if self.options.raw:
            save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
        else:
            pprinter = PrettyPrinter(indent=4)
            pprinter.pprint(version)

    def list(self, options, args):
        """
        list entries in named authority registry (List)
        """
        if len(args)!= 1:
            self.print_help()
            sys.exit(1)
        hrn = args[0]
        opts = {}
        if options.recursive:
            opts['recursive'] = options.recursive
        
        if options.show_credential:
            show_credentials(self.my_credential_string)
        try:
            list = self.registry().List(hrn, self.my_credential_string, options)
        except IndexError:
            raise Exception, "Not enough parameters for the 'list' command"

        # filter on person, slice, site, node, etc.
        # This really should be in the self.filter_records funct def comment...
        list = filter_records(options.type, list)
        terminal_render (list, options)
        if options.file:
            save_records_to_file(options.file, list, options.fileformat)
        return
    
    def show(self, options, args):
        """
        show details about named registry record (Resolve)
        """
        if len(args)!= 1:
            self.print_help()
            sys.exit(1)
        hrn = args[0]
        # explicitly require Resolve to run in details mode
        record_dicts = self.registry().Resolve(hrn, self.my_credential_string, {'details':True})
        record_dicts = filter_records(options.type, record_dicts)
        if not record_dicts:
            self.logger.error("No record of type %s"% options.type)
            return
        # user has required to focus on some keys
        if options.keys:
            def project (record):
                projected={}
                for key in options.keys:
                    try: projected[key]=record[key]
                    except: pass
                return projected
            record_dicts = [ project (record) for record in record_dicts ]
        records = [ Record(dict=record_dict) for record_dict in record_dicts ]
        for record in records:
            if (options.format == "text"):      record.dump(sort=True)  
            else:                               print record.save_as_xml() 
        if options.file:
            save_records_to_file(options.file, record_dicts, options.fileformat)
        return
    
    def add(self, options, args):
        "add record into registry from xml file (Register)"
        auth_cred = self.my_authority_credential_string()
        if options.show_credential:
            show_credentials(auth_cred)
        record_dict = {}
        if len(args) > 0:
            record_filepath = args[0]
            rec_file = self.get_record_file(record_filepath)
            record_dict.update(load_record_from_file(rec_file).todict())
        if options:
            record_dict.update(load_record_from_opts(options).todict())
        # we should have a type by now
        if 'type' not in record_dict :
            self.print_help()
            sys.exit(1)
        # this is still planetlab dependent.. as plc will whine without that
        # also, it's only for adding
        if record_dict['type'] == 'user':
            if not 'first_name' in record_dict:
                record_dict['first_name'] = record_dict['hrn']
            if 'last_name' not in record_dict:
                record_dict['last_name'] = record_dict['hrn'] 
        return self.registry().Register(record_dict, auth_cred)
    
    def update(self, options, args):
        "update record into registry from xml file (Update)"
        record_dict = {}
        if len(args) > 0:
            record_filepath = args[0]
            rec_file = self.get_record_file(record_filepath)
            record_dict.update(load_record_from_file(rec_file).todict())
        if options:
            record_dict.update(load_record_from_opts(options).todict())
        # at the very least we need 'type' here
        if 'type' not in record_dict:
            self.print_help()
            sys.exit(1)

        # don't translate into an object, as this would possibly distort
        # user-provided data; e.g. add an 'email' field to Users
        if record_dict['type'] == "user":
            if record_dict['hrn'] == self.user:
                cred = self.my_credential_string<commit_msg>correcting sfi add/update usage and help<commit_after>#
# sfi.py - basic SFA command-line client
# this module is also used in sfascan
#

import sys
sys.path.append('.')

import os, os.path
import socket
import re
import datetime
import codecs
import pickle
import json
import shutil
from lxml import etree
from StringIO import StringIO
from optparse import OptionParser
from pprint import PrettyPrinter
from tempfile import mkstemp

from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.gid import GID
from sfa.trust.credential import Credential
from sfa.trust.sfaticket import SfaTicket

from sfa.util.faults import SfaInvalidArgument
from sfa.util.sfalogging import sfi_logger
from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, Xrn
from sfa.util.config import Config
from sfa.util.version import version_core
from sfa.util.cache import Cache

from sfa.storage.record import Record

from sfa.rspecs.rspec import RSpec
from sfa.rspecs.rspec_converter import RSpecConverter
from sfa.rspecs.version_manager import VersionManager

from sfa.client.sfaclientlib import SfaClientBootstrap
from sfa.client.sfaserverproxy import SfaServerProxy, ServerException
from sfa.client.client_helper import pg_users_arg, sfa_users_arg
from sfa.client.return_value import ReturnValue
from sfa.client.candidates import Candidates

CM_PORT=12346

# utility methods here
def optparse_listvalue_callback(option, option_string, value, parser):
    setattr(parser.values, option.dest, value.split(','))

# a code fragment that could be helpful for argparse which unfortunately is 
# available with 2.7 only, so this feels like too strong a requirement for the client side
#class ExtraArgAction  (argparse.Action):
#    def __call__ (self, parser, namespace, values, option_string=None):
# would need a try/except of course
#        (k,v)=values.split('=')
#        d=getattr(namespace,self.dest)
#        d[k]=v
#####
#parser.add_argument ("-X","--extra",dest='extras', default={}, action=ExtraArgAction,
#                     help="set extra flags, testbed dependent, e.g. --extra enabled=true")
    
def optparse_dictvalue_callback (option, option_string, value, parser):
    try:
        (k,v)=value.split('=',1)
        d=getattr(parser.values, option.dest)
        d[k]=v
    except:
        parser.print_help()
        sys.exit(1)

# display methods
def display_rspec(rspec, format='rspec'):
    if format in ['dns']:
        tree = etree.parse(StringIO(rspec))
        root = tree.getroot()
        result = root.xpath("./network/site/node/hostname/text()")
    elif format in ['ip']:
        # The IP address is not yet part of the new RSpec
        # so this doesn't do anything yet.
        tree = etree.parse(StringIO(rspec))
        root = tree.getroot()
        result = root.xpath("./network/site/node/ipv4/text()")
    else:
        result = rspec

    print result
    return

def display_list(results):
    for result in results:
        print result

def display_records(recordList, dump=False):
    ''' Print all fields in the record'''
    for record in recordList:
        display_record(record, dump)

def display_record(record, dump=False):
    if dump:
        record.dump(sort=True)
    else:
        info = record.getdict()
        print "%s (%s)" % (info['hrn'], info['type'])
    return


def filter_records(type, records):
    filtered_records = []
    for record in records:
        if (record['type'] == type) or (type == "all"):
            filtered_records.append(record)
    return filtered_records


def credential_printable (credential_string):
    credential=Credential(string=credential_string)
    result=""
    result += credential.get_summary_tostring()
    result += "\n"
    rights = credential.get_privileges()
    result += "rights=%s"%rights
    result += "\n"
    return result

def show_credentials (cred_s):
    if not isinstance (cred_s,list): cred_s = [cred_s]
    for cred in cred_s:
        print "Using Credential %s"%credential_printable(cred)

# save methods
def save_raw_to_file(var, filename, format="text", banner=None):
    if filename == "-":
        # if filename is "-", send it to stdout
        f = sys.stdout
    else:
        f = open(filename, "w")
    if banner:
        f.write(banner+"\n")
    if format == "text":
        f.write(str(var))
    elif format == "pickled":
        f.write(pickle.dumps(var))
    elif format == "json":
        if hasattr(json, "dumps"):
            f.write(json.dumps(var))   # python 2.6
        else:
            f.write(json.write(var))   # python 2.5
    else:
        # this should never happen
        print "unknown output format", format
    if banner:
        f.write('\n'+banner+"\n")

def save_rspec_to_file(rspec, filename):
    if not filename.endswith(".rspec"):
        filename = filename + ".rspec"
    f = open(filename, 'w')
    f.write(rspec)
    f.close()
    return

def save_records_to_file(filename, record_dicts, format="xml"):
    if format == "xml":
        index = 0
        for record_dict in record_dicts:
            if index > 0:
                save_record_to_file(filename + "." + str(index), record_dict)
            else:
                save_record_to_file(filename, record_dict)
            index = index + 1
    elif format == "xmllist":
        f = open(filename, "w")
        f.write("<recordlist>\n")
        for record_dict in record_dicts:
            record_obj=Record(dict=record_dict)
            f.write('<record hrn="' + record_obj.hrn + '" type="' + record_obj.type + '" />\n')
        f.write("</recordlist>\n")
        f.close()
    elif format == "hrnlist":
        f = open(filename, "w")
        for record_dict in record_dicts:
            record_obj=Record(dict=record_dict)
            f.write(record_obj.hrn + "\n")
        f.close()
    else:
        # this should never happen
        print "unknown output format", format

def save_record_to_file(filename, record_dict):
    record = Record(dict=record_dict)
    xml = record.save_as_xml()
    f=codecs.open(filename, encoding='utf-8',mode="w")
    f.write(xml)
    f.close()
    return

# used in sfi list
def terminal_render (records,options):
    # sort records by type
    grouped_by_type={}
    for record in records:
        type=record['type']
        if type not in grouped_by_type: grouped_by_type[type]=[]
        grouped_by_type[type].append(record)
    group_types=grouped_by_type.keys()
    group_types.sort()
    for type in group_types:
        group=grouped_by_type[type]
#        print 20 * '-', type
        try:    renderer=eval('terminal_render_'+type)
        except: renderer=terminal_render_default
        for record in group: renderer(record,options)

def render_plural (how_many, name,names=None):
    if not names: names="%ss"%name
    if how_many<=0: return "No %s"%name
    elif how_many==1: return "1 %s"%name
    else: return "%d %s"%(how_many,names)

def terminal_render_default (record,options):
    print "%s (%s)" % (record['hrn'], record['type'])
def terminal_render_user (record, options):
    print "%s (User)"%record['hrn'],
    if record.get('reg-pi-authorities',None): print " [PI at %s]"%(" and ".join(record['reg-pi-authorities'])),
    if record.get('reg-slices',None): print " [IN slices %s]"%(" and ".join(record['reg-slices'])),
    user_keys=record.get('reg-keys',[])
    if not options.verbose:
        print " [has %s]"%(render_plural(len(user_keys),"key"))
    else:
        print ""
        for key in user_keys: print 8*' ',key.strip("\n")
        
def terminal_render_slice (record, options):
    print "%s (Slice)"%record['hrn'],
    if record.get('reg-researchers',None): print " [USERS %s]"%(" and ".join(record['reg-researchers'])),
#    print record.keys()
    print ""
def terminal_render_authority (record, options):
    print "%s (Authority)"%record['hrn'],
    if record.get('reg-pis',None): print " [PIS %s]"%(" and ".join(record['reg-pis'])),
    print ""
def terminal_render_node (record, options):
    print "%s (Node)"%record['hrn']

# minimally check a key argument
def check_ssh_key (key):
    good_ssh_key = r'^.*(?:ssh-dss|ssh-rsa)[ ]+[A-Za-z0-9+/=]+(?: .*)?$'
    return re.match(good_ssh_key, key, re.IGNORECASE)

# load methods
def load_record_from_opts(options):
    record_dict = {}
    if hasattr(options, 'xrn') and options.xrn:
        if hasattr(options, 'type') and options.type:
            xrn = Xrn(options.xrn, options.type)
        else:
            xrn = Xrn(options.xrn)
        record_dict['urn'] = xrn.get_urn()
        record_dict['hrn'] = xrn.get_hrn()
        record_dict['type'] = xrn.get_type()
    if hasattr(options, 'key') and options.key:
        try:
            pubkey = open(options.key, 'r').read()
        except IOError:
            pubkey = options.key
        if not check_ssh_key (pubkey):
            raise SfaInvalidArgument(name='key',msg="Could not find file, or wrong key format")
        record_dict['keys'] = [pubkey]
    if hasattr(options, 'slices') and options.slices:
        record_dict['slices'] = options.slices
    if hasattr(options, 'researchers') and options.researchers:
        record_dict['researcher'] = options.researchers
    if hasattr(options, 'email') and options.email:
        record_dict['email'] = options.email
    if hasattr(options, 'pis') and options.pis:
        record_dict['pi'] = options.pis

    # handle extra settings
    record_dict.update(options.extras)
    
    return Record(dict=record_dict)

def load_record_from_file(filename):
    f=codecs.open(filename, encoding="utf-8", mode="r")
    xml_string = f.read()
    f.close()
    return Record(xml=xml_string)


import uuid
def unique_call_id(): return uuid.uuid4().urn

class Sfi:
    
    # dirty hack to make this class usable from the outside
    required_options=['verbose',  'debug',  'registry',  'sm',  'auth',  'user', 'user_private_key']

    @staticmethod
    def default_sfi_dir ():
        if os.path.isfile("./sfi_config"): 
            return os.getcwd()
        else:
            return os.path.expanduser("~/.sfi/")

    # dummy to meet Sfi's expectations for its 'options' field
    # i.e. s/t we can do setattr on
    class DummyOptions:
        pass

    def __init__ (self,options=None):
        if options is None: options=Sfi.DummyOptions()
        for opt in Sfi.required_options:
            if not hasattr(options,opt): setattr(options,opt,None)
        if not hasattr(options,'sfi_dir'): options.sfi_dir=Sfi.default_sfi_dir()
        self.options = options
        self.user = None
        self.authority = None
        self.logger = sfi_logger
        self.logger.enable_console()
        self.available_names = [ tuple[0] for tuple in Sfi.available ]
        self.available_dict = dict (Sfi.available)
   
    # tuples command-name expected-args in the order in which they should appear in the help
    available = [ 
        ("version", ""),  
        ("list", "authority"),
        ("show", "name"),
        ("add", "[record]"),
        ("update", "[record]"),
        ("remove", "name"),
        ("slices", ""),
        ("resources", "[slice_hrn]"),
        ("create", "slice_hrn rspec"),
        ("delete", "slice_hrn"),
        ("status", "slice_hrn"),
        ("start", "slice_hrn"),
        ("stop", "slice_hrn"),
        ("reset", "slice_hrn"),
        ("renew", "slice_hrn time"),
        ("shutdown", "slice_hrn"),
        ("get_ticket", "slice_hrn rspec"),
        ("redeem_ticket", "ticket"),
        ("delegate", "to_hrn"),
        ("gid", "[name]"),
        ("trusted", "cred"),
        ("config", ""),
        ]

    def print_command_help (self, options):
        verbose=getattr(options,'verbose')
        format3="%18s %-15s %s"
        line=80*'-'
        if not verbose:
            print format3%("command","cmd_args","description")
            print line
        else:
            print line
            self.create_parser().print_help()
        for command in self.available_names:
            args=self.available_dict[command]
            method=getattr(self,command,None)
            doc=""
            if method: doc=getattr(method,'__doc__',"")
            if not doc: doc="*** no doc found ***"
            doc=doc.strip(" \t\n")
            doc=doc.replace("\n","\n"+35*' ')
            if verbose:
                print line
            print format3%(command,args,doc)
            if verbose:
                self.create_command_parser(command).print_help()

    def create_command_parser(self, command):
        if command not in self.available_dict:
            msg="Invalid command\n"
            msg+="Commands: "
            msg += ','.join(self.available_names)            
            self.logger.critical(msg)
            sys.exit(2)

        parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \
                                     % (command, self.available_dict[command]))

        if command in ("add", "update"):
            parser.add_option('-x', '--xrn', dest='xrn', metavar='<xrn>', help='object hrn/urn (mandatory)')
            parser.add_option('-t', '--type', dest='type', metavar='<type>', help='object type', default=None)
            parser.add_option('-e', '--email', dest='email', default="",  help="email (mandatory for users)") 
# use --extra instead
#            parser.add_option('-u', '--url', dest='url', metavar='<url>', default=None, help="URL, useful for slices") 
#            parser.add_option('-d', '--description', dest='description', metavar='<description>', 
#                              help='Description, useful for slices', default=None)
            parser.add_option('-k', '--key', dest='key', metavar='<key>', help='public key string or file', 
                              default=None)
            parser.add_option('-s', '--slices', dest='slices', metavar='<slices>', help='slice xrns',
                              default='', type="str", action='callback', callback=optparse_listvalue_callback)
            parser.add_option('-r', '--researchers', dest='researchers', metavar='<researchers>', 
                              help='slice researchers', default='', type="str", action='callback', 
                              callback=optparse_listvalue_callback)
            parser.add_option('-p', '--pis', dest='pis', metavar='<PIs>', help='Principal Investigators/Project Managers',
                              default='', type="str", action='callback', callback=optparse_listvalue_callback)
# use --extra instead
#            parser.add_option('-f', '--firstname', dest='firstname', metavar='<firstname>', help='user first name')
#            parser.add_option('-l', '--lastname', dest='lastname', metavar='<lastname>', help='user last name')
            parser.add_option ('-X','--extra',dest='extras',default={},type='str',metavar="<EXTRA_ASSIGNS>",
                               action="callback", callback=optparse_dictvalue_callback, nargs=1,
                               help="set extra/testbed-dependent flags, e.g. --extra enabled=true")

        # show_credential option
        if command in ("list","resources","create","add","update","remove","slices","delete","status","renew"):
            parser.add_option("-C","--credential",dest='show_credential',action='store_true',default=False,
                              help="show credential(s) used in human-readable form")
        # registy filter option
        if command in ("list", "show", "remove"):
            parser.add_option("-t", "--type", dest="type", type="choice",
                            help="type filter ([all]|user|slice|authority|node|aggregate)",
                            choices=("all", "user", "slice", "authority", "node", "aggregate"),
                            default="all")
        if command in ("show"):
            parser.add_option("-k","--key",dest="keys",action="append",default=[],
                              help="specify specific keys to be displayed from record")
        if command in ("resources"):
            # rspec version
            parser.add_option("-r", "--rspec-version", dest="rspec_version", default="SFA 1",
                              help="schema type and version of resulting RSpec")
            # disable/enable cached rspecs
            parser.add_option("-c", "--current", dest="current", default=False,
                              action="store_true",  
                              help="Request the current rspec bypassing the cache. Cached rspecs are returned by default")
            # display formats
            parser.add_option("-f", "--format", dest="format", type="choice",
                             help="display format ([xml]|dns|ip)", default="xml",
                             choices=("xml", "dns", "ip"))
            #panos: a new option to define the type of information about resources a user is interested in
            parser.add_option("-i", "--info", dest="info",
                                help="optional component information", default=None)
            # a new option to retreive or not reservation-oriented RSpecs (leases)
            parser.add_option("-l", "--list_leases", dest="list_leases", type="choice",
                                help="Retreive or not reservation-oriented RSpecs ([resources]|leases|all )",
                                choices=("all", "resources", "leases"), default="resources")


        # 'create' does return the new rspec, makes sense to save that too
        if command in ("resources", "show", "list", "gid", 'create'):
           parser.add_option("-o", "--output", dest="file",
                            help="output XML to file", metavar="FILE", default=None)

        if command in ("show", "list"):
           parser.add_option("-f", "--format", dest="format", type="choice",
                             help="display format ([text]|xml)", default="text",
                             choices=("text", "xml"))

           parser.add_option("-F", "--fileformat", dest="fileformat", type="choice",
                             help="output file format ([xml]|xmllist|hrnlist)", default="xml",
                             choices=("xml", "xmllist", "hrnlist"))
        if command == 'list':
           parser.add_option("-r", "--recursive", dest="recursive", action='store_true',
                             help="list all child records", default=False)
           parser.add_option("-v", "--verbose", dest="verbose", action='store_true',
                             help="gives details, like user keys", default=False)
        if command in ("delegate"):
           parser.add_option("-u", "--user",
                             action="store_true", dest="delegate_user", default=False,
                             help="delegate your own credentials; default if no other option is provided")
           parser.add_option("-s", "--slice", dest="delegate_slices",action='append',default=[],
                             metavar="slice_hrn", help="delegate cred. for slice HRN")
           parser.add_option("-a", "--auths", dest='delegate_auths',action='append',default=[],
                             metavar='auth_hrn', help="delegate cred for auth HRN")
           # this primarily is a shorthand for -a my_hrn^
           parser.add_option("-p", "--pi", dest='delegate_pi', default=None, action='store_true',
                             help="delegate your PI credentials, so s.t. like -a your_hrn^")
           parser.add_option("-A","--to-authority",dest='delegate_to_authority',action='store_true',default=False,
                             help="""by default the mandatory argument is expected to be a user, 
use this if you mean an authority instead""")
        
        if command in ("version"):
            parser.add_option("-R","--registry-version",
                              action="store_true", dest="version_registry", default=False,
                              help="probe registry version instead of sliceapi")
            parser.add_option("-l","--local",
                              action="store_true", dest="version_local", default=False,
                              help="display version of the local client")

        return parser

        
    def create_parser(self):

        # Generate command line parser
        parser = OptionParser(usage="sfi [sfi_options] command [cmd_options] [cmd_args]",
                             description="Commands: %s"%(" ".join(self.available_names)))
        parser.add_option("-r", "--registry", dest="registry",
                         help="root registry", metavar="URL", default=None)
        parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL",
                         help="slice API - in general a SM URL, but can be used to talk to an aggregate")
        parser.add_option("-R", "--raw", dest="raw", default=None,
                          help="Save raw, unparsed server response to a file")
        parser.add_option("", "--rawformat", dest="rawformat", type="choice",
                          help="raw file format ([text]|pickled|json)", default="text",
                          choices=("text","pickled","json"))
        parser.add_option("", "--rawbanner", dest="rawbanner", default=None,
                          help="text string to write before and after raw output")
        parser.add_option("-d", "--dir", dest="sfi_dir",
                         help="config & working directory - default is %default",
                         metavar="PATH", default=Sfi.default_sfi_dir())
        parser.add_option("-u", "--user", dest="user",
                         help="user name", metavar="HRN", default=None)
        parser.add_option("-a", "--auth", dest="auth",
                         help="authority name", metavar="HRN", default=None)
        parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0,
                         help="verbose mode - cumulative")
        parser.add_option("-D", "--debug",
                          action="store_true", dest="debug", default=False,
                          help="Debug (xml-rpc) protocol messages")
        # would it make sense to use ~/.ssh/id_rsa as a default here ?
        parser.add_option("-k", "--private-key",
                         action="store", dest="user_private_key", default=None,
                         help="point to the private key file to use if not yet installed in sfi_dir")
        parser.add_option("-t", "--timeout", dest="timeout", default=None,
                         help="Amout of time to wait before timing out the request")
        parser.add_option("-?", "--commands", 
                         action="store_true", dest="command_help", default=False,
                         help="one page summary on commands & exit")
        parser.disable_interspersed_args()

        return parser
        

    def print_help (self):
        print "==================== Generic sfi usage"
        self.sfi_parser.print_help()
        print "==================== Specific command usage"
        self.command_parser.print_help()

    #
    # Main: parse arguments and dispatch to command
    #
    def dispatch(self, command, command_options, command_args):
        method=getattr(self, command,None)
        if not method:
            print "Unknown command %s"%command
            return
        return method(command_options, command_args)

    def main(self):
        self.sfi_parser = self.create_parser()
        (options, args) = self.sfi_parser.parse_args()
        if options.command_help: 
            self.print_command_help(options)
            sys.exit(1)
        self.options = options

        self.logger.setLevelFromOptVerbose(self.options.verbose)

        if len(args) <= 0:
            self.logger.critical("No command given. Use -h for help.")
            self.print_command_help(options)
            return -1
    
        # complete / find unique match with command set
        command_candidates = Candidates (self.available_names)
        input = args[0]
        command = command_candidates.only_match(input)
        if not command:
            self.print_command_help(options)
            sys.exit(1)
        # second pass options parsing
        self.command_parser = self.create_command_parser(command)
        (command_options, command_args) = self.command_parser.parse_args(args[1:])
        self.command_options = command_options

        self.read_config () 
        self.bootstrap ()
        self.logger.debug("Command=%s" % command)

        try:
            self.dispatch(command, command_options, command_args)
        except:
            self.logger.log_exc ("sfi command %s failed"%command)
            sys.exit(1)

        return
    
    ####################
    def read_config(self):
        config_file = os.path.join(self.options.sfi_dir,"sfi_config")
        shell_config_file  = os.path.join(self.options.sfi_dir,"sfi_config.sh")
        try:
            if Config.is_ini(config_file):
                config = Config (config_file)
            else:
                # try upgrading from shell config format
                fp, fn = mkstemp(suffix='sfi_config', text=True)  
                config = Config(fn)
                # we need to preload the sections we want parsed 
                # from the shell config
                config.add_section('sfi')
                config.add_section('sface')
                config.load(config_file)
                # back up old config
                shutil.move(config_file, shell_config_file)
                # write new config
                config.save(config_file)
                 
        except:
            self.logger.critical("Failed to read configuration file %s"%config_file)
            self.logger.info("Make sure to remove the export clauses and to add quotes")
            if self.options.verbose==0:
                self.logger.info("Re-run with -v for more details")
            else:
                self.logger.log_exc("Could not read config file %s"%config_file)
            sys.exit(1)
     
        errors = 0
        # Set SliceMgr URL
        if (self.options.sm is not None):
           self.sm_url = self.options.sm
        elif hasattr(config, "SFI_SM"):
           self.sm_url = config.SFI_SM
        else:
           self.logger.error("You need to set e.g. SFI_SM='http://your.slicemanager.url:12347/' in %s" % config_file)
           errors += 1 

        # Set Registry URL
        if (self.options.registry is not None):
           self.reg_url = self.options.registry
        elif hasattr(config, "SFI_REGISTRY"):
           self.reg_url = config.SFI_REGISTRY
        else:
           self.logger.error("You need to set e.g. SFI_REGISTRY='http://your.registry.url:12345/' in %s" % config_file)
           errors += 1 

        # Set user HRN
        if (self.options.user is not None):
           self.user = self.options.user
        elif hasattr(config, "SFI_USER"):
           self.user = config.SFI_USER
        else:
           self.logger.error("You need to set e.g. SFI_USER='plc.princeton.username' in %s" % config_file)
           errors += 1 

        # Set authority HRN
        if (self.options.auth is not None):
           self.authority = self.options.auth
        elif hasattr(config, "SFI_AUTH"):
           self.authority = config.SFI_AUTH
        else:
           self.logger.error("You need to set e.g. SFI_AUTH='plc.princeton' in %s" % config_file)
           errors += 1 

        self.config_file=config_file
        if errors:
           sys.exit(1)

    def show_config (self):
        print "From configuration file %s"%self.config_file
        flags=[ 
            ('SFI_USER','user'),
            ('SFI_AUTH','authority'),
            ('SFI_SM','sm_url'),
            ('SFI_REGISTRY','reg_url'),
            ]
        for (external_name, internal_name) in flags:
            print "%s='%s'"%(external_name,getattr(self,internal_name))

    #
    # Get various credential and spec files
    #
    # Establishes limiting conventions
    #   - conflates MAs and SAs
    #   - assumes last token in slice name is unique
    #
    # Bootstraps credentials
    #   - bootstrap user credential from self-signed certificate
    #   - bootstrap authority credential from user credential
    #   - bootstrap slice credential from user credential
    #
    
    # init self-signed cert, user credentials and gid
    def bootstrap (self):
        client_bootstrap = SfaClientBootstrap (self.user, self.reg_url, self.options.sfi_dir,
                                               logger=self.logger)
        # if -k is provided, use this to initialize private key
        if self.options.user_private_key:
            client_bootstrap.init_private_key_if_missing (self.options.user_private_key)
        else:
            # trigger legacy compat code if needed 
            # the name has changed from just <leaf>.pkey to <hrn>.pkey
            if not os.path.isfile(client_bootstrap.private_key_filename()):
                self.logger.info ("private key not found, trying legacy name")
                try:
                    legacy_private_key = os.path.join (self.options.sfi_dir, "%s.pkey"%Xrn.unescape(get_leaf(self.user)))
                    self.logger.debug("legacy_private_key=%s"%legacy_private_key)
                    client_bootstrap.init_private_key_if_missing (legacy_private_key)
                    self.logger.info("Copied private key from legacy location %s"%legacy_private_key)
                except:
                    self.logger.log_exc("Can't find private key ")
                    sys.exit(1)
            
        # make it bootstrap
        client_bootstrap.bootstrap_my_gid()
        # extract what's needed
        self.private_key = client_bootstrap.private_key()
        self.my_credential_string = client_bootstrap.my_credential_string ()
        self.my_gid = client_bootstrap.my_gid ()
        self.client_bootstrap = client_bootstrap


    def my_authority_credential_string(self):
        if not self.authority:
            self.logger.critical("no authority specified. Use -a or set SF_AUTH")
            sys.exit(-1)
        return self.client_bootstrap.authority_credential_string (self.authority)

    def authority_credential_string(self, auth_hrn):
        return self.client_bootstrap.authority_credential_string (auth_hrn)

    def slice_credential_string(self, name):
        return self.client_bootstrap.slice_credential_string (name)

    #
    # Management of the servers
    # 

    def registry (self):
        # cache the result
        if not hasattr (self, 'registry_proxy'):
            self.logger.info("Contacting Registry at: %s"%self.reg_url)
            self.registry_proxy = SfaServerProxy(self.reg_url, self.private_key, self.my_gid, 
                                                 timeout=self.options.timeout, verbose=self.options.debug)  
        return self.registry_proxy

    def sliceapi (self):
        # cache the result
        if not hasattr (self, 'sliceapi_proxy'):
            # if the command exposes the --component option, figure it's hostname and connect at CM_PORT
            if hasattr(self.command_options,'component') and self.command_options.component:
                # resolve the hrn at the registry
                node_hrn = self.command_options.component
                records = self.registry().Resolve(node_hrn, self.my_credential_string)
                records = filter_records('node', records)
                if not records:
                    self.logger.warning("No such component:%r"% opts.component)
                record = records[0]
                cm_url = "http://%s:%d/"%(record['hostname'],CM_PORT)
                self.sliceapi_proxy=SfaServerProxy(cm_url, self.private_key, self.my_gid)
            else:
                # otherwise use what was provided as --sliceapi, or SFI_SM in the config
                if not self.sm_url.startswith('http://') or self.sm_url.startswith('https://'):
                    self.sm_url = 'http://' + self.sm_url
                self.logger.info("Contacting Slice Manager at: %s"%self.sm_url)
                self.sliceapi_proxy = SfaServerProxy(self.sm_url, self.private_key, self.my_gid, 
                                                     timeout=self.options.timeout, verbose=self.options.debug)  
        return self.sliceapi_proxy

    def get_cached_server_version(self, server):
        # check local cache first
        cache = None
        version = None 
        cache_file = os.path.join(self.options.sfi_dir,'sfi_cache.dat')
        cache_key = server.url + "-version"
        try:
            cache = Cache(cache_file)
        except IOError:
            cache = Cache()
            self.logger.info("Local cache not found at: %s" % cache_file)

        if cache:
            version = cache.get(cache_key)

        if not version: 
            result = server.GetVersion()
            version= ReturnValue.get_value(result)
            # cache version for 20 minutes
            cache.add(cache_key, version, ttl= 60*20)
            self.logger.info("Updating cache file %s" % cache_file)
            cache.save_to_file(cache_file)

        return version   
        
    ### resurrect this temporarily so we can support V1 aggregates for a while
    def server_supports_options_arg(self, server):
        """
        Returns true if server support the optional call_id arg, false otherwise. 
        """
        server_version = self.get_cached_server_version(server)
        result = False
        # xxx need to rewrite this 
        if int(server_version.get('geni_api')) >= 2:
            result = True
        return result

    def server_supports_call_id_arg(self, server):
        server_version = self.get_cached_server_version(server)
        result = False      
        if 'sfa' in server_version and 'code_tag' in server_version:
            code_tag = server_version['code_tag']
            code_tag_parts = code_tag.split("-")
            version_parts = code_tag_parts[0].split(".")
            major, minor = version_parts[0], version_parts[1]
            rev = code_tag_parts[1]
            if int(major) == 1 and minor == 0 and build >= 22:
                result = True
        return result                 

    ### ois = options if supported
    # to be used in something like serverproxy.Method (arg1, arg2, *self.ois(api_options))
    def ois (self, server, option_dict):
        if self.server_supports_options_arg (server): 
            return [option_dict]
        elif self.server_supports_call_id_arg (server):
            return [ unique_call_id () ]
        else: 
            return []

    ### cis = call_id if supported - like ois
    def cis (self, server):
        if self.server_supports_call_id_arg (server):
            return [ unique_call_id ]
        else:
            return []

    ######################################## miscell utilities
    def get_rspec_file(self, rspec):
       if (os.path.isabs(rspec)):
          file = rspec
       else:
          file = os.path.join(self.options.sfi_dir, rspec)
       if (os.path.isfile(file)):
          return file
       else:
          self.logger.critical("No such rspec file %s"%rspec)
          sys.exit(1)
    
    def get_record_file(self, record):
       if (os.path.isabs(record)):
          file = record
       else:
          file = os.path.join(self.options.sfi_dir, record)
       if (os.path.isfile(file)):
          return file
       else:
          self.logger.critical("No such registry record file %s"%record)
          sys.exit(1)


    #==========================================================================
    # Following functions implement the commands
    #
    # Registry-related commands
    #==========================================================================

    def version(self, options, args):
        """
        display an SFA server version (GetVersion)
or version information about sfi itself
        """
        if options.version_local:
            version=version_core()
        else:
            if options.version_registry:
                server=self.registry()
            else:
                server = self.sliceapi()
            result = server.GetVersion()
            version = ReturnValue.get_value(result)
        if self.options.raw:
            save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
        else:
            pprinter = PrettyPrinter(indent=4)
            pprinter.pprint(version)

    def list(self, options, args):
        """
        list entries in named authority registry (List)
        """
        if len(args)!= 1:
            self.print_help()
            sys.exit(1)
        hrn = args[0]
        opts = {}
        if options.recursive:
            opts['recursive'] = options.recursive
        
        if options.show_credential:
            show_credentials(self.my_credential_string)
        try:
            list = self.registry().List(hrn, self.my_credential_string, options)
        except IndexError:
            raise Exception, "Not enough parameters for the 'list' command"

        # filter on person, slice, site, node, etc.
        # This really should be in the self.filter_records funct def comment...
        list = filter_records(options.type, list)
        terminal_render (list, options)
        if options.file:
            save_records_to_file(options.file, list, options.fileformat)
        return
    
    def show(self, options, args):
        """
        show details about named registry record (Resolve)
        """
        if len(args)!= 1:
            self.print_help()
            sys.exit(1)
        hrn = args[0]
        # explicitly require Resolve to run in details mode
        record_dicts = self.registry().Resolve(hrn, self.my_credential_string, {'details':True})
        record_dicts = filter_records(options.type, record_dicts)
        if not record_dicts:
            self.logger.error("No record of type %s"% options.type)
            return
        # user has required to focus on some keys
        if options.keys:
            def project (record):
                projected={}
                for key in options.keys:
                    try: projected[key]=record[key]
                    except: pass
                return projected
            record_dicts = [ project (record) for record in record_dicts ]
        records = [ Record(dict=record_dict) for record_dict in record_dicts ]
        for record in records:
            if (options.format == "text"):      record.dump(sort=True)  
            else:                               print record.save_as_xml() 
        if options.file:
            save_records_to_file(options.file, record_dicts, options.fileformat)
        return
    
    def add(self, options, args):
        "add record into registry by using the command options (Recommended) or from xml file (Register)"
        auth_cred = self.my_authority_credential_string()
        if options.show_credential:
            show_credentials(auth_cred)
        record_dict = {}
        if len(args) > 0:
            record_filepath = args[0]
            rec_file = self.get_record_file(record_filepath)
            record_dict.update(load_record_from_file(rec_file).todict())
        if options:
            record_dict.update(load_record_from_opts(options).todict())
        # we should have a type by now
        if 'type' not in record_dict :
            self.print_help()
            sys.exit(1)
        # this is still planetlab dependent.. as plc will whine without that
        # also, it's only for adding
        if record_dict['type'] == 'user':
            if not 'first_name' in record_dict:
                record_dict['first_name'] = record_dict['hrn']
            if 'last_name' not in record_dict:
                record_dict['last_name'] = record_dict['hrn'] 
        return self.registry().Register(record_dict, auth_cred)
    
    def update(self, options, args):
        "update record into registry by using the command options (Recommended) or from xml file (Update)"
        record_dict = {}
        if len(args) > 0:
            record_filepath = args[0]
            rec_file = self.get_record_file(record_filepath)
            record_dict.update(load_record_from_file(rec_file).todict())
        if options:
            record_dict.update(load_record_from_opts(options).todict())
        # at the very least we need 'type' here
        if 'type' not in record_dict:
            self.print_help()
            sys.exit(1)

        # don't translate into an object, as this would possibly distort
        # user-provided data; e.g. add an 'email' field to Users
        if record_dict['type'] == "user":
            if record_dict['hrn'] == self.user:
                cred = self.my_credential_string
{'fraction_non_alphanumeric': 0.12541541480164325, 'fraction_numerical': 0.002575265190998835, 'mean_word_length': 2.2260948688531075, 'pattern_counts': {'":': 16, '<': 39, '<?xml version=': 0, '>': 45, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 48}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4287606', 'n_tokens_mistral': 23574, 'n_tokens_neox': 23107, 'n_words': 5686}