<commit_before># ==================================================================
# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ====================================================================

import os, sys, re, time
from fbutil import *
from cmdutil import *

try:
    import simplejson as json
except ImportError:
    import json

import freebase.rison as rison
from freebase.api import attrdict

from freebase.schema import connect_object, disconnect_object

def cmd_help(fb, command=None):
    """get help on commands

    %prog help [cmd]
    """

    if command is not None:
        cmd = fb.commands[command]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)
        print cmd.doc
        return

    print """
the interface to this tool is loosely modeled on the
"svn" command-line tool.  many commands work slightly
differently from their unix and svn equivalents - 
read the doc for the command first.

use "%s help <subcommand>" for help on a particular
subcommand.
""" % fb.progpath

    fb.oparser.print_help()

    print 'available subcommands:'
    cmds = sorted(fb.commands.keys())

    for fk in cmds:
        cmd = fb.commands[fk]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)


def cmd_wikihelp(fb):
    """get help on commands in mediawiki markup format

    %prog wikihelp
    """

    print """usage: %s subcommand [ args ... ]

    the interface to this tool is loosely modeled on the
    "svn" command-line tool.  many commands work slightly
    differently from their unix and svn equivalents - 
    read the doc for the command first.
    """ % fb.progpath

    print 'subcommands:'
    for fk in sorted(fb.commands.keys()):
        cmd = fb.commands[fk]

        # mediawiki markup cares about the difference between
        #  a blank line and a blank line with a bunch of
        #  spaces at the start of it.  ewww.
        doc = re.sub(r'\r?\n {0,8}', '\n        ', cmd.doc)

        print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc)
        print doc
        print


# this command is disabled because it relies on some svn glue that
# was not sufficiently well thought out.
def old_cmd_pwid(fb):
    """show the "current working namespace id"

    %prog pwid

    by default, relative ids can't be resolved.  however, if you
    run the fb tool from a directory that has the svn property
    'freebase:id' set, relative ids will be resolved relative
    to that id.  the idea is that you can create an svn tree
    that parallels the freebase namespace tree.  

    for example:
        $ %prog pwid

        $ svn propset freebase:id "/freebase" .

        $ %prog pwid
        /freebase

    """
    if fb.cwid is not None:
        print fb.cwid
    else:
        print ''


@option('long', '-l', default=False, action='store_true', help='show time and permission')
@option('timesort', '-t', default=False, action='store_true', help='sort by time')
@option('revsort', '-r', default=False, action='store_true', help='reverse sort order')
def cmd_ls(fb, path=None, long=False, timesort=False, revsort=False):
    """list the keys in a namespace

    %prog ls [id]
    """
    path = fb.absid(path)
    # ignore trailing /
    if len(path) > 1:
        path = re.sub('/$', '', path)

    kq = {
        'value': None,
        'namespace': {
            'id': None
        },
        'sort': 'value',
        'optional':True
     }

    if long or timesort:
        kq['link'] = {
            'timestamp': None,
            'creator': None
          }
    if timesort:
        kq['sort'] = 'link.timestamp'

    if revsort:
        kq['sort'] = '-' + kq['sort']

    q = {'id': path,
         '/type/namespace/keys': [kq]
         }

    r = fb.mss.mqlread(q)
    if r is None:
        raise CmdException('query for id %r failed' % path)

    #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']]))

    for mk in r['/type/namespace/keys']:
        if long:
            # trim off the /user/
            creator = re.sub('^/user/', '', mk.link.creator)
            timestamp = mk.link.timestamp
            fb.out(mk.value, creator, timestamp)
        else:
            fb.out(mk.value)

    if 0:
        suffix = ''
        if ('/type/namespace' in mk.namespace.type
            or '/type/domain' in mk.namespace.type):
            suffix = '/'
        print mk.value, mk.namespace.id+suffix



def cmd_mkdir(fb, path):
    """create a new freebase namespace
    %prog mkdir id

    create a new instance of /type/namespace at the given
    point in id space.  if id already exists, it should be
    a namespace.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)
    dir,file = dirsplit(path)
    wq = { 'create': 'unless_exists',
           'key':{
               'connect': 'insert',
               'namespace': dir,
               'value': file
               },
           'name': path,
           'type': '/type/namespace'
         }

    r = fb.mss.mqlwrite(wq)

def cmd_ln(fb, src, dst):
    """create a namespace key
    %prog ln srcid dstid

    create a new namespace link at dstid to the object
    currently at srcid.
    """
    src = fb.absid(src)
    dst = fb.absid(dst)
    
    # ignore trailing /
    src = re.sub('/$', '', src)
    dst = re.sub('/$', '', dst)

    return connect_object(fb.mss, src, dst)

def cmd_rm(fb, path):
    """unlink a namespace key
    %prog rm id
    
    remove the /type/key that connects the given id to its
    parent.  id must be a path for this to make any sense.

    note that is like unix 'unlink' rather than 'rm'.
    it won't complain if the 'subdirectory' contains data,
    since that data will still be accessible to other queries.
    it's not like 'rm -rf' either, because it doesn't
    disturb anything other than the one directory entry.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)

    return disconnect_object(fb.mss, path)

def cmd_mv(fb, src, dst):
    """rename srcid to dstid.
    %prog mv srcid dstid

    equivalent to:
    $ fb ln <srcid> <dstid>
    $ fb rm <srcid>
    """
    cmd_ln(fb, src, dst)
    cmd_rm(fb, src)

def cmd_cd(fb, path):
    path = fb.absid(path)
    fb.pwd = path

def cmd_pwd(fb):
    print fb.pwd

def cmd_cat(fb, id, include_headers=False):
    """download a document from freebase to stdout
    %prog cat id

    equivalent to "%prog get id -".
    """
    return cmd_get(fb, id, localfile='-', include_headers=include_headers)

def cmd_get(fb, id, localfile=None, include_headers=False):
    """download a file from freebase<commit_msg>basic shell<commit_after># ==================================================================
# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ====================================================================

import os, sys, re, time
from fbutil import *
from cmdutil import *

try:
    import simplejson as json
except ImportError:
    import json

import freebase.rison as rison
from freebase.api import attrdict

from freebase.schema import connect_object, disconnect_object

def cmd_help(fb, command=None):
    """get help on commands

    %prog help [cmd]
    """

    if command is not None:
        cmd = fb.commands[command]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)
        print cmd.doc
        return

    print """
the interface to this tool is loosely modeled on the
"svn" command-line tool.  many commands work slightly
differently from their unix and svn equivalents - 
read the doc for the command first.

use "%s help <subcommand>" for help on a particular
subcommand.
""" % fb.progpath

    fb.oparser.print_help()

    print 'available subcommands:'
    cmds = sorted(fb.commands.keys())

    for fk in cmds:
        cmd = fb.commands[fk]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)


def cmd_wikihelp(fb):
    """get help on commands in mediawiki markup format

    %prog wikihelp
    """

    print """usage: %s subcommand [ args ... ]

    the interface to this tool is loosely modeled on the
    "svn" command-line tool.  many commands work slightly
    differently from their unix and svn equivalents - 
    read the doc for the command first.
    """ % fb.progpath

    print 'subcommands:'
    for fk in sorted(fb.commands.keys()):
        cmd = fb.commands[fk]

        # mediawiki markup cares about the difference between
        #  a blank line and a blank line with a bunch of
        #  spaces at the start of it.  ewww.
        doc = re.sub(r'\r?\n {0,8}', '\n        ', cmd.doc)

        print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc)
        print doc
        print


# this command is disabled because it relies on some svn glue that
# was not sufficiently well thought out.
def old_cmd_pwid(fb):
    """show the "current working namespace id"

    %prog pwid

    by default, relative ids can't be resolved.  however, if you
    run the fb tool from a directory that has the svn property
    'freebase:id' set, relative ids will be resolved relative
    to that id.  the idea is that you can create an svn tree
    that parallels the freebase namespace tree.  

    for example:
        $ %prog pwid

        $ svn propset freebase:id "/freebase" .

        $ %prog pwid
        /freebase

    """
    if fb.cwid is not None:
        print fb.cwid
    else:
        print ''


@option('long', '-l', default=False, action='store_true', help='show time and permission')
@option('timesort', '-t', default=False, action='store_true', help='sort by time')
@option('revsort', '-r', default=False, action='store_true', help='reverse sort order')
def cmd_ls(fb, path=None, long=False, timesort=False, revsort=False):
    """list the keys in a namespace

    %prog ls [id]
    """
    path = fb.absid(path)
    # ignore trailing /
    if len(path) > 1:
        path = re.sub('/$', '', path)

    kq = {
        'value': None,
        'namespace': {
            'id': None
        },
        'sort': 'value',
        'optional':True
     }

    if long or timesort:
        kq['link'] = {
            'timestamp': None,
            'creator': None
          }
    if timesort:
        kq['sort'] = 'link.timestamp'

    if revsort:
        kq['sort'] = '-' + kq['sort']

    q = {'id': path,
         '/type/namespace/keys': [kq]
         }

    r = fb.mss.mqlread(q)
    if r is None:
        raise CmdException('query for id %r failed' % path)

    #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']]))

    for mk in r['/type/namespace/keys']:
        if long:
            # trim off the /user/
            creator = re.sub('^/user/', '', mk.link.creator)
            timestamp = mk.link.timestamp
            fb.out(mk.value, creator, timestamp)
        else:
            fb.out(mk.value)

    if 0:
        suffix = ''
        if ('/type/namespace' in mk.namespace.type
            or '/type/domain' in mk.namespace.type):
            suffix = '/'
        print mk.value, mk.namespace.id+suffix



def cmd_mkdir(fb, path):
    """create a new freebase namespace
    %prog mkdir id

    create a new instance of /type/namespace at the given
    point in id space.  if id already exists, it should be
    a namespace.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)
    dir,file = dirsplit(path)
    wq = { 'create': 'unless_exists',
           'key':{
               'connect': 'insert',
               'namespace': dir,
               'value': file
               },
           'name': path,
           'type': '/type/namespace'
         }

    r = fb.mss.mqlwrite(wq)

def cmd_ln(fb, src, dst):
    """create a namespace key
    %prog ln srcid dstid

    create a new namespace link at dstid to the object
    currently at srcid.
    """
    src = fb.absid(src)
    dst = fb.absid(dst)
    
    # ignore trailing /
    src = re.sub('/$', '', src)
    dst = re.sub('/$', '', dst)

    return connect_object(fb.mss, src, dst)

def cmd_rm(fb, path):
    """unlink a namespace key
    %prog rm id
    
    remove the /type/key that connects the given id to its
    parent.  id must be a path for this to make any sense.

    note that is like unix 'unlink' rather than 'rm'.
    it won't complain if the 'subdirectory' contains data,
    since that data will still be accessible to other queries.
    it's not like 'rm -rf' either, because it doesn't
    disturb anything other than the one directory entry.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)

    return disconnect_object(fb.mss, path)

def cmd_mv(fb, src, dst):
    """rename srcid to dstid.
    %prog mv srcid dstid

    equivalent to:
    $ fb ln <srcid> <dstid>
    $ fb rm <srcid>
    """
    cmd_ln(fb, src, dst)
    cmd_rm(fb, src)

def cmd_cd(fb, path):
    path = fb.absid(path)
    fb.pwd = path

def cmd_pwd(fb):
    print fb.pwd

def cmd_cat(fb, id, include_headers=False):
    """download a document from freebase to stdout
    %prog cat id

    equivalent to "%prog get id -".
    """
    return cmd_get(fb, id, localfile='-', include_headers=include_headers)

def cmd_shell(fb):
    import readline
    def complete_cmd(text, state):
        return ''
    readline.set_completer(complete_cmd)
#    readline.parse_and_bind('tab: complete')
    graphs = {'http://trunk.qa.metaweb.com':'qa',
              'http://branch.qa.metaweb.com':'qa',
              'http://www.freebase.com':'otg',
              'http://api.freebase.com':'otg',
              'http://freebase.com':'otg',
              'http://www.sandbox-freebase.com':'sandbox',
              'http://api.sandbox-freebase.com':'sandbox',
              'http://sandbox-freebase.com':'sandbox'
              }

    gname = graphs.get(fb.mss.service_url, 'unknown')
    while True:
        try:
            line = raw_input((fb.mss.username+'@' if fb.mss.username else '') +gname+': '+fb.pwd+' > ')
        except EOFError:
            print
            break

        if line:
            args = line.split()
            cmd = args.pop(0)
            if cmd in fb.commands:
                fb.dispatch(fb.commands[cmd], args, {});
        
def cmd_get(fb, id, localfile=None, include_headers=False):
    """download a file from freebase
{'fraction_non_alphanumeric': 0.12590627763041556, 'fraction_numerical': 0.0019451812555260832, 'mean_word_length': 2.4873586844809865, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '779125', 'n_tokens_mistral': 5369, 'n_tokens_neox': 5010, 'n_words': 1831}