<commit_before>#
#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Lead Developers: <NAME> and <NAME>
#   Authors: <NAME>, <NAME>, <NAME>, <NAME>
#   Research Leads: <NAME>, <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import time
import inspect
import os
import json
import datetime
import re
import operator
import copy
import math
import ast
import sys
import random
import pandas

import pylab
import numpy
import matplotlib.cm
from collections import defaultdict

import bayesdb.settings as S
from persistence_layer import PersistenceLayer
import api_utils
import data_utils
import utils
import pairwise
import functions
import select_utils
import estimate_columns_utils
import plotting_utils
import parser as p

class Engine(object):
  def __init__(self, crosscat_host=None, crosscat_port=8007, crosscat_engine_type='multiprocessing', seed=None, **kwargs):
    """ One optional argument that you may find yourself using frequently is seed.
    It defaults to random seed, but for testing/reproduceability purposes you may
    want a deterministic one. """
    
    self.persistence_layer = PersistenceLayer()
    self.parser = p.Parser()

    if crosscat_host is None or crosscat_host == 'localhost':
      self.online = False
      
      # Only dependent on CrossCat when you actually instantiate Engine
      # (i.e., allow engine to be imported in order to examine the API, without CrossCat)
      from crosscat.CrossCatClient import get_CrossCatClient
      self.backend = get_CrossCatClient(crosscat_engine_type, seed=seed, **kwargs)
    else:
      self.online = True
      self.hostname = crosscat_host
      self.port = crosscat_port
      self.URI = 'http://' + self.hostname + ':%d' % self.port

  def call_backend(self, method_name, args_dict):
    """
    Helper function used to call the CrossCat backend, whether it is remote or local.
    Accept method name and arguments for that method as input.
    """
    if self.online:
      out, id = api_utils.call(method_name, args_dict, self.URI)
    else:
      method = getattr(self.backend, method_name)
      out = method(**args_dict)
    return out
  
  def help(self, method=None):
    help_methods = dict()
    help_methods['select']="""
    SELECT <columns|functions> FROM <btable> [WHERE <whereclause>] [ORDER BY <columns|functions>] [LIMIT <limit>]
    """
    help_methods['infer'] = """
    INFER <columns|functions> FROM <btable> [WHERE <whereclause>] [WITH CONFIDENCE <confidence>] [WITH <numsamples> SAMPLES] [ORDER BY <columns|functions>] [LIMIT <limit>]
    """
    help_methods['simulate'] = """
    SIMULATE [HIST] <columns> FROM <btable> [WHERE <whereclause>] [GIVEN <column>=<value>] TIMES <times> [SAVE TO <file>]
    """
    help_methods['estimate'] = """
    ESTIMATE COLUMNS FROM <btable> [WHERE <whereclause>] [ORDER BY <functions>] [LIMIT <limit>] [AS <column_list>] 
    
    ESTIMATE PAIRWISE <function> FROM <btable> [FOR <columns>] [SAVE TO <file>] [SAVE CONNECTED COMPONENTS WITH THRESHOLD <threshold> AS <column_list>] 
    
    ESTIMATE PAIRWISE ROW SIMILARITY [WITH RESPECT TO <columns|column_lists>]FROM <btable> [FOR <rows>] [SAVE TO <file>] [SAVE CONNECTED COMPONENTS WITH THRESHOLD <threshold> [INTO|AS] <btable>]
    """

    ## Important to make sure that http://probcomp.csail.mit.edu/bayesdb/docs/bql.html is up to date
    help_all = """
    Welcome to BQL help. 
    
    For the BQL documentation, please visit:
    http://probcomp.csail.mit.edu/bayesdb/docs/bql.html
    
    Here is a list of BQL commands and their syntax:
    """
    help_basic = """
    Welcome to BQL help. 
  
    If you know the query you wish to use, type 'HELP <query_name>'

    For all queries, type 'HELP ALL'
    
    For the complete BQL documentation, please visit: 
    http://probcomp.csail.mit.edu/bayesdb/docs/bql.html
    """
    
    if method == None:
      help_string = help_basic
    elif method == 'all':
      help_string = help_all + ''.join(help_methods.values())
    elif method in help_methods:
      help_string = help_all + help_methods[method]
    else:
      help_string = help_basic + '\n\tThe method you typed was not recognize. Try "HELP" or "HELP ALL"'
    print help_string
    return help_string<commit_msg>added more help methods<commit_after>#
#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Lead Developers: <NAME> and <NAME>
#   Authors: <NAME>, <NAME>, <NAME>, <NAME>
#   Research Leads: <NAME>, <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import time
import inspect
import os
import json
import datetime
import re
import operator
import copy
import math
import ast
import sys
import random
import pandas

import pylab
import numpy
import matplotlib.cm
from collections import defaultdict

import bayesdb.settings as S
from persistence_layer import PersistenceLayer
import api_utils
import data_utils
import utils
import pairwise
import functions
import select_utils
import estimate_columns_utils
import plotting_utils
import parser as p

class Engine(object):
  def __init__(self, crosscat_host=None, crosscat_port=8007, crosscat_engine_type='multiprocessing', seed=None, **kwargs):
    """ One optional argument that you may find yourself using frequently is seed.
    It defaults to random seed, but for testing/reproduceability purposes you may
    want a deterministic one. """
    
    self.persistence_layer = PersistenceLayer()
    self.parser = p.Parser()

    if crosscat_host is None or crosscat_host == 'localhost':
      self.online = False
      
      # Only dependent on CrossCat when you actually instantiate Engine
      # (i.e., allow engine to be imported in order to examine the API, without CrossCat)
      from crosscat.CrossCatClient import get_CrossCatClient
      self.backend = get_CrossCatClient(crosscat_engine_type, seed=seed, **kwargs)
    else:
      self.online = True
      self.hostname = crosscat_host
      self.port = crosscat_port
      self.URI = 'http://' + self.hostname + ':%d' % self.port

  def call_backend(self, method_name, args_dict):
    """
    Helper function used to call the CrossCat backend, whether it is remote or local.
    Accept method name and arguments for that method as input.
    """
    if self.online:
      out, id = api_utils.call(method_name, args_dict, self.URI)
    else:
      method = getattr(self.backend, method_name)
      out = method(**args_dict)
    return out
  
  def help(self, method=None):
    help_methods = dict()
    help_methods['select']="""
    SELECT <columns|functions> FROM <btable> [WHERE <whereclause>] [ORDER BY <columns|functions>] [LIMIT <limit>]
    """
    help_methods['infer'] = """
    INFER <columns|functions> FROM <btable> [WHERE <whereclause>] [WITH CONFIDENCE <confidence>] [WITH <numsamples> SAMPLES] [ORDER BY <columns|functions>] [LIMIT <limit>]
    """
    help_methods['simulate'] = """
    SIMULATE [HIST] <columns> FROM <btable> [WHERE <whereclause>] [GIVEN <column>=<value>] TIMES <times> [SAVE TO <file>]
    """
    help_methods['estimate'] = """
    ESTIMATE COLUMNS FROM <btable> [WHERE <whereclause>] [ORDER BY <functions>] [LIMIT <limit>] [AS <column_list>] 
    
    ESTIMATE PAIRWISE <function> FROM <btable> [FOR <columns>] [SAVE TO <file>] [SAVE CONNECTED COMPONENTS WITH THRESHOLD <threshold> AS <column_list>] 
    
    ESTIMATE PAIRWISE ROW SIMILARITY [WITH RESPECT TO <columns|column_lists>]FROM <btable> [FOR <rows>] [SAVE TO <file>] [SAVE CONNECTED COMPONENTS WITH THRESHOLD <threshold> [INTO|AS] <btable>]
    """

    help_methods['execute'] = """
    EXECUTE FILE <filename.bql>
    """
    
    help_methods['update'] = """
    UPDATE SCHEMA FOR <btable> SET <col1>=<type1>[,<col2>=<type2>...]
    """

    help_methods['initialize'] = """
    INITIALIZE <num_models> MODELS FOR <btable> [WITH CONFIG <config>]
    """

    help_methods['analyze'] = """
    ANALYZE <btable> [MODEL[S] <model_index>-<model_index>] FOR (<num_iterations> ITERATIONS | <seconds> SECONDS)
    """

    help_methods['list'] = """
    LIST BTABLES
    """

    help_methods['show'] = """
    SHOW SCHEMA FOR <btable>

    SHOW MODELS FOR <btable>

    SHOW DIAGNOSTICS FOR <btable>

    SHOW COLUMN LISTS FOR <btable>

    SHOW COLUMNS FOR <column_list|btable>

    SHOW ROW LISTS FOR <table>

    SHOW METADATA FOR <btable> [<metadata-key1> [, <metadata-key2>...]]

    SHOW LABEL FOR <btable> [<column-name-1> [, <column-name-2>...]]
    """

    help_methods['load'] = """
    LOAD MODELS <filename.pkl.gz> INTO <btable>
    """

    help_methods['save'] = """
    SAVE MODELS FROM <btable> TO <filename.pkl.gz>
    """

    help_methods['drop'] = """
    DROP BTABLE <btable>

    DROP MODEL[S] [<model_index>-<model_index>] FROM <btable>
    """

    help_methods['summarize'] = """
    SUMMARIZE <SELECT|INFER|SIMULATE>
    """

    help_methods['plot'] = """
    PLOT <SELECT|INFER|SIMULATE>
    """

    ## Important to make sure that http://probcomp.csail.mit.edu/bayesdb/docs/bql.html is up to date
    help_all = """
    Welcome to BQL help. 
    
    For the BQL documentation, please visit:
    http://probcomp.csail.mit.edu/bayesdb/docs/bql.html
    
    Here is a list of BQL commands and their syntax:
    """
    help_basic = """
    Welcome to BQL help. 
  
    If you know the query you wish to use, type 'HELP <query_name>'

    For all queries, type 'HELP ALL'
    
    For the complete BQL documentation, please visit: 
    http://probcomp.csail.mit.edu/bayesdb/docs/bql.html
    """
    
    if method == None:
      help_string = help_basic
    elif method == 'all':
      help_string = help_all + ''.join(help_methods.values())
    elif method in help_methods:
      help_string = help_all + help_methods[method]
    else:
      help_string = help_basic + '''
    The method you typed was not recognize. Try "HELP", "HELP ALL",
    or one of the following: 
    \tHELP ''' + '\n\tHELP '.join(help_methods.keys()).upper()

    print help_string
    return help_string
{'fraction_non_alphanumeric': 0.10986221899927484, 'fraction_numerical': 0.0036258158085569255, 'mean_word_length': 3.527287648748461, 'pattern_counts': {'":': 0, '<': 129, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6492368', 'n_tokens_mistral': 3558, 'n_tokens_neox': 3386, 'n_words': 1115}