<commit_before>#
#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Lead Developers: <NAME> and <NAME>
#   Authors: <NAME>, <NAME>, <NAME>, <NAME>
#   Research Leads: <NAME>, <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import utils
import os
import bql_grammar as bql
import pyparsing as pp
#import ast
import functions
import operator

class Parser(object):
    def __init__(self):
        self.reset_root_dir()

    def pyparse_input(self, input_string):
        """Uses the grammar defined in bql_grammar to create a pyparsing object out of an input string"""
        try:
            bql_blob_ast = bql.bql_input.parseString(input_string, parseAll=True)
        except pp.ParseException as x:
            raise utils.BayesDBParseError("Invalid query. Could not parse (Line {e.lineno}, column {e.col}):\n\t'{e.line}'\n\t".format(e=x) + ' ' * x.col + '^')
        return bql_blob_ast
        
    def split_statements(self,bql_blob_ast):
        """
        returns a list of bql statements, not necessarily useful. 
        """
        return [bql_statement_ast for bql_statement_ast in bql_blob_ast]

    def parse_single_statement(self,bql_statement_ast):
        ## TODO Check for nest
        parse_method = getattr(self,'parse_' + bql_statement_ast.statement_id)
        return parse_method(bql_statement_ast)

#####################################################################################
## -------------------------- Individual Parse Methods --------------------------- ##
#####################################################################################

    def parse_list_btables(self,bql_statement_ast):
        if bql_statement_ast.statement_id == "list_btables":
            return 'list_btables', dict(), None
        else:
            raise utils.BayesDBParseError("Parsing statement as LIST BTABLES failed")

    def parse_execute_file(self,bql_statement_ast):
        return 'execute_file', dict(filename=self.get_absolute_path(bql_statement_ast.filename)), None

    def parse_show_schema(self,bql_statement_ast):
        return 'show_schema', dict(tablename=bql_statement_ast.btable), None

    def parse_show_models(self,bql_statement_ast):
        return 'show_models', dict(tablename=bql_statement_ast.btable), None

    def parse_show_diagnostics(self,bql_statement_ast):
        return 'show_diagnostics', dict(tablename=bql_statement_ast.btable), None

    def parse_drop_models(self,bql_statement_ast):
        model_indices = None
        if bql_statement_ast.index_clause != '':
            model_indices = bql_statement_ast.index_clause.asList()
        return 'drop_models', dict(tablename=bql_statement_ast.btable, model_indices=model_indices), None

    def parse_initialize_models(self,bql_statement_ast):
        n_models = int(bql_statement_ast.num_models)
        tablename = bql_statement_ast.btable
        arguments_dict = dict(tablename=tablename, n_models=n_models, model_config=None)
        if bql_statement_ast.config != '':
            arguments_dict['model_config'] = bql_statement_ast.config
        return 'initialize_models', arguments_dict, None

    def parse_create_btable(self,bql_statement_ast):
        tablename = bql_statement_ast.btable
        filename = self.get_absolute_path(bql_statement_ast.filename)
        return 'create_btable', dict(tablename=tablename, cctypes_full=None), dict(csv_path=filename)
        #TODO types?

    def parse_update_schema(self,bql_statement_ast):
        tablename = bql_statement_ast.btable
        mappings = dict()
        type_clause = bql_statement_ast.type_clause
        for update in type_clause:
            mappings[update[0]]=update[1]
        return 'update_schema', dict(tablename=tablename, mappings=mappings), None

    def parse_drop_btable(self,bql_statement_ast):
        return 'drop_btable', dict(tablename=bql_statement_ast.btable), None

    def parse_analyze(self,bql_statement_ast):
        model_indices = None
        iterations = None
        seconds = None
        kernel = 0
        tablename = bql_statement_ast.btable
        if bql_statement_ast.index_clause != '':
            model_indices = bql_statement_ast.index_clause.asList()
        if bql_statement_ast.num_seconds !='':
            seconds = int(bql_statement_ast.num_seconds)
        if bql_statement_ast.num_iterations !='':
            iterations = int(bql_statement_ast.num_iterations)
        if bql_statement_ast.with_kernel_clause != '':
            kernel = bql_statement_ast.with_kernel_clause.kernel_id
            if kernel == 'mh': ## TODO should return None or something for invalid kernels
                kernel=1
        return 'analyze', dict(tablename=tablename, model_indices=model_indices,
                                   iterations=iterations, seconds=seconds, ct_kernel=kernel), None
        
    def parse_show_row_lists(self,bql_statement_ast):
        return 'show_row_lists', dict(tablename=bql_statement_ast.btable), None

    def parse_show_column_lists(self,bql_statement_ast):
        return 'show_column_lists', dict(tablename=bql_statement_ast.btable), None

    def parse_show_columns(self,bql_statement_ast):
        return 'show_columns', dict(tablename=bql_statement_ast.btable), None

    def parse_save_models(self,bql_statement_ast):
        return 'save_models', dict(tablename=bql_statement_ast.btable), dict(pkl_path=bql_statement_ast.filename)

    def parse_load_models(self,bql_statement_ast):
        return 'load_models', dict(tablename=bql_statement_ast.btable), dict(pkl_path=bql_statement_ast.filename)

    def parse_label_columns(self, bql_statement_ast): ##TODO only smoke test right now
        tablename = bql_statement_ast.btable
        source = None
        mappings = None
        if bql_statement_ast.label_clause != '':
            source = 'inline'
            mappings = {}
            for label_set in bql_statement_ast.label_clause:
                mappings[label_set[0]] = label_set[1]
        csv_path = None
        if bql_statement_ast.filename != '':
            csv_path = bql_statement_ast.filename
            source = 'file'
        return 'label_columns', \
            dict(tablename=tablename, mappings=mappings), \
            dict(source=source, csv_path=csv_path)

    def parse_show_metadata(self, bql_statement_ast):
        tablename = None
        if bql_statement_ast.btable != '':
            tablename = bql_statement_ast.btable
        keyset = None
        if bql_statement_ast.keyset != '':
            keyset = bql_statement_ast.keyset
        return 'show_metadata', dict(tablename=tablename, keyset=keyset), None

    def parse_show_label(self, bql_statement_ast):
        tablename = None
        if bql_statement_ast.btable != '':
            tablename = bql_statement_ast.btable
        columnset = None
        if bql_statement_ast.columnset != '':
            columnset = bql_statement_ast.columnset
        return 'show_labels', dict(tablename=tablename, columnset=columnset), None

    def parse_update_metadata(self, bql_statement_ast):
        tablename = bql_statement_ast.btable
        source = None
        mappings = None
        if bql_statement_ast.label_clause != '':
            source = 'inline'
            mappings = {}
            for label_set in bql_statement_ast.label_clause:
                mappings[label_set[0]] = label_set[1]
        csv_path = None
        if bql_statement_ast.filename != '':
            csv_path = bql_statement_ast.filename
            source = 'file'
        return 'update_metadata', \
            dict(tablename=tablename, mappings=mappings), \
            dict(source=source, csv_path=csv_path)
    
    def parse_query(self, bql_statement_ast):
        '''
        master parser for queries (select, infer, simulate, estimate pairwise, etc)
        returns a general args dict which the specific versions of those functions 
        will then trim and check for illegal aruguments through assertions. 
        '''
        statement_id = bql_statement_ast.statement_id 
        
        confidence = 0
        if bql_statement_ast.confidence != '':
            confidence = float(bql_statement_ast.confidence)
            if confidence > 1: 
                raise utils.BayesDBParseError("Confidence cannot be greater than 0.")
        filename = None
        if bql_statement_ast.filename != '':
            filename = bql_statement_ast.filename
        functions = bql_statement_ast.functions
        givens = None
        if bql_statement_ast.given_clause != '':
            givens = bql_statement_ast.given_clause
        limit = float('inf')
        if bql_statement_ast.limit != '':
            limit = int(bql_statement_ast.limit)
        modelids = None
        if bql_statement_ast.using_models_index_clause != '':
            modelids = bql_statement_ast.using_models_index_clause.asList()
        name = None
        if bql_statement_ast.as_column_list != '':
            ## TODO name is a bad name
            name = bql_statement_ast.as_column_list
        newtablename=None ##TODO implement into
        numpredictions = None
        if bql_statement_ast.times != '':
            numpredictions = int(bql_statement_ast.times)
        numsamples = None
        if bql_statement_ast.samples != '':
            numsamples = int(bql_statement_ast.samples)
        order_by = False
        if bql_statement_ast.order_by != '':
            order_by = bql_statement_ast.order_by
        plot=(bql_statement_ast.plot == 'plot')
        column_list = None
        if bql_statement_ast.columns != '':
            column_list = bql_statement_ast.columns[0] ##TODO implement allowing comma separated columns here
            assert len(bql_statement_ast.columns) < 2
        row_list = None
        if bql_statement_ast.rows != '':
            row_list = bql_statement_ast.rows ##TODO parse to list of rows
        summarize=(bql_statement_ast.summarize == 'summarize')
        hist = (bql_statement_ast.hist == 'hist')
        freq = (bql_statement_ast.freq == 'freq')
        tablename = bql_statement_ast.btable
        clusters_name = None
        threshold = None
        if bql_statement_ast.clusters_clause != '':
            clusters_name = bql_statement_ast.clusters_clause.as_label
            threshold = float(bql_statement_ast.clusters_clause.threshold)
        whereclause = None
        if bql_statement_ast.where_conditions != '':
            whereclause = bql_statement_ast.where_conditions

        print "here"
        print clusters_name
        print threshold

        return statement_id, \
            dict(clusters_name=clusters_name,
                 confidence = confidence,
                 functions=functions,
                 givens=givens,
                 limit=limit,
                 modelids=modelids,
                 name=name,
                 newtablename=newtablename,
                 numpredictions=numpredictions,
                 numsamples=numsamples,
                 order_by=order_by,
                 plot=plot,
                 column_list=column_list,
                 row_list=row_list,
                 summarize=summarize,
                 hist=hist,
                 freq=freq,
                 tablename=tablename,
                 threshold=threshold,
                 whereclause=whereclause), \
            dict(plot=plot, 
                 scatter=False, ##TODO remove scatter from args
                 pairwise=False, ##TODO remove pairwise from args
                 filename=filename)

    def parse_infer(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        newtablename = args_dict['newtablename']
        confidence = args_dict['confidence']
        numsamples = args_dict['numsamples']

        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['components_name'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in INFER"
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in INFER"
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS clause not allowed in INFER"
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in INFER"
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES clause not allowed in INFER"
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in INFER"
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in INFER"
        for function in functions:
            assert function.function_id == '', "BayesDBParsingError: %s not valid in INFER" % function.function_id
                
        
        return 'infer', \
            dict(tablename=tablename, functions=functions, 
                 newtablename=newtablename, confidence=confidence, 
                 whereclause=whereclause, limit=limit,
                 numsamples=numsamples, order_by=order_by, 
                 plot=plot, modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(plot=plot, scatter=scatter, pairwise=pairwise, filename=filename)

    def parse_select(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']

        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['components_name'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in SELECT"
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in SELECT"
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS clause not allowed in SELECT"
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in SELECT"
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES clause not allowed in SELECT"
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in SELECT"
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in SELECT"
        assert args_dict['confidence'] == 0, "BayesDBParsingError: CONFIDENCE not allowed in SELECT"
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in SELECT"

        return 'select', \
            dict(tablename=tablename, whereclause=whereclause, 
                 functions=functions, limit=limit, order_by=order_by, plot=plot, 
                 modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(pairwise=pairwise, scatter=scatter, filename=filename, plot=plot)

    def parse_simulate(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot'] 
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        newtablename = args_dict['newtablename']
        givens = args_dict['givens']
        numpredictions = args_dict['numpredictions']
       
        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['components_name'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in SIMULATE."
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS clause not allowed in SIMULATE."
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in SIMULATE."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in SIMULATE."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in SIMULATE."
        assert args_dict['confidence'] == 0, "BayesDBParsingError: CONFIDENCE not allowed in SIMULATE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in SIMULATE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: whereclause not allowed in SIMULATE. Use GIVEN instead."
        for function in functions:
            assert function.function_id == '', "BayesDBParsingError: %s not valid in SIMULATE" % function.function_id

        return 'simulate', \
            dict(tablename=tablename, functions=functions, 
                 newtablename=newtablename, givens=givens, 
                 numpredictions=numpredictions, order_by=order_by, 
                 plot=plot, modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(filename=filename, plot=plot, scatter=scatter, pairwise=pairwise)

    def parse_estimate(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        assert args_dict['functions'][0] == 'column', "BayesDBParseError: must be ESTIMATE COLUMNS."
        functions = args_dict['functions']
        tablename = args_dict['tablename']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        name = args_dict['name']

        assert args_dict['components_name'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS not allowed in estimate columns."
        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in estimate columns."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in estimate columns."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in estimate columns."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in estimate columns."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in estimate columns."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR COLUMNS not allowed in estimate columns."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR ROWS not allowed in estimate columns."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in estimate columns."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in estimated columns."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in estimated columns."
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CONNECTED COMPONENTS not allowed in estimate columns."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in estimate columns."

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in estimate columns."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in estimate columns."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in estimate columns."
        assert client_dict['filename'] == None, "BayesDBParsingError: AS FILE not allowed in estimate columns."

        return 'estimate_columns', \
            dict(tablename=tablename, functions=functions, 
                 whereclause=whereclause, limit=limit, 
                 order_by=order_by, name=name, modelids=modelids), \
            None

    def parse_estimate_pairwise_row(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        functions = args_dict['functions'][0]
        assert len(args_dict['functions']) == 1, "BayesDBParsingError: Only one function allowed in estimate pairwise."
        tablename = args_dict['tablename']
        row_list = args_dict['row_list']
        components_name = args_dict['components_name']
        threshold = args_dict['threshold']
        modelids = args_dict['modelids']
        filename = client_dict['filename']

        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR COLUMNS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in ESTIMATE PAIRWISE."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in estimated columns."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: WHERE not allowed in ESTIMATE PAIRWISE."

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in ESTIMATE PAIRWISE."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in ESTIMATE PAIRWISE."

        return 'estimate_pairwise_row', \
            dict(tablename=tablename, function=functions,
                 row_list=row_list, components_name=components_name, 
                 threshold=threshold, modelids=modelids), \
            dict(filename=filename)

    def parse_estimate_pairwise(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        functions = args_dict['functions']
        assert len(args_dict['functions']) == 1, "BayesDBParsingError: Only one function allowed in estimate pairwise."
        assert functions[0].function_id in ['correlation', 'mutual information', 'dependence probability']
        function_name = functions[0].function_id

        tablename = args_dict['tablename']
        column_list = args_dict['column_list']
        components_name = args_dict['components_name']
        threshold = args_dict['threshold']
        modelids = args_dict['modelids']
        filename = client_dict['filename']

        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR ROWS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in ESTIMATE PAIRWISE."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in estimated columns."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: whereclause not allowed in ESTIMATE PAIRWISE"

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in ESTIMATE PAIRWISE."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in ESTIMATE PAIRWISE."

        return 'estimate_pairwise', \
            dict(tablename=tablename, function_name=function_name,
                 column_list=column_list, components_name=components_name, 
                 threshold=threshold, modelids=modelids), \
            dict(filename=filename)

#####################################################################################
## ------------------------------ Function parsing ------------------------------- ##
#####################################################################################
    def get_args_pred_prob(self, function_group, M_c):
        """
        returns the column index from a predictive probability function
        raises exceptions for unfound columns
        """
        if function_group.column != '' and function_group.column in M_c['name_to_idx']:
            column = function_group.column
            c_idx = M_c['name_to_idx'][column]
            return c_idx<commit_msg>Update components_name to clusters_name throughout parser file<commit_after>#
#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Lead Developers: <NAME> and <NAME>
#   Authors: <NAME>, <NAME>, <NAME>, <NAME>
#   Research Leads: Vikash Mansinghka, <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import utils
import os
import bql_grammar as bql
import pyparsing as pp
#import ast
import functions
import operator

class Parser(object):
    def __init__(self):
        self.reset_root_dir()

    def pyparse_input(self, input_string):
        """Uses the grammar defined in bql_grammar to create a pyparsing object out of an input string"""
        try:
            bql_blob_ast = bql.bql_input.parseString(input_string, parseAll=True)
        except pp.ParseException as x:
            raise utils.BayesDBParseError("Invalid query. Could not parse (Line {e.lineno}, column {e.col}):\n\t'{e.line}'\n\t".format(e=x) + ' ' * x.col + '^')
        return bql_blob_ast
        
    def split_statements(self,bql_blob_ast):
        """
        returns a list of bql statements, not necessarily useful. 
        """
        return [bql_statement_ast for bql_statement_ast in bql_blob_ast]

    def parse_single_statement(self,bql_statement_ast):
        ## TODO Check for nest
        parse_method = getattr(self,'parse_' + bql_statement_ast.statement_id)
        return parse_method(bql_statement_ast)

#####################################################################################
## -------------------------- Individual Parse Methods --------------------------- ##
#####################################################################################

    def parse_list_btables(self,bql_statement_ast):
        if bql_statement_ast.statement_id == "list_btables":
            return 'list_btables', dict(), None
        else:
            raise utils.BayesDBParseError("Parsing statement as LIST BTABLES failed")

    def parse_execute_file(self,bql_statement_ast):
        return 'execute_file', dict(filename=self.get_absolute_path(bql_statement_ast.filename)), None

    def parse_show_schema(self,bql_statement_ast):
        return 'show_schema', dict(tablename=bql_statement_ast.btable), None

    def parse_show_models(self,bql_statement_ast):
        return 'show_models', dict(tablename=bql_statement_ast.btable), None

    def parse_show_diagnostics(self,bql_statement_ast):
        return 'show_diagnostics', dict(tablename=bql_statement_ast.btable), None

    def parse_drop_models(self,bql_statement_ast):
        model_indices = None
        if bql_statement_ast.index_clause != '':
            model_indices = bql_statement_ast.index_clause.asList()
        return 'drop_models', dict(tablename=bql_statement_ast.btable, model_indices=model_indices), None

    def parse_initialize_models(self,bql_statement_ast):
        n_models = int(bql_statement_ast.num_models)
        tablename = bql_statement_ast.btable
        arguments_dict = dict(tablename=tablename, n_models=n_models, model_config=None)
        if bql_statement_ast.config != '':
            arguments_dict['model_config'] = bql_statement_ast.config
        return 'initialize_models', arguments_dict, None

    def parse_create_btable(self,bql_statement_ast):
        tablename = bql_statement_ast.btable
        filename = self.get_absolute_path(bql_statement_ast.filename)
        return 'create_btable', dict(tablename=tablename, cctypes_full=None), dict(csv_path=filename)
        #TODO types?

    def parse_update_schema(self,bql_statement_ast):
        tablename = bql_statement_ast.btable
        mappings = dict()
        type_clause = bql_statement_ast.type_clause
        for update in type_clause:
            mappings[update[0]]=update[1]
        return 'update_schema', dict(tablename=tablename, mappings=mappings), None

    def parse_drop_btable(self,bql_statement_ast):
        return 'drop_btable', dict(tablename=bql_statement_ast.btable), None

    def parse_analyze(self,bql_statement_ast):
        model_indices = None
        iterations = None
        seconds = None
        kernel = 0
        tablename = bql_statement_ast.btable
        if bql_statement_ast.index_clause != '':
            model_indices = bql_statement_ast.index_clause.asList()
        if bql_statement_ast.num_seconds !='':
            seconds = int(bql_statement_ast.num_seconds)
        if bql_statement_ast.num_iterations !='':
            iterations = int(bql_statement_ast.num_iterations)
        if bql_statement_ast.with_kernel_clause != '':
            kernel = bql_statement_ast.with_kernel_clause.kernel_id
            if kernel == 'mh': ## TODO should return None or something for invalid kernels
                kernel=1
        return 'analyze', dict(tablename=tablename, model_indices=model_indices,
                                   iterations=iterations, seconds=seconds, ct_kernel=kernel), None
        
    def parse_show_row_lists(self,bql_statement_ast):
        return 'show_row_lists', dict(tablename=bql_statement_ast.btable), None

    def parse_show_column_lists(self,bql_statement_ast):
        return 'show_column_lists', dict(tablename=bql_statement_ast.btable), None

    def parse_show_columns(self,bql_statement_ast):
        return 'show_columns', dict(tablename=bql_statement_ast.btable), None

    def parse_save_models(self,bql_statement_ast):
        return 'save_models', dict(tablename=bql_statement_ast.btable), dict(pkl_path=bql_statement_ast.filename)

    def parse_load_models(self,bql_statement_ast):
        return 'load_models', dict(tablename=bql_statement_ast.btable), dict(pkl_path=bql_statement_ast.filename)

    def parse_label_columns(self, bql_statement_ast): ##TODO only smoke test right now
        tablename = bql_statement_ast.btable
        source = None
        mappings = None
        if bql_statement_ast.label_clause != '':
            source = 'inline'
            mappings = {}
            for label_set in bql_statement_ast.label_clause:
                mappings[label_set[0]] = label_set[1]
        csv_path = None
        if bql_statement_ast.filename != '':
            csv_path = bql_statement_ast.filename
            source = 'file'
        return 'label_columns', \
            dict(tablename=tablename, mappings=mappings), \
            dict(source=source, csv_path=csv_path)

    def parse_show_metadata(self, bql_statement_ast):
        tablename = None
        if bql_statement_ast.btable != '':
            tablename = bql_statement_ast.btable
        keyset = None
        if bql_statement_ast.keyset != '':
            keyset = bql_statement_ast.keyset
        return 'show_metadata', dict(tablename=tablename, keyset=keyset), None

    def parse_show_label(self, bql_statement_ast):
        tablename = None
        if bql_statement_ast.btable != '':
            tablename = bql_statement_ast.btable
        columnset = None
        if bql_statement_ast.columnset != '':
            columnset = bql_statement_ast.columnset
        return 'show_labels', dict(tablename=tablename, columnset=columnset), None

    def parse_update_metadata(self, bql_statement_ast):
        tablename = bql_statement_ast.btable
        source = None
        mappings = None
        if bql_statement_ast.label_clause != '':
            source = 'inline'
            mappings = {}
            for label_set in bql_statement_ast.label_clause:
                mappings[label_set[0]] = label_set[1]
        csv_path = None
        if bql_statement_ast.filename != '':
            csv_path = bql_statement_ast.filename
            source = 'file'
        return 'update_metadata', \
            dict(tablename=tablename, mappings=mappings), \
            dict(source=source, csv_path=csv_path)
    
    def parse_query(self, bql_statement_ast):
        '''
        master parser for queries (select, infer, simulate, estimate pairwise, etc)
        returns a general args dict which the specific versions of those functions 
        will then trim and check for illegal aruguments through assertions. 
        '''
        statement_id = bql_statement_ast.statement_id 
        
        confidence = 0
        if bql_statement_ast.confidence != '':
            confidence = float(bql_statement_ast.confidence)
            if confidence > 1: 
                raise utils.BayesDBParseError("Confidence cannot be greater than 0.")
        filename = None
        if bql_statement_ast.filename != '':
            filename = bql_statement_ast.filename
        functions = bql_statement_ast.functions
        givens = None
        if bql_statement_ast.given_clause != '':
            givens = bql_statement_ast.given_clause
        limit = float('inf')
        if bql_statement_ast.limit != '':
            limit = int(bql_statement_ast.limit)
        modelids = None
        if bql_statement_ast.using_models_index_clause != '':
            modelids = bql_statement_ast.using_models_index_clause.asList()
        name = None
        if bql_statement_ast.as_column_list != '':
            ## TODO name is a bad name
            name = bql_statement_ast.as_column_list
        newtablename=None ##TODO implement into
        numpredictions = None
        if bql_statement_ast.times != '':
            numpredictions = int(bql_statement_ast.times)
        numsamples = None
        if bql_statement_ast.samples != '':
            numsamples = int(bql_statement_ast.samples)
        order_by = False
        if bql_statement_ast.order_by != '':
            order_by = bql_statement_ast.order_by
        plot=(bql_statement_ast.plot == 'plot')
        column_list = None
        if bql_statement_ast.columns != '':
            column_list = bql_statement_ast.columns[0] ##TODO implement allowing comma separated columns here
            assert len(bql_statement_ast.columns) < 2
        row_list = None
        if bql_statement_ast.rows != '':
            row_list = bql_statement_ast.rows ##TODO parse to list of rows
        summarize=(bql_statement_ast.summarize == 'summarize')
        hist = (bql_statement_ast.hist == 'hist')
        freq = (bql_statement_ast.freq == 'freq')
        tablename = bql_statement_ast.btable
        clusters_name = None
        threshold = None
        if bql_statement_ast.clusters_clause != '':
            clusters_name = bql_statement_ast.clusters_clause.as_label
            threshold = float(bql_statement_ast.clusters_clause.threshold)
        whereclause = None
        if bql_statement_ast.where_conditions != '':
            whereclause = bql_statement_ast.where_conditions

        return statement_id, \
            dict(clusters_name=clusters_name,
                 confidence = confidence,
                 functions=functions,
                 givens=givens,
                 limit=limit,
                 modelids=modelids,
                 name=name,
                 newtablename=newtablename,
                 numpredictions=numpredictions,
                 numsamples=numsamples,
                 order_by=order_by,
                 plot=plot,
                 column_list=column_list,
                 row_list=row_list,
                 summarize=summarize,
                 hist=hist,
                 freq=freq,
                 tablename=tablename,
                 threshold=threshold,
                 whereclause=whereclause), \
            dict(plot=plot, 
                 scatter=False, ##TODO remove scatter from args
                 pairwise=False, ##TODO remove pairwise from args
                 filename=filename)

    def parse_infer(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        newtablename = args_dict['newtablename']
        confidence = args_dict['confidence']
        numsamples = args_dict['numsamples']

        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['clusters_name'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in INFER"
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in INFER"
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS clause not allowed in INFER"
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in INFER"
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES clause not allowed in INFER"
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in INFER"
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in INFER"
        for function in functions:
            assert function.function_id == '', "BayesDBParsingError: %s not valid in INFER" % function.function_id
                
        
        return 'infer', \
            dict(tablename=tablename, functions=functions, 
                 newtablename=newtablename, confidence=confidence, 
                 whereclause=whereclause, limit=limit,
                 numsamples=numsamples, order_by=order_by, 
                 plot=plot, modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(plot=plot, scatter=scatter, pairwise=pairwise, filename=filename)

    def parse_select(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']

        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['clusters_name'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in SELECT"
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in SELECT"
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS clause not allowed in SELECT"
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in SELECT"
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES clause not allowed in SELECT"
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in SELECT"
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in SELECT"
        assert args_dict['confidence'] == 0, "BayesDBParsingError: CONFIDENCE not allowed in SELECT"
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in SELECT"

        return 'select', \
            dict(tablename=tablename, whereclause=whereclause, 
                 functions=functions, limit=limit, order_by=order_by, plot=plot, 
                 modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(pairwise=pairwise, scatter=scatter, filename=filename, plot=plot)

    def parse_simulate(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        tablename = args_dict['tablename']
        functions = args_dict['functions']
        summarize = args_dict['summarize']
        hist = args_dict['hist']
        freq = args_dict['freq']
        plot = args_dict['plot'] 
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        newtablename = args_dict['newtablename']
        givens = args_dict['givens']
        numpredictions = args_dict['numpredictions']
       
        pairwise = client_dict['pairwise']
        filename = client_dict['filename']
        scatter = client_dict['scatter']

        assert args_dict['clusters_name'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in SIMULATE."
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CLUSTERS clause not allowed in SIMULATE."
        assert args_dict['name'] == None, "BayesDBParsingError: SAVE AS <column_list> clause not allowed in SIMULATE."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR <columns> clause not allowed in SIMULATE."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR <rows> not allowed in SIMULATE."
        assert args_dict['confidence'] == 0, "BayesDBParsingError: CONFIDENCE not allowed in SIMULATE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in SIMULATE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: whereclause not allowed in SIMULATE. Use GIVEN instead."
        for function in functions:
            assert function.function_id == '', "BayesDBParsingError: %s not valid in SIMULATE" % function.function_id

        return 'simulate', \
            dict(tablename=tablename, functions=functions, 
                 newtablename=newtablename, givens=givens, 
                 numpredictions=numpredictions, order_by=order_by, 
                 plot=plot, modelids=modelids, summarize=summarize, hist=hist, freq=freq), \
            dict(filename=filename, plot=plot, scatter=scatter, pairwise=pairwise)

    def parse_estimate(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        assert args_dict['functions'][0] == 'column', "BayesDBParseError: must be ESTIMATE COLUMNS."
        functions = args_dict['functions']
        tablename = args_dict['tablename']
        whereclause = args_dict['whereclause']
        limit = args_dict['limit']
        order_by = args_dict['order_by']
        modelids = args_dict['modelids']
        name = args_dict['name']

        assert args_dict['clusters_name'] == None, "BayesDBParsingError: SAVE CLUSTERS not allowed in estimate columns."
        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in estimate columns."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in estimate columns."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in estimate columns."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in estimate columns."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in estimate columns."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR COLUMNS not allowed in estimate columns."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR ROWS not allowed in estimate columns."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in estimate columns."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in estimated columns."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in estimated columns."
        assert args_dict['threshold'] == None, "BayesDBParsingError: SAVE CLUSTERS not allowed in estimate columns."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in estimate columns."

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in estimate columns."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in estimate columns."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in estimate columns."
        assert client_dict['filename'] == None, "BayesDBParsingError: AS FILE not allowed in estimate columns."

        return 'estimate_columns', \
            dict(tablename=tablename, functions=functions, 
                 whereclause=whereclause, limit=limit, 
                 order_by=order_by, name=name, modelids=modelids), \
            None

    def parse_estimate_pairwise_row(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        functions = args_dict['functions'][0]
        assert len(args_dict['functions']) == 1, "BayesDBParsingError: Only one function allowed in estimate pairwise."
        tablename = args_dict['tablename']
        row_list = args_dict['row_list']
        clusters_name = args_dict['clusters_name']
        threshold = args_dict['threshold']
        modelids = args_dict['modelids']
        filename = client_dict['filename']

        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['column_list'] == None, "BayesDBParsingError: FOR COLUMNS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in ESTIMATE PAIRWISE."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in ESTIMATE PAIRWISE."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: WHERE not allowed in ESTIMATE PAIRWISE."

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in ESTIMATE PAIRWISE."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in ESTIMATE PAIRWISE."

        return 'estimate_pairwise_row', \
            dict(tablename=tablename, function=functions,
                 row_list=row_list, clusters_name=clusters_name,
                 threshold=threshold, modelids=modelids), \
            dict(filename=filename)

    def parse_estimate_pairwise(self,bql_statement_ast):
        method_name, args_dict, client_dict = self.parse_query(bql_statement_ast)
        functions = args_dict['functions']
        assert len(args_dict['functions']) == 1, "BayesDBParsingError: Only one function allowed in estimate pairwise."
        assert functions[0].function_id in ['correlation', 'mutual information', 'dependence probability']
        function_name = functions[0].function_id

        tablename = args_dict['tablename']
        column_list = args_dict['column_list']
        clusters_name = args_dict['clusters_name']
        threshold = args_dict['threshold']
        modelids = args_dict['modelids']
        filename = client_dict['filename']

        assert args_dict['confidence'] == 0, "BayesDBParsingError: WITH CONFIDENCE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['givens'] == None, "BayesDBParsingError: GIVENS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['newtablename'] == None, "BayesDBParsingError: INTO TABLE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numpredictions'] == None, "BayesDBParsingError: TIMES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['numsamples'] == None, "BayesDBParsingError: WITH SAMPLES not allowed in ESTIMATE PAIRWISE."
        assert args_dict['row_list'] == None, "BayesDBParsingError: FOR ROWS not allowed in ESTIMATE PAIRWISE."
        assert args_dict['summarize'] == False, "BayesDBParsingError: SUMMARIZE not allowed in ESTIMATE PAIRWISE."
        assert args_dict['hist'] == False, "BayesDBParsingError: HIST not allowed in ESTIMATE PAIRWISE."
        assert args_dict['freq'] == False, "BayesDBParsingError: FREQ not allowed in ESTIMATE PAIRWISE."
        assert args_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert args_dict['whereclause'] == None, "BayesDBParsingError: whereclause not allowed in ESTIMATE PAIRWISE"

        assert client_dict['plot'] == False, "BayesDBParsingError: PLOT not allowed in ESTIMATE PAIRWISE."
        assert client_dict['scatter'] == False, "BayesDBParsingError: SCATTER not allowed in ESTIMATE PAIRWISE."
        assert client_dict['pairwise'] == False, "BayesDBParsingError: PAIRWISE not allowed in ESTIMATE PAIRWISE."

        return 'estimate_pairwise', \
            dict(tablename=tablename, function_name=function_name,
                 column_list=column_list, clusters_name=clusters_name,
                 threshold=threshold, modelids=modelids), \
            dict(filename=filename)

#####################################################################################
## ------------------------------ Function parsing ------------------------------- ##
#####################################################################################
    def get_args_pred_prob(self, function_group, M_c):
        """
        returns the column index from a predictive probability function
        raises exceptions for unfound columns
        """
        if function_group.column != '' and function_group.column in M_c['name_to_idx']:
            column = function_group.column
            c_idx = M_c['name_to_idx'][column]
            return c_idx
{'fraction_non_alphanumeric': 0.10702141175591345, 'fraction_numerical': 0.0013452412092223758, 'mean_word_length': 3.134007878272959, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4777537', 'n_tokens_mistral': 15750, 'n_tokens_neox': 15514, 'n_words': 3552}