<commit_before># This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import os
import fnmatch
import re
import glob
import json
from flask import Flask, request, jsonify
from smv import SmvApp
from smv.smvapp import DataSetRepoFactory
from shutil import copyfile
import py_compile
import json
from smv.smvdataset import SmvCsvFile
from smv.smvdataset import SmvHiveTable
import ast
import errno

app = Flask(__name__)

TAB_SIZE = 4

# ---------- Helper Functions ---------- #

def getStagesInApp():
    """returns list of all stages defined in app"""
    return list(SmvApp.getInstance().stages)

def getFqnsInApp():
    """returns all known module FQNs in app. Note: excluded links"""
    repo = DataSetRepoFactory(SmvApp.getInstance()).createRepo()
    # generate list of URNs in a stage for each stage (list-of-list)
    urnsLL = [repo.dataSetsForStage(s) for s in getStagesInApp()]
    # flatten the list-of-list to simple list of urns and remove the "mod:" prefix
    urns = [u.split(":")[1] for ul in urnsLL for u in ul]
    return urns

def indentation(tabbed_str):
    '''returns the number of indentation spaces for a given string'''
    no_tabs_str = tabbed_str.expandtabs(TAB_SIZE)
    # if string has only whitespace, return 0 indentation.
    # else return length of string minus len of str with preceding whitespace stripped
    return 0 if no_tabs_str.isspace() else len(no_tabs_str) - len(no_tabs_str.lstrip())

def test_compile_for_errors(fullname):
    '''returns true if a given file has compilation errors. False otherwise'''
    error = None
    try:
        ok = py_compile.compile(fullname, None, None, True)
    except py_compile.PyCompileError,err:
        error = err.msg
    except IOError, e:
        error = e
    else:
        if ok == 0:
            error = 1
    return error

def getPlutoImportsStartEnd(linesOfCodeList, importsSectionName):
    '''retuns start and end lines of a given imports section'''
    blockStart = "###---{}_IMPORTS_START---###".format(importsSectionName)
    blockEnd = "###---{}_IMPORTS_END---###".format(importsSectionName)

    blockStartLine = None
    blockEndLine = None
    for i, line in enumerate(linesOfCodeList):
        if not blockStartLine and line.lstrip().startswith(blockStart):
            blockStartLine = i
            continue
        if blockStartLine is not None and line.lstrip().startswith(blockEnd):
            blockEndLine = i
            break
    return (blockStartLine, blockEndLine)

def getCodeBlockStartEnd(linesOfCodeList, className, blockName):
    '''returns start and end lines of a given method within a given class'''
    blockStartByName = {
        "imports": "###---GLOBAL_IMPORTS_START---###",  # TODO: Remove. use getPlutoImportsStartEnd
        "run": "def run(",
        "description": "def description(",
        "requiresDS": "def requiresDS(",
        "isEphemeral": "def isEphemeral(",
        "path": "def path(",
        "tableName": "def tableName("
    }
    blockEndByName = { "imports": "###---GLOBAL_IMPORTS_END---###" }

    testModuleClassDef = "class {}(".format(className)
    classDefFound = False
    blockStartLine = None
    blockEndLine = None
    classIndentation = None
    blockIndentation = None
    lastNonEmptyLine = None

    classStart = None
    classEnd = None
    blockStartLine = None
    blockEndLine = None
    numLinesOfCode = len(linesOfCodeList)

    # detect module class def
    for i, line in enumerate(linesOfCodeList):
        if blockName == "imports":  # TODO: remove. use getPlutoImportsStartEnd
            if not blockStartLine and line.lstrip().startswith(blockStartByName[blockName]):
                blockStartLine = i
                continue
            if blockStartLine is not None and line.lstrip().startswith(blockEndByName[blockName]):
                blockEndLine = i
                break
        else:
            # detect class start
            if line.lstrip().startswith(testModuleClassDef):
                classDefFound = True
                classStart = i
                classIndentation = indentation(line)
                continue
            # detect method if not already found
            if (classDefFound and blockStartLine is None) and line.lstrip().startswith(blockStartByName[blockName]):
                blockStartLine = i
                blockIndentation = indentation(line)
                lastNonEmptyLine = i
                continue
            # detect end of method
            if blockStartLine is not None and blockEndLine is None:
                # check if eof or next block started
                if (not line.isspace()) and (indentation(line) <= blockIndentation):
                    blockEndLine = lastNonEmptyLine
                elif i == (numLinesOfCode - 1):
                    # if last line before eof is not empty space, current line is end of method, else last non empty
                    blockEndLine = i if not line.isspace() else lastNonEmptyLine
            # detect class end by start of another class (or block with eq or less indentation)
            if (not line.isspace()) and (classDefFound and (indentation(line) <= classIndentation)):
                classEnd = lastNonEmptyLine
                break;
            # detect class end by eof
            if i == (numLinesOfCode - 1) and classDefFound:
                classEnd = i if not line.isspace() else lastNonEmptyLine
            # mark last line with code. Will be end of method body when next block of code or eof found
            lastNonEmptyLine = i if (not line.isspace()) else lastNonEmptyLine

    return (blockStartLine, blockEndLine, classStart, classEnd)

def getDatasetClassStartEnd(lines_of_code_list, module_name):
    '''returns beginning and end lines of a class, and the indentation of its first method'''
    test_module_class_def = "class {}(".format(module_name)
    start = None
    end = None
    methodIndentation = None
    lastNonEmptyLine = None

    # detect module class def
    for i, line in enumerate(lines_of_code_list):
        if start is None and line.lstrip().startswith(test_module_class_def):
            start = i
            continue
        if start is not None:
            # determine indentation of first method # TODO: should check tabsize.. FIXME
            methodDef = re.match(r'^([ \t]+)def', line)
            if methodDef and methodIndentation is None:
                methodIndentation = methodDef.group(1)
                continue
            # check if string not blank and has no indentation, or this is the last line
            if (not line.isspace() and indentation(line) == 0):
                end = lastNonEmptyLine
                break
        lastNonEmptyLine = i if (not line.isspace()) else lastNonEmptyLine
    if start is not None and end is None:
        end = len(lines_of_code_list)
    return (start, end, methodIndentation)


def get_filepath_from_moduleFqn(module_fqn):
    '''given a fqn, returns the fullname of its file'''
    # TODO: do not use hardcoded value... FIXME
    prefix = "/projects/sample_smv_project/src/main/python/"
    # dir1.dir2.file.class => [dir1, dir2, file]
    fqn_dirs_filename_list = module_fqn.split(".")[:-1]
    # concats fqn_dirs_filename_list into string with "/" intermezzo, appends .py, prepends prefix
    filepath = prefix + "/".join(fqn_dirs_filename_list) + ".py"
    return filepath

def get_output_dir():
    '''returns the smv app's output directory'''
    output_dir = SmvApp.getInstance().outputDir()
    if (output_dir.startswith('file://')):
        output_dir = output_dir[7:]
    return output_dir

def get_latest_file_dir(output_dir, module_name, suffix):
    '''
    There could be multiple output snapshots for a module in the output dir, like:
        com.mycompany.myproj.stage1.employment.PythonEmploymentByState_13e82a9b.csv
        com.mycompany.myproj.stage1.employment.PythonEmploymentByState_51d96a9b.csv
        ...
    This function is to find the latest output version for certain module.
    '''
    latest_file_dir = max([f for f in os.listdir(output_dir) \
        if f.startswith(module_name) and f.endswith(suffix)], \
        key=lambda f: os.path.getctime(os.path.join(output_dir, f)))
    return os.path.join(output_dir, latest_file_dir)

def read_file_dir(file_dir, limit=999999999):
    '''
    "file_dir" is the data/schema path of SMV output. The SMV output is split into
    multiple parts and stored under "file_dir" like part-00001, part-00002...
    This function is to read lines of all the files and return them as a list.
    User can use the "limit" parameter to return the first n lines.
    '''
    lines = []
    for file in glob.glob('%s/part-*' % file_dir):
        with open(file, 'rb') as readfile:
            for line in readfile.readlines():
                lines.append(line.rstrip())
                if len(lines) >= limit:
                    break
        if len(lines) >= limit:
            break
    return lines

def get_module_code_file_mapping():
    '''
    This function returns a dictionary where the key is the module name and the
    value is the absolute file path of this module.
    '''
    def get_all_files_with_suffix(path, suffix):
        '''
        This function recurseively searches for all the files with certain
        suffix under certain path and return the absolute file names in a list.
        '''
        matches = []
        for root, dirnames, filenames in os.walk(path):
            for filename in fnmatch.filter(filenames, '*.%s' % suffix):
                matches.append(os.path.join(root, filename))
        return matches

    def get_module_file_mapping(files, patterns):
        module_dict = {}
        # compile the patterns
        patterns = [re.compile(pattern) for pattern in patterns]
        for file in files:
            with open(file, 'rb') as readfile:
                for line in readfile.readlines():
                    # in Python 3, readlines() return a bytes-like object
                    if sys.version >= '3': line = line.decode()
                    for pattern in patterns:
                        m = pattern.search(line)
                        if m:
                            module_name = m.group(1).strip()
                            file_name = file
                            fqn = get_fqn(module_name, file_name)
                            if (fqn):
                                module_dict[fqn] = file_name
        return module_dict

    def get_fqn(module_name, file_name):
        '''
        "module_name" is like "EmploymentByState".
        "file_name" is the absolute file path containing this module, like
            "/xxx/xxx/MyApp/src/main/scala/com/mycompany/myproj/stage1/EmploymentByState.scala".
        This function will return the fqn of the module, like
            "com.mycompany.myproj.stage1.EmploymentByState".
        It will work for both scala and python modules.
        '''
        sep = os.path.sep
        patterns = [
            '(.+?)%s(.+?)$' % sep.join(['src', 'main', 'scala', '']),
            '(.+?)%s(.+?)$' % sep.join(['src', 'main', 'python', '']),
        ]
        for pattern in patterns:
            m = re.search(pattern, file_name)
            if m:
                fqn_split = m.group(2).strip().split(sep)
                if fqn_split[-1].endswith('.scala'):
                    fqn_split.pop()
                elif fqn_split[-1].endswith('.py'):
                    fqn_split[-1] = fqn_split[-1][:-3]
                fqn_split.append(module_name)
                fqn = '.'.join(fqn_split)
                return fqn

    code_dir = os.getcwd() + '/src'
    scala_files = get_all_files_with_suffix(code_dir, 'scala')
    python_files = get_all_files_with_suffix(code_dir, 'py')

    files = scala_files + python_files
    patterns = [
        'object (.+?) extends( )+SmvModule\(',
        'object (.+?) extends( )+SmvCsvFile\(',
        'class (.+?)\(SmvModule',
        'class (.+?)\(SmvCsvFile',
    ]
    module_dict = get_module_file_mapping(files, patterns)
    return module_dict

def err_res(err, err_msg="", res={}): # err, errmsg="", res={}
    '''returns a json object with the default format for error responses'''
    retval = {}
    retval["err"] = err
    retval["err_msg"] = err_msg
    retval["res"] = res
    return jsonify(retval)

def ok_res(res):
    '''returns a json object with the default format for a successful response'''
    retval = {}
    retval["err"] = OK
    retval["res"] = res
    return jsonify(retval)

def get_module_name_from_fqn(fqn):
    '''Returns a string contaiting the basename for a given fqn'''
    return fqn.split(".")[-1:][0]   # a.b.c => [a,b,c] => [c] => c

def getMsnFromFqn(fqn):
    '''based on a fqn, returns an object with the basename, and stage for that given fqn'''
    fqn_split = fqn.split(".")
    stage = ".".join(fqn_split[:-2])
    baseName = fqn_split[-1]
    return { "stage": stage, "baseName": baseName }

# TODO: FIXME: test if works with new smv version
def getDatasetInfo(fqn, baseName, stage):
    '''returns an object containing the properties of a dataset. Namely, fqn, name, stage, isEphemeral, dsType, description,
        requiresDS (modules), tableName (Hive), csvFile (csv)'''
    try:
        module = DataSetRepoFactory(SmvApp.getInstance()).createRepo().loadDataSet(fqn)
    except:
        return None # module does noe exist

    moduleDsType = module.dsType();
    res = {
        "fqn": module.fqn(),
        "moduleName": baseName,
        "stage": stage,
        "ephemeral": module.isEphemeral(),
        "dsType": moduleDsType,
        "description": module.description()
    }

    if moduleDsType.lower() == "input":
        if isinstance(module, SmvCsvFile):
            res["dsInputType"] = 'csv'
            res["inputFile"] = module.path()
        if isinstance(module, SmvHiveTable):
            res["dsInputType"] = 'hive'
            res["tableName"] = module.tableName()
        if not res["dsInputType"]: raise ValueError('dsInputType not supported')
    elif moduleDsType.lower() == "module":
        res["requiresDS"] = map(lambda ds: ds.fqn(), module.requiresDS())
    else:
        raise ValueError("dsType not supported")

    file_name = get_filepath_from_moduleFqn(fqn)
    module_name = get_module_name_from_fqn(fqn)

    # try:
    with open(file_name, 'rb') as f:
        lines_of_code_list = f.readlines()
    file_content = [line.rstrip() for line in lines_of_code_list]
    (run_start, run_end, _, _) = getCodeBlockStartEnd(lines_of_code_list, module_name, 'run')

    # remove indentation from run method body
    raw_run_method_body = file_content[(run_start + 1):(run_end + 1)]
    run_body_indent = indentation(raw_run_method_body[0])
    run_method_body = [line[run_body_indent:] for line in raw_run_method_body]

    res["srcFile"] = file_name,
    res["srcCode"] = run_method_body
    return res


def extractImportsFromFqns(fqns = []):
    '''
    Given a list of fqns, returns a list of file qualified names. E.g.:
    for: fqns = ['a.b.c.d', 'x.y.z'], returns: ['a.b.c', 'x.y'], which are files located in a/b/c.py, and x/y.py
    return list(map(lambda fqn: ".".join(fqn.split(".")[:-1]), fqns))
    '''

def buildImports(className = None, requiresDS = None):
    '''Template for generating the imports section of a dataset'''
    if not className:
        return """\
###---GLOBAL_IMPORTS_START---###
from smv import *
from pyspark.sql.functions import *
###---GLOBAL_IMPORTS_END---###\n"""

    dependencies = extractImportsFromFqns(requiresDS)
    importStatements = "\n".join(map(lambda dep: "import {}".format(dep), dependencies))
    return """\
###---{0}_IMPORTS_START---###
{1}
###---{0}_IMPORTS_END---###\n""".format(className, importStatements)

# class start
def buildClassStart(className, dsType): # what about SmvOutput?
    '''Template for generating the class definition of a dataset'''
    extendsByDsType = { "csv": "SmvCsvFile", "hive": "SmvHiveTable", "module": "SmvModule" }
    extends = extendsByDsType[dsType.lower()]
    return "class {}({}):\n".format(className, extends)

# build description
def buildDescription(description, indentation="\t"):
    '''Template for generating the description method\'s code'''
    return "\
{1}def description(self):\n\
{1}{1}return \"{0}\"\n".format(description, indentation)

def buildRun(dsType, body=None, indentation="\t"):
    '''Template for generating the run method code'''
    if body is None:
        newBody = "{0}{0}return None\n".format(indentation)
    else: # give 2 level indentation to body
        bodyList = body.splitlines(True)
        newBody = "".join(["{0}{0}{1}".format(indentation, line) for line in bodyList])
    arg = "i" if dsType.lower() == "module" else "df"
    return "{2}def run(self, {0}):\n{1}".format(arg, newBody, indentation)

def buildIsEphemeral(ephemeral, indentation="\t"):
    '''Template for generating isEphemeral function code'''
    return "\
{1}def isEphemeral(self):\n\
{1}{1}return {0}\n".format(ephemeral, indentation)

# module
def buildRequiresDS(requiresDS, indentation="\t"):
    '''Template for generating requiresDS\' method code'''
    requires = "" if not requiresDS else ", ".join(map((lambda r: r.encode("utf-8")), requiresDS))
    return "\
{1}def requiresDS(self):\n\
{1}{1}return [{0}]\n".format(requires, indentation)

# csv
def buildPath(path, indentation="\t"):
    '''Template for generating path method code'''
    path = "" if path is None else path
    return "\
{1}def path(self):\n\
{1}{1}return \"{0}\"\n".format(path, indentation)

# hive
def buildTableName(tableName, indentation="\t"):
    '''Template for generating tableName method code'''
    tableName = "" if tableName is None else tableName
    return "\
{1}def tableName(self):\n\
{1}{1}return \"{0}\"\n".format(tableName, indentation)


def generateModuleCode(stages, className, dsType, description, isEphemeral, run=None, requiresDS=None):
    '''Basic template for generating datasets extending SmvModule'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildRequiresDS(requiresDS),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces

def generateCsvCode(stages, className, dsType, description, isEphemeral, run=None, path=None):
    '''Basic template for generating datasets extending SmvCsvFile'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildPath(path),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces

def generateHiveCode(stages, className, dsType, description, isEphemeral, run=None, tableName=None):
    '''Basic template for generating datasets extending SmvHiveTable'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildTableName(tableName),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces


# ---------- API Definition ---------- #

MODULE_NOT_PROVIDED_ERR = 'ERROR: No module name provided!'
MODULE_NOT_FOUND_ERR = 'ERROR: Job failed to run. Please check whether the module name is valid!'
MODULE_ALREADY_EXISTS_ERR = 'ERROR: Module already exists!'
TYPE_NOT_PROVIDED_ERR = 'ERROR: No module type provided!'
TYPE_NOT_SUPPORTED_ERR = 'ERROR: Module type not supported!'
CODE_NOT_PROVIDED_ERR = 'ERROR: No module code provided!'
COMPILATION_ERROR = 'ERROR: Compilation error'
JOB_SUCCESS = 'SUCCESS: Code updated!' # TODO: rename CODE_UPDATE_SUCCESS
OK = ""

MODULE_DOES_NOT_EXIST= 'MODULE_DOES_NOT_EXIST'

@app.route("/api/get_app_info", methods = ['POST'])
def get_app_info():
    '''
    body: empty
    function: retrieve list of stages and fqns in app.
    '''
    res = {
        "stages": getStagesInApp(),
        "fqns": getFqnsInApp()
    }
    return ok_res(res)

@app.route("/api/run_module", methods = ['POST'])<commit_msg>#766: Added functionality to detect start and end of a class' docstring<commit_after># This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import os
import fnmatch
import re
import glob
import json
from flask import Flask, request, jsonify
from smv import SmvApp
from smv.smvapp import DataSetRepoFactory
from shutil import copyfile
import py_compile
import json
from smv.smvdataset import SmvCsvFile
from smv.smvdataset import SmvHiveTable
import ast
import errno

app = Flask(__name__)

TAB_SIZE = 4

# ---------- Helper Functions ---------- #

def getStagesInApp():
    """returns list of all stages defined in app"""
    return list(SmvApp.getInstance().stages)

def getFqnsInApp():
    """returns all known module FQNs in app. Note: excluded links"""
    repo = DataSetRepoFactory(SmvApp.getInstance()).createRepo()
    # generate list of URNs in a stage for each stage (list-of-list)
    urnsLL = [repo.dataSetsForStage(s) for s in getStagesInApp()]
    # flatten the list-of-list to simple list of urns and remove the "mod:" prefix
    urns = [u.split(":")[1] for ul in urnsLL for u in ul]
    return urns

def indentation(tabbed_str):
    '''returns the number of indentation spaces for a given string'''
    no_tabs_str = tabbed_str.expandtabs(TAB_SIZE)
    # if string has only whitespace, return 0 indentation.
    # else return length of string minus len of str with preceding whitespace stripped
    return 0 if no_tabs_str.isspace() else len(no_tabs_str) - len(no_tabs_str.lstrip())

def test_compile_for_errors(fullname):
    '''returns true if a given file has compilation errors. False otherwise'''
    error = None
    try:
        ok = py_compile.compile(fullname, None, None, True)
    except py_compile.PyCompileError,err:
        error = err.msg
    except IOError, e:
        error = e
    else:
        if ok == 0:
            error = 1
    return error

def getPlutoImportsStartEnd(linesOfCodeList, importsSectionName):
    '''retuns start and end lines of a given imports section'''
    blockStart = "###---{}_IMPORTS_START---###".format(importsSectionName)
    blockEnd = "###---{}_IMPORTS_END---###".format(importsSectionName)

    blockStartLine = None
    blockEndLine = None
    for i, line in enumerate(linesOfCodeList):
        if not blockStartLine and line.lstrip().startswith(blockStart):
            blockStartLine = i
            continue
        if blockStartLine is not None and line.lstrip().startswith(blockEnd):
            blockEndLine = i
            break
    return (blockStartLine, blockEndLine)

def getCodeBlockStartEnd(linesOfCodeList, className, blockName):
    '''returns start and end lines of a given method within a given class'''
    blockStartByName = {
        "imports": "###---GLOBAL_IMPORTS_START---###",  # TODO: Remove. use getPlutoImportsStartEnd
        "run": "def run(",
        "description": "def description(",
        "requiresDS": "def requiresDS(",
        "isEphemeral": "def isEphemeral(",
        "path": "def path(",
        "tableName": "def tableName(",
        "*": "def"
    }
    blockEndByName = { "imports": "###---GLOBAL_IMPORTS_END---###" }

    testModuleClassDef = "class {}(".format(className)
    classDefFound = False
    blockStartLine = None
    blockEndLine = None
    classIndentation = None
    blockIndentation = None
    lastNonEmptyLine = None

    classStart = None
    classEnd = None
    blockStartLine = None
    blockEndLine = None
    numLinesOfCode = len(linesOfCodeList)

    # detect module class def
    for i, line in enumerate(linesOfCodeList):
        if blockName == "imports":  # TODO: remove. use getPlutoImportsStartEnd
            if not blockStartLine and line.lstrip().startswith(blockStartByName[blockName]):
                blockStartLine = i
                continue
            if blockStartLine is not None and line.lstrip().startswith(blockEndByName[blockName]):
                blockEndLine = i
                break
        else:
            # detect class start
            if line.lstrip().startswith(testModuleClassDef):
                classDefFound = True
                classStart = i
                classIndentation = indentation(line)
                continue
            # detect method if not already found
            if (classDefFound and blockStartLine is None) and line.lstrip().startswith(blockStartByName[blockName]):
                blockStartLine = i
                blockIndentation = indentation(line)
                lastNonEmptyLine = i
                continue
            # detect end of method
            if blockStartLine is not None and blockEndLine is None:
                # check if eof or next block started
                if (not line.isspace()) and (indentation(line) <= blockIndentation):
                    blockEndLine = lastNonEmptyLine
                elif i == (numLinesOfCode - 1):
                    # if last line before eof is not empty space, current line is end of method, else last non empty
                    blockEndLine = i if not line.isspace() else lastNonEmptyLine
            # detect class end by start of another class (or block with eq or less indentation)
            if (not line.isspace()) and (classDefFound and (indentation(line) <= classIndentation)):
                classEnd = lastNonEmptyLine
                break;
            # detect class end by eof
            if i == (numLinesOfCode - 1) and classDefFound:
                classEnd = i if not line.isspace() else lastNonEmptyLine
            # mark last line with code. Will be end of method body when next block of code or eof found
            lastNonEmptyLine = i if (not line.isspace()) else lastNonEmptyLine

    return (blockStartLine, blockEndLine, classStart, classEnd)

def getDatasetClassStartEnd(lines_of_code_list, module_name):
    '''returns beginning and end lines of a class, and the indentation of its first method'''
    test_module_class_def = "class {}(".format(module_name)
    start = None
    end = None
    methodIndentation = None
    lastNonEmptyLine = None

    # detect module class def
    for i, line in enumerate(lines_of_code_list):
        if start is None and line.lstrip().startswith(test_module_class_def):
            start = i
            continue
        if start is not None:
            # determine indentation of first method # TODO: should check tabsize.. FIXME
            methodDef = re.match(r'^([ \t]+)def', line)
            if methodDef and methodIndentation is None:
                methodIndentation = methodDef.group(1)
                continue
            # check if string not blank and has no indentation, or this is the last line
            if (not line.isspace() and indentation(line) == 0):
                end = lastNonEmptyLine
                break
        lastNonEmptyLine = i if (not line.isspace()) else lastNonEmptyLine
    if start is not None and end is None:
        end = len(lines_of_code_list)
    return (start, end, methodIndentation)

def getDatasetDescriptionStartEnd(linesOfCodeList, className):
    '''
    returns a datasets description start and end line numbers
    '''
    # detect first method definition and class
    (mStart, _, cStart, cEnd) = getCodeBlockStartEnd(linesOfCodeList, className, "*")
    if not cStart:
        return (None, None)
    (start, end) = (None, None) # placeholders for start and end of description
    classDefinitionEnd = None
    for i, line in enumerate(linesOfCodeList):
        if not classDefinitionEnd:
            # detect end of class definition, exclude comments after it
            if line.startswith('#'): continue
            lineSplitAtComment = line.split("#")
            nonCommentStr = lineSplitAtComment[0]
            if not classDefinitionEnd and nonCommentStr.rstrip().endswith(":"):
                classDefinitionEnd = True
                continue
        lstrippedLine = line.lstrip()
        if not start and (lstrippedLine.startswith("\"\"\"") or lstrippedLine.startswith("'''")):
            start = i

        if start:
            uncommentedLine = line.split("#")[0].rstrip()
            if uncommentedLine.endswith("\"\"\"") or uncommentedLine.endswith("'''"):
                # if i is start, then """ must have two occurrences for i to be beginning and end
                if start == i and not (uncommentedLine.count("\"\"\"") == 2 or uncommentedLine.count("'''") == 2):
                    continue
                end = i
                break
        # line is already at first method, so there is no docstring
        if i >= mStart: return (None, None)
    return (start, end)


def get_filepath_from_moduleFqn(module_fqn):
    '''given a fqn, returns the fullname of its file'''
    # TODO: do not use hardcoded value... FIXME
    prefix = "/projects/sample_smv_project/src/main/python/"
    # dir1.dir2.file.class => [dir1, dir2, file]
    fqn_dirs_filename_list = module_fqn.split(".")[:-1]
    # concats fqn_dirs_filename_list into string with "/" intermezzo, appends .py, prepends prefix
    filepath = prefix + "/".join(fqn_dirs_filename_list) + ".py"
    return filepath

def get_output_dir():
    '''returns the smv app's output directory'''
    output_dir = SmvApp.getInstance().outputDir()
    if (output_dir.startswith('file://')):
        output_dir = output_dir[7:]
    return output_dir

def get_latest_file_dir(output_dir, module_name, suffix):
    '''
    There could be multiple output snapshots for a module in the output dir, like:
        com.mycompany.myproj.stage1.employment.PythonEmploymentByState_13e82a9b.csv
        com.mycompany.myproj.stage1.employment.PythonEmploymentByState_51d96a9b.csv
        ...
    This function is to find the latest output version for certain module.
    '''
    latest_file_dir = max([f for f in os.listdir(output_dir) \
        if f.startswith(module_name) and f.endswith(suffix)], \
        key=lambda f: os.path.getctime(os.path.join(output_dir, f)))
    return os.path.join(output_dir, latest_file_dir)

def read_file_dir(file_dir, limit=999999999):
    '''
    "file_dir" is the data/schema path of SMV output. The SMV output is split into
    multiple parts and stored under "file_dir" like part-00001, part-00002...
    This function is to read lines of all the files and return them as a list.
    User can use the "limit" parameter to return the first n lines.
    '''
    lines = []
    for file in glob.glob('%s/part-*' % file_dir):
        with open(file, 'rb') as readfile:
            for line in readfile.readlines():
                lines.append(line.rstrip())
                if len(lines) >= limit:
                    break
        if len(lines) >= limit:
            break
    return lines

def get_module_code_file_mapping():
    '''
    This function returns a dictionary where the key is the module name and the
    value is the absolute file path of this module.
    '''
    def get_all_files_with_suffix(path, suffix):
        '''
        This function recurseively searches for all the files with certain
        suffix under certain path and return the absolute file names in a list.
        '''
        matches = []
        for root, dirnames, filenames in os.walk(path):
            for filename in fnmatch.filter(filenames, '*.%s' % suffix):
                matches.append(os.path.join(root, filename))
        return matches

    def get_module_file_mapping(files, patterns):
        module_dict = {}
        # compile the patterns
        patterns = [re.compile(pattern) for pattern in patterns]
        for file in files:
            with open(file, 'rb') as readfile:
                for line in readfile.readlines():
                    # in Python 3, readlines() return a bytes-like object
                    if sys.version >= '3': line = line.decode()
                    for pattern in patterns:
                        m = pattern.search(line)
                        if m:
                            module_name = m.group(1).strip()
                            file_name = file
                            fqn = get_fqn(module_name, file_name)
                            if (fqn):
                                module_dict[fqn] = file_name
        return module_dict

    def get_fqn(module_name, file_name):
        '''
        "module_name" is like "EmploymentByState".
        "file_name" is the absolute file path containing this module, like
            "/xxx/xxx/MyApp/src/main/scala/com/mycompany/myproj/stage1/EmploymentByState.scala".
        This function will return the fqn of the module, like
            "com.mycompany.myproj.stage1.EmploymentByState".
        It will work for both scala and python modules.
        '''
        sep = os.path.sep
        patterns = [
            '(.+?)%s(.+?)$' % sep.join(['src', 'main', 'scala', '']),
            '(.+?)%s(.+?)$' % sep.join(['src', 'main', 'python', '']),
        ]
        for pattern in patterns:
            m = re.search(pattern, file_name)
            if m:
                fqn_split = m.group(2).strip().split(sep)
                if fqn_split[-1].endswith('.scala'):
                    fqn_split.pop()
                elif fqn_split[-1].endswith('.py'):
                    fqn_split[-1] = fqn_split[-1][:-3]
                fqn_split.append(module_name)
                fqn = '.'.join(fqn_split)
                return fqn

    code_dir = os.getcwd() + '/src'
    scala_files = get_all_files_with_suffix(code_dir, 'scala')
    python_files = get_all_files_with_suffix(code_dir, 'py')

    files = scala_files + python_files
    patterns = [
        'object (.+?) extends( )+SmvModule\(',
        'object (.+?) extends( )+SmvCsvFile\(',
        'class (.+?)\(SmvModule',
        'class (.+?)\(SmvCsvFile',
    ]
    module_dict = get_module_file_mapping(files, patterns)
    return module_dict

def err_res(err, err_msg="", res={}): # err, errmsg="", res={}
    '''returns a json object with the default format for error responses'''
    retval = {}
    retval["err"] = err
    retval["err_msg"] = err_msg
    retval["res"] = res
    return jsonify(retval)

def ok_res(res):
    '''returns a json object with the default format for a successful response'''
    retval = {}
    retval["err"] = OK
    retval["res"] = res
    return jsonify(retval)

def get_module_name_from_fqn(fqn):
    '''Returns a string contaiting the basename for a given fqn'''
    return fqn.split(".")[-1:][0]   # a.b.c => [a,b,c] => [c] => c

def getMsnFromFqn(fqn):
    '''based on a fqn, returns an object with the basename, and stage for that given fqn'''
    fqn_split = fqn.split(".")
    stage = ".".join(fqn_split[:-2])
    baseName = fqn_split[-1]
    return { "stage": stage, "baseName": baseName }

# TODO: FIXME: test if works with new smv version
def getDatasetInfo(fqn, baseName, stage):
    '''returns an object containing the properties of a dataset. Namely, fqn, name, stage, isEphemeral, dsType, description,
        requiresDS (modules), tableName (Hive), csvFile (csv)'''
    try:
        module = DataSetRepoFactory(SmvApp.getInstance()).createRepo().loadDataSet(fqn)
    except:
        return None # module does noe exist

    moduleDsType = module.dsType();
    res = {
        "fqn": module.fqn(),
        "moduleName": baseName,
        "stage": stage,
        "ephemeral": module.isEphemeral(),
        "dsType": moduleDsType,
        "description": module.description()
    }

    if moduleDsType.lower() == "input":
        if isinstance(module, SmvCsvFile):
            res["dsInputType"] = 'csv'
            res["inputFile"] = module.path()
        if isinstance(module, SmvHiveTable):
            res["dsInputType"] = 'hive'
            res["tableName"] = module.tableName()
        if not res["dsInputType"]: raise ValueError('dsInputType not supported')
    elif moduleDsType.lower() == "module":
        res["requiresDS"] = map(lambda ds: ds.fqn(), module.requiresDS())
    else:
        raise ValueError("dsType not supported")

    file_name = get_filepath_from_moduleFqn(fqn)
    module_name = get_module_name_from_fqn(fqn)

    # try:
    with open(file_name, 'rb') as f:
        lines_of_code_list = f.readlines()
    file_content = [line.rstrip() for line in lines_of_code_list]
    (run_start, run_end, _, _) = getCodeBlockStartEnd(lines_of_code_list, module_name, 'run')

    # remove indentation from run method body
    raw_run_method_body = file_content[(run_start + 1):(run_end + 1)]
    run_body_indent = indentation(raw_run_method_body[0])
    run_method_body = [line[run_body_indent:] for line in raw_run_method_body]

    res["srcFile"] = file_name,
    res["srcCode"] = run_method_body
    return res


def extractImportsFromFqns(fqns = []):
    '''
    Given a list of fqns, returns a list of file qualified names. E.g.:
    for: fqns = ['a.b.c.d', 'x.y.z'], returns: ['a.b.c', 'x.y'], which are files located in a/b/c.py, and x/y.py
    return list(map(lambda fqn: ".".join(fqn.split(".")[:-1]), fqns))
    '''

def buildImports(className = None, requiresDS = None):
    '''Template for generating the imports section of a dataset'''
    if not className:
        return """\
###---GLOBAL_IMPORTS_START---###
from smv import *
from pyspark.sql.functions import *
###---GLOBAL_IMPORTS_END---###\n"""

    dependencies = extractImportsFromFqns(requiresDS)
    importStatements = "\n".join(map(lambda dep: "import {}".format(dep), dependencies))
    return """\
###---{0}_IMPORTS_START---###
{1}
###---{0}_IMPORTS_END---###\n""".format(className, importStatements)

# class start
def buildClassStart(className, dsType): # what about SmvOutput?
    '''Template for generating the class definition of a dataset'''
    extendsByDsType = { "csv": "SmvCsvFile", "hive": "SmvHiveTable", "module": "SmvModule" }
    extends = extendsByDsType[dsType.lower()]
    return "class {}({}):\n".format(className, extends)

# build description
def buildDescription(description, indentation="\t"):
    '''Template for generating the description method\'s code'''
    return "\
{1}def description(self):\n\
{1}{1}return \"{0}\"\n".format(description, indentation)

def buildRun(dsType, body=None, indentation="\t"):
    '''Template for generating the run method code'''
    if body is None:
        newBody = "{0}{0}return None\n".format(indentation)
    else: # give 2 level indentation to body
        bodyList = body.splitlines(True)
        newBody = "".join(["{0}{0}{1}".format(indentation, line) for line in bodyList])
    arg = "i" if dsType.lower() == "module" else "df"
    return "{2}def run(self, {0}):\n{1}".format(arg, newBody, indentation)

def buildIsEphemeral(ephemeral, indentation="\t"):
    '''Template for generating isEphemeral function code'''
    return "\
{1}def isEphemeral(self):\n\
{1}{1}return {0}\n".format(ephemeral, indentation)

# module
def buildRequiresDS(requiresDS, indentation="\t"):
    '''Template for generating requiresDS\' method code'''
    requires = "" if not requiresDS else ", ".join(map((lambda r: r.encode("utf-8")), requiresDS))
    return "\
{1}def requiresDS(self):\n\
{1}{1}return [{0}]\n".format(requires, indentation)

# csv
def buildPath(path, indentation="\t"):
    '''Template for generating path method code'''
    path = "" if path is None else path
    return "\
{1}def path(self):\n\
{1}{1}return \"{0}\"\n".format(path, indentation)

# hive
def buildTableName(tableName, indentation="\t"):
    '''Template for generating tableName method code'''
    tableName = "" if tableName is None else tableName
    return "\
{1}def tableName(self):\n\
{1}{1}return \"{0}\"\n".format(tableName, indentation)


def generateModuleCode(stages, className, dsType, description, isEphemeral, run=None, requiresDS=None):
    '''Basic template for generating datasets extending SmvModule'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildRequiresDS(requiresDS),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces

def generateCsvCode(stages, className, dsType, description, isEphemeral, run=None, path=None):
    '''Basic template for generating datasets extending SmvCsvFile'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildPath(path),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces

def generateHiveCode(stages, className, dsType, description, isEphemeral, run=None, tableName=None):
    '''Basic template for generating datasets extending SmvHiveTable'''
    return "{}\n{}{}\n{}\n{}\n{}\n".format(
        buildImports(),  # should be list of stages, not hardcoded
        buildClassStart(className, dsType),
        buildDescription(description),
        buildTableName(tableName),
        buildIsEphemeral(isEphemeral),
        buildRun(dsType, run)).expandtabs(TAB_SIZE) # ...and turn tabs to spaces


# ---------- API Definition ---------- #

MODULE_NOT_PROVIDED_ERR = 'ERROR: No module name provided!'
MODULE_NOT_FOUND_ERR = 'ERROR: Job failed to run. Please check whether the module name is valid!'
MODULE_ALREADY_EXISTS_ERR = 'ERROR: Module already exists!'
TYPE_NOT_PROVIDED_ERR = 'ERROR: No module type provided!'
TYPE_NOT_SUPPORTED_ERR = 'ERROR: Module type not supported!'
CODE_NOT_PROVIDED_ERR = 'ERROR: No module code provided!'
COMPILATION_ERROR = 'ERROR: Compilation error'
JOB_SUCCESS = 'SUCCESS: Code updated!' # TODO: rename CODE_UPDATE_SUCCESS
OK = ""

MODULE_DOES_NOT_EXIST= 'MODULE_DOES_NOT_EXIST'

@app.route("/api/test_description", methods = ['POST'])
def testDescription():
    fqn = request.form['fqn'].encode("utf-8")
    fileName = get_filepath_from_moduleFqn(fqn)
    msn = getMsnFromFqn(fqn)

    try:
        with open(fileName, 'r') as f:
            linesOfCodeList = f.readlines()
    except IOError:
        return err_res("Failed to get src code")

    (start, end) = getDatasetDescriptionStartEnd(linesOfCodeList, msn["baseName"])

    return jsonify({ "start": start, "end": end})



@app.route("/api/get_app_info", methods = ['POST'])
def get_app_info():
    '''
    body: empty
    function: retrieve list of stages and fqns in app.
    '''
    res = {
        "stages": getStagesInApp(),
        "fqns": getFqnsInApp()
    }
    return ok_res(res)

@app.route("/api/run_module", methods = ['POST'])
{'fraction_non_alphanumeric': 0.11771670575377399, 'fraction_numerical': 0.004872606389034358, 'mean_word_length': 2.7976653696498053, 'pattern_counts': {'":': 54, '<': 7, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5920188', 'n_tokens_mistral': 13480, 'n_tokens_neox': 13011, 'n_words': 3843}