<commit_before>#!/usr/bin/python
#
#  Copyright 2002-2019 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

# -*- coding: utf-8 -*-

"""
PyCOMPSs Binding - Interactive Mode Helpers
===========================================
Provides auxiliary methods for the interactive mode.
"""

import os

# Debug mode
# Changed to true from interactive.py if specified by the user when starting the runtime
# Enables the explicit prints
DEBUG = False


SEPARATORS = {'globals_separator': "### GLOBALS ###",      # for user defined lines in the entire/global scope
              'classes_separator': '### CLASSES ###',      # for user defined classes
              'functions_separator': "### FUNCTIONS ###",  # for user defined functions (that are not decorated)
              'tasks_separator': "### TASKS ###"}          # for user defined tasks


# ###################################################################
# ################# MAIN FUNCTION ###################################
# ###################################################################


def update_tasks_code_file(f, file_path):
    """
    Main interactive helper function.
    Analyses the user code that has been executed and parses it looking for:
        - imports
        - tasks
        - functions
    Builds a file where the necessary contents for the worker are.
    Also updates the old code with the new if functions or tasks are redefined.

    :param f: new task function
    :param file_path: file where the code is stored
    :return: None
    """

    if not os.path.exists(file_path):
        _create_tasks_code_file(file_path)

    if DEBUG:
        print("Task definition detected.")

    # Intercept the code
    imports = _get_ipython_imports()      # [import\n, import\n, ...]
    global_code = _get_ipython_globals()  # [var\n, var\n, ...]
    classes_code = _get_classes()         # {'name': str(line\nline\n...)}
    functions_code = _get_functions()     # {'name': str(line\nline\n...)}
    task_code = _get_task_code(f)         # {'name': str(line\nline\n...)}
    old_code = _get_old_code(file_path)   # Next line defines the structure of old_code
    # {'imports':[import\n, import\n, ...], 'tasks':{'name':str(line\nline\n...), 'name':str(line\nline\n...), ...}}

    # Look for new/modified pieces of code. Compares the existing code with the new additions.
    new_imports = _update_imports(imports, old_code['imports'])
    new_globals = _update_globals(global_code, old_code['globals'])
    new_classes = _update_classes(classes_code, old_code['classes'])
    new_functions = _update_functions(functions_code, old_code['functions'])
    new_tasks = _update_tasks(task_code, old_code['tasks'])

    # Update the file where the code is stored.
    _update_code_file(new_imports, new_globals, new_classes, new_functions, new_tasks, file_path)


# ###################################################################
# ############### AUXILIAR METHODS ##################################
# ###################################################################

# CODE INTERCEPTION FUNCTIONS

def _create_tasks_code_file(file_path):
    """
    Creates a file where to store the user code.

    :param file_path: File location and name
    :return: None
    """

    user_code_file = open(file_path, 'a')
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['globals_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['classes_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['functions_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['tasks_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.close()


def _get_raw_code():
    """
    Retrieve the raw code from interactive session.

    :return: the list of the blocks defined by the user that are currently loaded in globals
    """
    import IPython
    ipython = IPython.get_ipython()
    raw_code = ipython.user_ns['In']
    return raw_code


def _get_ipython_imports():
    """
    Finds the user imports.

    :return: A list of imports: [import\n, import\n, ...]
    """

    raw_code = _get_raw_code()
    imports = []
    for i in raw_code:
        # Each i can have more than one line (jupyter-notebook block)
        # We only get the lines that start with from or import and do not
        # have blank spaces before.
        lines = i.split('\n')
        for l in lines:
            if l.startswith("from") or l.startswith("import"):
                imports.append(l + '\n')
    return imports


def _get_ipython_globals():
    """
    Finds the user global variables.

    :return: A list of lines: [var\n, var\n, ...]
    """

    raw_code = _get_raw_code()
    glob_lines = {}
    for i in raw_code:
        # Each i can have more than one line (jupyter-notebook block)
        # We only get the lines that start with from or import and do not
        # have blank spaces before.
        lines = i.split('\n')
        found_one = False
        for l in lines:
            # if the line starts without spaces and is a variable assignation
            glob_name = ''
            if not (l.startswith(' ') or l.startswith('\t')) and _is_variable_assignation(l):
                glob_name = l.split()[0]
                glob_lines[glob_name] = l.strip()
                found_one = True
                continue
            # if the next line/s start with space or tab belong also to the global variable
            if found_one and (l.startswith(' ') or l.startswith('\t')):
                # It is a multiple lines global variable definition
                glob_lines[glob_name] += l.strip()
            else:
                found_one = False
    return glob_lines


def _is_variable_assignation(line):
    """
    This function is used to check if a line of code represents a variable assignation:
    * if contains a '=' (assignation) and does not start with import, nor @, nor def, nor class.
    * then it is ==> is a global variable assignation.

    :param line: Line to parse
    :return: <Boolean>
    """

    if '=' in line:
        parts = line.split()
        if not (line.startswith("from") or
                line.startswith("import") or
                line.startswith("@") or
                line.startswith("def") or
                line.startswith("class")) \
                and len(parts) >= 3 and parts[1] == '=':
            # It is actually an assignation
            return True
        else:
            # It is an import/function/decorator/class definition
            return False
    else:
        # Not an assignation if does not contain '='
        return False


def _get_classes():
    """
    Finds the user defined classes in the code.

    :return: A dictionary with the user classes code: {'name': str(line\nline\n...)}
    """

    raw_code = _get_raw_code()
    classes = {}
    for block in raw_code:
        lines = block.split('\n')
        # Look for classes in the block
        class_found = False
        for line in lines:<commit_msg>Ignore global assignations with contain a call to a master api call.<commit_after>#!/usr/bin/python
#
#  Copyright 2002-2019 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

# -*- coding: utf-8 -*-

"""
PyCOMPSs Binding - Interactive Mode Helpers
===========================================
Provides auxiliary methods for the interactive mode.
"""

import os

# Debug mode
# Changed to true from interactive.py if specified by the user when starting the runtime
# Enables the explicit prints
DEBUG = False


SEPARATORS = {'globals_separator': "### GLOBALS ###",      # for user defined lines in the entire/global scope
              'classes_separator': '### CLASSES ###',      # for user defined classes
              'functions_separator': "### FUNCTIONS ###",  # for user defined functions (that are not decorated)
              'tasks_separator': "### TASKS ###"}          # for user defined tasks


# ###################################################################
# ################# MAIN FUNCTION ###################################
# ###################################################################


def update_tasks_code_file(f, file_path):
    """
    Main interactive helper function.
    Analyses the user code that has been executed and parses it looking for:
        - imports
        - tasks
        - functions
    Builds a file where the necessary contents for the worker are.
    Also updates the old code with the new if functions or tasks are redefined.

    :param f: new task function
    :param file_path: file where the code is stored
    :return: None
    """

    if not os.path.exists(file_path):
        _create_tasks_code_file(file_path)

    if DEBUG:
        print("Task definition detected.")

    # Intercept the code
    imports = _get_ipython_imports()      # [import\n, import\n, ...]
    global_code = _get_ipython_globals()  # [var\n, var\n, ...]
    classes_code = _get_classes()         # {'name': str(line\nline\n...)}
    functions_code = _get_functions()     # {'name': str(line\nline\n...)}
    task_code = _get_task_code(f)         # {'name': str(line\nline\n...)}
    old_code = _get_old_code(file_path)   # Next line defines the structure of old_code
    # {'imports':[import\n, import\n, ...], 'tasks':{'name':str(line\nline\n...), 'name':str(line\nline\n...), ...}}

    # Look for new/modified pieces of code. Compares the existing code with the new additions.
    new_imports = _update_imports(imports, old_code['imports'])
    new_globals = _update_globals(global_code, old_code['globals'])
    new_classes = _update_classes(classes_code, old_code['classes'])
    new_functions = _update_functions(functions_code, old_code['functions'])
    new_tasks = _update_tasks(task_code, old_code['tasks'])

    # Update the file where the code is stored.
    _update_code_file(new_imports, new_globals, new_classes, new_functions, new_tasks, file_path)


# ###################################################################
# ############### AUXILIAR METHODS ##################################
# ###################################################################

# CODE INTERCEPTION FUNCTIONS

def _create_tasks_code_file(file_path):
    """
    Creates a file where to store the user code.

    :param file_path: File location and name
    :return: None
    """

    user_code_file = open(file_path, 'a')
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['globals_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['classes_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['functions_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.write(SEPARATORS['tasks_separator'] + "\n")
    user_code_file.write('\n')
    user_code_file.close()


def _get_raw_code():
    """
    Retrieve the raw code from interactive session.

    :return: the list of the blocks defined by the user that are currently loaded in globals
    """
    import IPython
    ipython = IPython.get_ipython()
    raw_code = ipython.user_ns['In']
    return raw_code


def _get_ipython_imports():
    """
    Finds the user imports.

    :return: A list of imports: [import\n, import\n, ...]
    """

    raw_code = _get_raw_code()
    imports = []
    for i in raw_code:
        # Each i can have more than one line (jupyter-notebook block)
        # We only get the lines that start with from or import and do not
        # have blank spaces before.
        lines = i.split('\n')
        for l in lines:
            if l.startswith("from") or l.startswith("import"):
                imports.append(l + '\n')
    return imports


def _get_ipython_globals():
    """
    Finds the user global variables.

    WARNING: Assignations using any of the master api calls will be ignored
             in order to avoid the worker to try to call the runtime.
             This is checked in the _is_variable_assignation function.

    :return: A list of lines: [var\n, var\n, ...]
    """

    raw_code = _get_raw_code()
    glob_lines = {}
    for i in raw_code:
        # Each i can have more than one line (jupyter-notebook block)
        # We only get the lines that start with from or import and do not
        # have blank spaces before.
        lines = i.split('\n')
        found_one = False
        for l in lines:
            # if the line starts without spaces and is a variable assignation
            glob_name = ''
            if not (l.startswith(' ') or l.startswith('\t')) and _is_variable_assignation(l):
                glob_name = l.split()[0]
                glob_lines[glob_name] = l.strip()
                found_one = True
                continue
            # if the next line/s start with space or tab belong also to the global variable
            if found_one and (l.startswith(' ') or l.startswith('\t')):
                # It is a multiple lines global variable definition
                glob_lines[glob_name] += l.strip()
            else:
                found_one = False
    return glob_lines


def _is_variable_assignation(line):
    """
    This function is used to check if a line of code represents a variable assignation:
    * if contains a '=' (assignation) and does not start with import, nor @, nor def, nor class.
    * then it is ==> is a global variable assignation.

    WARNING; It will ignore the assignations that contain any of the master api calls that
             return something.

    :param line: Line to parse
    :return: <Boolean>
    """
    api_calls = ['compss_open',
                 'compss_delete_file',
                 'compss_wait_on_file',
                 'compss_delete_object',
                 'compss_wait_on']

    if '=' in line:
        if any(call in line for call in api_calls):
            return False
        else:
            # It is an assignation that does not contain an apy call
            parts = line.split()
            if not (line.startswith("from") or
                    line.startswith("import") or
                    line.startswith("@") or
                    line.startswith("def") or
                    line.startswith("class")) \
                    and len(parts) >= 3 and parts[1] == '=':
                # It is actually an assignation
                return True
            else:
                # It is an import/function/decorator/class definition
                return False
    else:
        # Not an assignation if does not contain '='
        return False


def _get_classes():
    """
    Finds the user defined classes in the code.

    :return: A dictionary with the user classes code: {'name': str(line\nline\n...)}
    """

    raw_code = _get_raw_code()
    classes = {}
    for block in raw_code:
        lines = block.split('\n')
        # Look for classes in the block
        class_found = False
        for line in lines:
{'fraction_non_alphanumeric': 0.14901960784313725, 'fraction_numerical': 0.0019919078742608154, 'mean_word_length': 2.6857077311309934, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3526595', 'n_tokens_mistral': 4671, 'n_tokens_neox': 4558, 'n_words': 1482}