<commit_before>import logging
import sys
import os
import traceback
import re
# Local modules
from fortls.parse_fortran import process_file, read_use_stmt, detect_fixed_format, \
    detect_comment_start
from fortls.objects import find_in_scope, fortran_meth, get_use_tree

log = logging.getLogger(__name__)
PY3K = sys.version_info >= (3, 0)
# Global regexes
FORTRAN_EXT_REGEX = re.compile(r'^\.F(77|90|95|03|08|OR|PP)?$', re.I)
OBJBREAK_REGEX = re.compile(r'[\/\-(.,+*<>=$: ]', re.I)
WORD_REGEX = re.compile(r'[a-z_][a-z0-9_]*', re.I)
CALL_REGEX = re.compile(r'[ \t]*CALL[ \t]+[a-z0-9_%]*$', re.I)
TYPE_STMNT_REGEX = re.compile(r'[ \t]*(TYPE|CLASS)[ \t]*(IS)?[ \t]*[a-z0-9_]*$', re.I)
FIXED_CONT_REGEX = re.compile(r'(     [\S])')
FREE_OPT_CONT_REGEX = re.compile(r'([ \t]*&)')


def path_from_uri(uri):
    # Convert file uri to path (strip html like head part)
    if not uri.startswith("file://"):
        return uri
    if os.name == "nt":
        _, path = uri.split("file:///", 1)
    else:
        _, path = uri.split("file://", 1)
    return path


def path_to_uri(path):
    # Convert path to file uri (add html like head part)
    if os.name == "nt":
        return "file:///"+path
    else:
        return "file://"+path


def init_file(filepath):
    #
    if PY3K:
        with open(filepath, 'r', encoding="utf-8") as fhandle:
            contents_split = fhandle.readlines()
    else:
        with open(filepath, 'r') as fhandle:
            contents_split = fhandle.readlines()
    #
    try:
        fixed_flag = detect_fixed_format(contents_split)
        ast_new = process_file(contents_split, True, filepath, fixed_flag)
    except:
        return None
    else:
        # Construct new file object and add to workspace
        tmp_obj = {
            "contents": contents_split,
            "ast": ast_new,
            "fixed": fixed_flag
        }
        return tmp_obj


def tokenize_line(line):
    var_list = []
    paren_stack = [[]]
    paren_count = [0]
    iCurr = -1
    for (i, character) in enumerate(line):
        if character == "(":
            iCurr += 1
            if len(paren_stack) < iCurr+1:
                paren_stack.append([])
                paren_count.append(0)
            paren_stack[iCurr].append([i, -1])
        elif character == ")":
            paren_stack[iCurr][-1][1] = i
            iCurr -= 1
    #
    if len(paren_stack[0]) == 0:
        var_list.append([[[[0, -1]], line]])
    else:
        var_groups = [[] for i in range(len(paren_stack)+1)]
        var_list, out_len = paren_split(line, paren_stack, 0, var_groups)
    return var_list


def paren_split(line, paren_groups, level, var_groups, i1=0, i2=-1):
    tmp_list = []
    tmp_str = ''
    nline = len(line)
    if i2 < 0:
        i2 = nline
    i0 = i1
    for paren_group in paren_groups[level]:
        pg0 = paren_group[0]
        pg1 = paren_group[1]
        if pg0 < i1 or pg1 > i2:
            continue
        tmp_str += line[i0:pg0]
        tmp_list.append([i0, pg0])
        if pg1 < 0:
            pg1 = nline
        if len(paren_groups) > level+1:
            var_groups, out_len = \
                paren_split(line, paren_groups, level+1,
                            var_groups, pg0+1, pg1)
            if out_len == 0:
                var_groups[level+1].append([[[pg0+1, pg1]], line[pg0+1:pg1]])
        else:
            var_groups[level+1].append([[[pg0+1, pg1]], line[pg0+1:pg1]])
        if pg1 == nline:
            continue
        i0 = pg1+1
    #
    if i0 > tmp_list[-1][1]:
        tmp_str += line[i0:i2]
        tmp_list.append([i0, i2])
    out_list = [[tmp_list, tmp_str]]
    var_groups[level].extend(out_list)
    return var_groups, len(out_list)


def get_var_stack(line):
    if len(line) == 0:
        return None
    var_list = tokenize_line(line)
    deepest_var = None
    final_var = None
    final_paren = None
    deepest_paren = None
    n = len(line)
    for var_group in var_list:
        for var_tmp in var_group:
            for parens in var_tmp[0]:
                if n >= parens[0]:
                    if n <= parens[1]:
                        final_var = var_tmp[1]
                        final_paren = parens
                        break
                    elif parens[1] == -1:
                        deepest_var = var_tmp[1]
                        deepest_paren = parens
    if final_var is None:
        if deepest_var is not None:<commit_msg>Improve line segmentation from parentheses<commit_after>import logging
import sys
import os
import traceback
import re
# Local modules
from fortls.parse_fortran import process_file, read_use_stmt, detect_fixed_format, \
    detect_comment_start
from fortls.objects import find_in_scope, fortran_meth, get_use_tree

log = logging.getLogger(__name__)
PY3K = sys.version_info >= (3, 0)
# Global regexes
FORTRAN_EXT_REGEX = re.compile(r'^\.F(77|90|95|03|08|OR|PP)?$', re.I)
OBJBREAK_REGEX = re.compile(r'[\/\-(.,+*<>=$: ]', re.I)
WORD_REGEX = re.compile(r'[a-z_][a-z0-9_]*', re.I)
CALL_REGEX = re.compile(r'[ \t]*CALL[ \t]+[a-z0-9_%]*$', re.I)
TYPE_STMNT_REGEX = re.compile(r'[ \t]*(TYPE|CLASS)[ \t]*(IS)?[ \t]*[a-z0-9_]*$', re.I)
FIXED_CONT_REGEX = re.compile(r'(     [\S])')
FREE_OPT_CONT_REGEX = re.compile(r'([ \t]*&)')


def path_from_uri(uri):
    # Convert file uri to path (strip html like head part)
    if not uri.startswith("file://"):
        return uri
    if os.name == "nt":
        _, path = uri.split("file:///", 1)
    else:
        _, path = uri.split("file://", 1)
    return path


def path_to_uri(path):
    # Convert path to file uri (add html like head part)
    if os.name == "nt":
        return "file:///"+path
    else:
        return "file://"+path


def init_file(filepath):
    #
    if PY3K:
        with open(filepath, 'r', encoding="utf-8") as fhandle:
            contents_split = fhandle.readlines()
    else:
        with open(filepath, 'r') as fhandle:
            contents_split = fhandle.readlines()
    #
    try:
        fixed_flag = detect_fixed_format(contents_split)
        ast_new = process_file(contents_split, True, filepath, fixed_flag)
    except:
        return None
    else:
        # Construct new file object and add to workspace
        tmp_obj = {
            "contents": contents_split,
            "ast": ast_new,
            "fixed": fixed_flag
        }
        return tmp_obj


def tokenize_line(line):
    paren_list = [[[-1, len(line)]], []]
    level = 1
    in_string = False
    string_char = ""
    for i, char in enumerate(line):
        if in_string:
            if char == string_char:
                in_string = False
            continue
        if char == '(':
            paren_list[level].append([i, len(line)])
            level += 1
            if len(paren_list) < level+1:
                paren_list.append([])
        elif char == ')':
            paren_list[level-1][-1][1] = i
            level -= 1
        elif char == "'" or char == '"':
            in_string = True
            string_char = char
    return paren_split(line, paren_list[:-1])


def paren_split(line, paren_list):
    sections = []
    for ilev, level in enumerate(paren_list):
        sections.append([])
        for igroup, group in enumerate(level):
            i1 = group[0]
            i2 = group[1]
            tmp_str = ""
            i3 = i1 + 1
            ranges = []
            if len(paren_list) > ilev+1:
                for lower_group in paren_list[ilev+1]:
                    if lower_group[0] > i1 and lower_group[1] <= i2:
                        tmp_str += line[i3:lower_group[0]]
                        ranges.append([i3, lower_group[0]])
                        i3 = lower_group[1] + 1
            if i3 < i2:
                tmp_str += line[i3:i2]
                ranges.append([i3, i2])
            if i3 == len(line):
                tmp_str += line[i3:i2]
                ranges.append([i3, i2])
            sections[ilev].append([ranges, tmp_str])
    return sections


def get_var_stack(line):
    if len(line) == 0:
        return None
    var_list = tokenize_line(line)
    deepest_var = None
    final_var = None
    final_paren = None
    deepest_paren = None
    n = len(line)
    for var_group in var_list:
        for var_tmp in var_group:
            for parens in var_tmp[0]:
                if n >= parens[0]:
                    if n <= parens[1]:
                        final_var = var_tmp[1]
                        final_paren = parens
                        break
                    elif parens[1] == -1:
                        deepest_var = var_tmp[1]
                        deepest_paren = parens
    if final_var is None:
        if deepest_var is not None:
{'fraction_non_alphanumeric': 0.13163042227591762, 'fraction_numerical': 0.01979058796456104, 'mean_word_length': 1.9394656746702739, 'pattern_counts': {'":': 12, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470327', 'n_tokens_mistral': 3051, 'n_tokens_neox': 2985, 'n_words': 678}