<commit_before># coding: utf-8
''' 
Functions which call VSDebugConnector Add-In API for expression evaluation in VS debugger.
'''

import os
import socket
import tempfile
import re


class ReqType:
    VALUE = 1
    TYPE = 2
    MEMBERS = 3
    ALL = 4


class RespType:
    ERROR = 1
    VALUE = 2
    ARRAY = 3

BEL = '\x07'
PORT_FNAME = 'vsdbg.port'

_port = 0
_host = "localhost"


def _dbg_call(tp, expr):
    global _host, _port
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((_host, _port))
    s.sendall('%d%s%s' % (tp, BEL, expr))
    res = s.recv(4096)
    s.close()
    res = res.split(BEL)
    if len(res) < 2:
        raise Exception('Invalid respose')
    if int(res[0]) == RespType.ERROR:
        raise Exception(res[1])
    elif int(res[0]) == RespType.ARRAY:
        return res[1:]
    else:
        return res[1]


def detect():
    ''' Get port number from temp file '''
    port_fname = os.path.join(tempfile.gettempdir(), PORT_FNAME)
    if os.path.exists(port_fname):
        with open(port_fname) as f:
            global _port
            _port = int(f.readlines()[0])
    else:
        raise Exception('Could not find port number. Please load/realod VS add-in.')


try:
    detect()  # call detect during module loading
except Exception as e:
    print(e)


def port(p):
    global _port
    _port = p


def host(h):
    global _host
    _host = h


def p(expr):
    ''' retrieves expression with all memebers '''
    return _dbg_call(ReqType.ALL, expr)


def v(expr):
    ''' retrieves expression's value '''
    return _dbg_call(ReqType.VALUE, expr)


def t(expr):
    ''' retrieves expression's type '''
    return _dbg_call(ReqType.TYPE, expr)


def m(expr):
    ''' retrieves list of expression's members '''
    return _dbg_call(ReqType.MEMBERS, expr)


def enum_array(expr):
    val = int(re.search('\d+', v(expr)).group(0))
    for x in range(0, val):
        yield '%s[%d]' % (expr, x)



def cast_expr(expr, type):
    return "((%s)%s)" % (type, expr)

RE_CLASS_NAME = '[A-Za-z_][\w\.<>]+'


def get_types(expr):
    def get_types_rec(expr, tp):
        for x in m(cast_expr(expr, tp)):
            v = re.search('base {(%s)}' % RE_CLASS_NAME, x)
            if v:
                yield v.group(1)
                for x in get_types_rec(expr, v.group(1)): yield x
                break
    tp = t(expr)
    v = re.search('{(%s)}' % RE_CLASS_NAME, tp)
    if v: tp = v.group(1)
    yield tp
    for x in get_types_rec(expr, tp): yield x


def __dump(expr, level, depth):
    if depth == 0:
        return '%s : Max depth reached.' % expr
    res = []
    try:
        # dictionaries and list requires specic formatters
        res.append('%s%s: %s' % ('  ' * level, expr.split('.')[-1].split('>')[-1].replace(')', ''), v(expr)))

        exps = [cast_expr(expr, x) for x in get_types(expr)]
        if len(exps) == 0: exps.append(expr)
        for e in exps:
            for x in [i.strip() for i in m(e)]:
                if x == '' or x == 'Raw View' or 'base ' in x:  # skip "Raw View", "base {class_name}"
                    continue
                elif x[0] == '[':  # [???]
                    if x[1] != '0':  # [class_name]
                        pass
                    else:  # we have index [0x0???]
                        res.append(__dump('%s%s' % (e, x), level + 1, depth - 1))<commit_msg>Add specific case in dump() for handling generic dictionaries<commit_after># coding: utf-8
''' 
Functions which call VSDebugConnector Add-In API for expression evaluation in VS debugger.
'''

import os
import socket
import tempfile
import re


class ReqType:
    VALUE = 1
    TYPE = 2
    MEMBERS = 3
    ALL = 4


class RespType:
    ERROR = 1
    VALUE = 2
    ARRAY = 3

BEL = '\x07'
PORT_FNAME = 'vsdbg.port'

_port = 0
_host = "localhost"


def _dbg_call(tp, expr):
    global _host, _port
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((_host, _port))
    s.sendall('%d%s%s' % (tp, BEL, expr))
    res = s.recv(4096)
    s.close()
    res = res.split(BEL)
    if len(res) < 2:
        raise Exception('Invalid respose')
    if int(res[0]) == RespType.ERROR:
        raise Exception(res[1])
    elif int(res[0]) == RespType.ARRAY:
        return res[1:]
    else:
        return res[1]


def detect():
    ''' Get port number from temp file '''
    port_fname = os.path.join(tempfile.gettempdir(), PORT_FNAME)
    if os.path.exists(port_fname):
        with open(port_fname) as f:
            global _port
            _port = int(f.readlines()[0])
    else:
        raise Exception('Could not find port number. Please load/realod VS add-in.')


try:
    detect()  # call detect during module loading
except Exception as e:
    print(e)


def port(p):
    global _port
    _port = p


def host(h):
    global _host
    _host = h


def p(expr):
    ''' retrieves expression with all memebers '''
    return _dbg_call(ReqType.ALL, expr)


def v(expr):
    ''' retrieves expression's value '''
    return _dbg_call(ReqType.VALUE, expr)


def t(expr):
    ''' retrieves expression's type '''
    return _dbg_call(ReqType.TYPE, expr)


def m(expr):
    ''' retrieves list of expression's members '''
    return _dbg_call(ReqType.MEMBERS, expr)


def enum_array(expr):
    val = int(re.search('\d+', v(expr)).group(0))
    for x in range(0, val):
        yield '%s[%d]' % (expr, x)


def cast_expr(expr, type):
    return "((%s)%s)" % (type, expr)


def cast_dictionary(subtypes, expr, idx):
    return "(new System.Collections.Generic.Mscorlib_DictionaryDebugView<%s>(%s)).Items%s" % (subtypes, expr, idx)


RE_CLASS_NAME = '[A-Za-z_][\w\.<>,]+'


def get_types(expr):
    def get_types_rec(expr, tp):
        for x in m(cast_expr(expr, tp)):
            v = re.search('base {(%s)}' % RE_CLASS_NAME, x)
            if v:
                yield v.group(1)
                for x in get_types_rec(expr, v.group(1)): yield x
                break
    tp = t(expr)
    v = re.search('{(%s)}' % RE_CLASS_NAME, tp)
    if v: tp = v.group(1)
    yield tp
    for x in get_types_rec(expr, tp): yield x


def get_generic_subtypes(expr):
    for x in m('%s, raw' % expr):
        v = re.search('base {(%s)}' % RE_CLASS_NAME, x)
        if v:
            v = re.search('<(%s)>' % RE_CLASS_NAME, x)
            if v:
                return v.group(1)
    return ''


def __dump(expr, level, depth):
    if depth == 0:
        return '%s : Max depth reached.' % expr
    res = []
    try:
        #TODO: need to check type, i.e. we need to detect if this variable is a generic dictionary
        val = v(expr)
        if 'System.Collections.Generic.KeyNotFoundException' in val:  #BUG: this approach is failing if we have type of key is int and we have keys similar to indexes
            # generic dictionaries should be handled in a specific way
            expr = expr.split('[')
            idx = '[' + expr[-1]
            expr = '['.join(expr[:-1])
            expr = cast_dictionary(get_generic_subtypes(expr), expr, idx)
            val = v(expr)
        res.append('%s%s: %s' % ('  ' * level, expr.split('.')[-1].split('>')[-1].replace(')', ''), val))

        exps = [cast_expr(expr, x) for x in get_types(expr)]
        if len(exps) == 0: exps.append(expr)
        for e in exps:
            for x in [i.strip() for i in m(e)]:
                if x == '' or x == 'Raw View' or 'base ' in x:  # skip "Raw View", "base {class_name}"
                    continue
                elif x[0] == '[':  # [???]
                    if x[1] != '0':  # [class_name]
                        pass
                    else:  # we have index [0x0???]
                        res.append(__dump('%s%s' % (e, x), level + 1, depth - 1))
{'fraction_non_alphanumeric': 0.14741759667791332, 'fraction_numerical': 0.010251751881650662, 'mean_word_length': 2.2099125364431487, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4352107', 'n_tokens_mistral': 2768, 'n_tokens_neox': 2654, 'n_words': 765}