<commit_before>from __future__ import print_function, absolute_import
import inspect
import re
from collections import Mapping, defaultdict, OrderedDict
import textwrap
from contextlib import closing
from numba.io_support import StringIO
from numba import ir
from . import step
import os
import sys


class SourceLines(Mapping):
    def __init__(self, func):

        try:
            lines, startno = inspect.getsourcelines(func)
        except IOError:
            self.lines = ()
            self.startno = 0
        else:
            self.lines = textwrap.dedent(''.join(lines)).splitlines()
            self.startno = startno

    def __getitem__(self, lineno):
        try:
            return self.lines[lineno - self.startno].rstrip()
        except IndexError:
            return ''

    def __iter__(self):
        return iter((self.startno + i) for i in range(len(self.lines)))

    def __len__(self):
        return len(self.lines)

    @property
    def avail(self):
        return bool(self.lines)


class TypeAnnotation(object):
    
    data = {'func_data': OrderedDict()}

    def __init__(self, interp, typemap, calltypes, lifted, args, return_type,
                 func_attr, fancy=None):
        self.filename = interp.bytecode.filename
        self.func = interp.bytecode.func
        self.blocks = interp.blocks
        self.typemap = typemap
        self.calltypes = calltypes
        self.lifted = lifted
        self.fancy = os.path.join(os.getcwd(), fancy)
        self.filename = interp.loc.filename
        self.linenum = str(interp.loc.line)
        self.signature = str(args) + ' -> ' + str(return_type)
        self.func_attr = func_attr

    def prepare_annotations(self):
        # Prepare annotations
        groupedinst = defaultdict(list)
        for blkid, blk in self.blocks.items():
            groupedinst[blk.loc.line].append("label %s" % blkid)
            for inst in blk.body:
                lineno = inst.loc.line

                if isinstance(inst, ir.Assign):
                    if (isinstance(inst.value, ir.Expr) and
                            inst.value.op ==  'call'):
                        atype = self.calltypes[inst.value]
                    else:
                        atype = self.typemap[inst.target.name]

                    aline = "%s = %s  :: %s" % (inst.target, inst.value, atype)
                elif isinstance(inst, ir.SetItem):
                    atype = self.calltypes[inst]
                    aline = "%s  :: %s" % (inst, atype)<commit_msg>Add check for None value for fancy annotation flag<commit_after>from __future__ import print_function, absolute_import
import inspect
import re
from collections import Mapping, defaultdict, OrderedDict
import textwrap
from contextlib import closing
from numba.io_support import StringIO
from numba import ir
from . import step
import os
import sys


class SourceLines(Mapping):
    def __init__(self, func):

        try:
            lines, startno = inspect.getsourcelines(func)
        except IOError:
            self.lines = ()
            self.startno = 0
        else:
            self.lines = textwrap.dedent(''.join(lines)).splitlines()
            self.startno = startno

    def __getitem__(self, lineno):
        try:
            return self.lines[lineno - self.startno].rstrip()
        except IndexError:
            return ''

    def __iter__(self):
        return iter((self.startno + i) for i in range(len(self.lines)))

    def __len__(self):
        return len(self.lines)

    @property
    def avail(self):
        return bool(self.lines)


class TypeAnnotation(object):
    
    data = {'func_data': OrderedDict()}

    def __init__(self, interp, typemap, calltypes, lifted, args, return_type,
                 func_attr, fancy=None):
        self.filename = interp.bytecode.filename
        self.func = interp.bytecode.func
        self.blocks = interp.blocks
        self.typemap = typemap
        self.calltypes = calltypes
        self.lifted = lifted
        if fancy is None:
            self.fancy = None
        else:
            self.fancy = os.path.join(os.getcwd(), fancy)
        self.filename = interp.loc.filename
        self.linenum = str(interp.loc.line)
        self.signature = str(args) + ' -> ' + str(return_type)
        self.func_attr = func_attr

    def prepare_annotations(self):
        # Prepare annotations
        groupedinst = defaultdict(list)
        for blkid, blk in self.blocks.items():
            groupedinst[blk.loc.line].append("label %s" % blkid)
            for inst in blk.body:
                lineno = inst.loc.line

                if isinstance(inst, ir.Assign):
                    if (isinstance(inst.value, ir.Expr) and
                            inst.value.op ==  'call'):
                        atype = self.calltypes[inst.value]
                    else:
                        atype = self.typemap[inst.target.name]

                    aline = "%s = %s  :: %s" % (inst.target, inst.value, atype)
                elif isinstance(inst, ir.SetItem):
                    atype = self.calltypes[inst]
                    aline = "%s  :: %s" % (inst, atype)
{'fraction_non_alphanumeric': 0.1005629974762182, 'fraction_numerical': 0.00038827412152980003, 'mean_word_length': 1.9107344632768362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6185661', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1446, 'n_words': 360}