<commit_before>import os
import sys
import time
import imp
import py_compile
from pysmi.writer.base import AbstractWriter
from pysmi import debug
from pysmi import error

class PyFileWriter(AbstractWriter):
    suffixes = {}
    for sfx, mode, typ in imp.get_suffixes():
        if typ not in suffixes:
            suffixes[typ] = []
        suffixes[typ].append((sfx, mode))

    def __init__(self, path):
        self._path = os.path.normpath(path)

    def __str__(self): return '%s{"%s"}' % (self.__class__.__name__, self._path)

    def putData(self, mibname, data, alias='', dryRun=False):
        if dryRun:
            debug.logger & debug.flagWriter and debug.logger('dry run mode')
            return
        if not os.path.exists(self._path):
            try:
                os.makedirs(self._path)
            except OSError:
                raise error.PySmiWriterError('failure creating destination directory %s: %s' % (self._path, sys.exc_info()[1]), writer=self)

        pyfile = os.path.join(self._path, mibname) + self.suffixes[imp.PY_SOURCE][0][0]
        try:
            f = open(pyfile, 'wb')
            f.write(data.encode('utf-8'))
            f.close()
        except (IOError, UnicodeEncodeError):
            raise error.PySmiWriterError('failure writing file %s: %s' % (pyfile, sys.exc_info()[1]), file=pyfile, writer=self)

        debug.logger & debug.flagWriter and debug.logger('created file %s' % pyfile)

        if alias:
            pyalias = os.path.join(self._path, alias) + self.suffixes[imp.PY_SOURCE][0][0]
            try:
                os.unlink(pyalias)
                os.link(pyfile, pyalias)
            except (AttributeError, OSError):  # not all platforms support it
                try:
                    f = open(pyalias, 'wb')
                    f.write(data.encode('utf-8'))
                    f.close()
                except (IOError, UnicodeEncodeError):
                    raise error.PySmiWriterError('failure writing file %s: %s' % (pyalias, sys.exc_info()[1]), file=pyalias, alias=true, writer=self)
                else:
                    debug.logger & debug.flagWriter and debug.logger('a copy of file %s created as %s' % (pyfile, pyalias))
            else:
                debug.logger & debug.flagWriter and debug.logger('file %s linked to %s' % (pyalias, pyfile))
        
        for filename in mibname, alias:
            if not filename:
                continue
            try:
                py_compile.compile(pyfile, doraise=True)
            except (SyntaxError, py_compile.PyCompileError):
                pass  # XXX
            except:
                try:
                    os.unlink(pyfile)
                except:
                    pass
                raise error.PySmiWriterError('failure compiling %s: %s' % (filename, sys.exc_info()[1]), file=filename, writer=self)

            debug.logger & debug.flagWriter and debug.logger('compiled %s' % filename)
<commit_msg>Replace the concept of linking an alias MIB to canonical one with another workaround: create an empty stub file for each alias to keep track of canonical MIB modification time (which should be the same as stub's. However the whole alias thing should better be moved from writer to compiler.<commit_after>import os
import sys
import time
import imp
import py_compile
from pysmi.writer.base import AbstractWriter
from pysmi import debug
from pysmi import error

class PyFileWriter(AbstractWriter):
    suffixes = {}
    for sfx, mode, typ in imp.get_suffixes():
        if typ not in suffixes:
            suffixes[typ] = []
        suffixes[typ].append((sfx, mode))

    def __init__(self, path):
        self._path = os.path.normpath(path)

    def __str__(self): return '%s{"%s"}' % (self.__class__.__name__, self._path)

    def putData(self, mibname, data, alias='', dryRun=False):
        if dryRun:
            debug.logger & debug.flagWriter and debug.logger('dry run mode')
            return
        if not os.path.exists(self._path):
            try:
                os.makedirs(self._path)
            except OSError:
                raise error.PySmiWriterError('failure creating destination directory %s: %s' % (self._path, sys.exc_info()[1]), writer=self)

        pyfile = os.path.join(self._path, mibname) + self.suffixes[imp.PY_SOURCE][0][0]
        try:
            f = open(pyfile, 'wb')
            f.write(data.encode('utf-8'))
            f.close()
        except (IOError, UnicodeEncodeError):
            raise error.PySmiWriterError('failure writing file %s: %s' % (pyfile, sys.exc_info()[1]), file=pyfile, writer=self)

        debug.logger & debug.flagWriter and debug.logger('created file %s' % pyfile)

        if alias:
            pyalias = os.path.join(self._path, alias) + self.suffixes[imp.PY_SOURCE][0][0]
            comment = """#
# This is a stub pysnmp (http://pysnmp.sf.net) MIB file for %s
# Real contents of %s resides in %s
# The sole purpose of this stub file is to keep track of 
# %s's modification time 
# compared to MIB source file (%s)
#""" % (mibname, mibname, pyfile, pyfile, alias)
            try:
                f = open(pyalias, 'wb')
                f.write(comment.encode('utf-8'))
                f.close()
            except (IOError, UnicodeEncodeError):
                raise error.PySmiWriterError('failure writing file %s: %s' % (pyalias, sys.exc_info()[1]), file=pyalias, alias=true, writer=self)
            else:
                debug.logger & debug.flagWriter and debug.logger('a stub for file %s created as %s' % (pyfile, pyalias))
        
        for filename in mibname, alias:
            if not filename:
                continue
            try:
                py_compile.compile(pyfile, doraise=True)
            except (SyntaxError, py_compile.PyCompileError):
                pass  # XXX
            except:
                try:
                    os.unlink(pyfile)
                except:
                    pass
                raise error.PySmiWriterError('failure compiling %s: %s' % (filename, sys.exc_info()[1]), file=filename, writer=self)

            debug.logger & debug.flagWriter and debug.logger('compiled %s' % filename)

{'fraction_non_alphanumeric': 0.11168579279133667, 'fraction_numerical': 0.0032325844512687894, 'mean_word_length': 2.1079859367152185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2605714', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1760, 'n_words': 494}