<commit_before># $Id: AST.py,v 1.31 2004/01/13 07:37:03 stefan Exp $
#
# Copyright (C) 2000 <NAME>
# Copyright (C) 2000 <NAME>
# All rights reserved.
# Licensed to the public under the terms of the GNU LGPL (>= 2),
# see the file COPYING for details.
#

"""Abstract Syntax Tree classes.

This file contains classes which encapsulate nodes in the AST. The base class
is the Declaration class that encapsulates a named declaration. All names used
are scoped tuples.

Also defined in module scope are the constants DEFAULT, PUBLIC, PROTECTED and
PRIVATE.
"""

import Util, Type

import string, sys, cPickle, types, stat

# The version of the file format - this should be increased everytime
# incompatible changes are made to the AST or Type classes
FILE_VERSION = 5

# Accessibility constants
DEFAULT = 0
PUBLIC = 1
PROTECTED = 2
PRIVATE = 3

def ccmp(a,b):
   """Compares classes of two objects"""
   return cmp(type(a),type(b)) or cmp(a.__class__,b.__class__)

def load(filename):
   """Loads an AST object from the given filename"""
   try:
      file = open(filename, "r")
      unpickler = cPickle.Unpickler(file)
      version = unpickler.load()
      if version is not FILE_VERSION:
         file.close()
         raise Exception, 'Wrong file version'
      deps = unpickler.load() # ignored here
      ast = unpickler.load()
      file.close()
      return ast
   except:
      exc, msg = sys.exc_info()[0:2]
      if exc is Exception:
         raise Exception, "Loading AST from '%s': %s"%(filename, msg)
      raise Exception, "Loading AST from '%s', %s: %s"%(filename, exc, msg)

def load_deps(filename):
   """Loads a dependencies object from the given filename. The file will be
   an AST save file (usually *.syn), but this method only reads up to the
   dependencies object stored before the actual AST. The object returned is a
   list of (filename, timestamp) pairs."""
   try:
      file = open(filename, "r")
      unpickler = cPickle.Unpickler(file)
      version = unpickler.load()
      if version is not FILE_VERSION:
         file.close()
         raise Exception, 'Wrong file version'
      deps = unpickler.load()
      # stop here, before loading the (probably huge) AST object
      file.close()
      return deps
   except:
      exc, msg = sys.exc_info()[0:2]
      if exc is Exception:
         raise Exception, "Loading dependencies from '%s': %s"%(filename, msg)
      raise Exception, "Loading dependencies from '%s', %s: %s"%(filename, exc, msg)

def save(filename, ast):
   """Saves an AST object to the given filename"""
   try:
      deps = make_deps(ast)
      file = open(filename, "w")
      pickler = cPickle.Pickler(file, 1)
      pickler.dump(FILE_VERSION)
      pickler.dump(deps)
      pickler.dump(ast)
      file.close()
   except:
      exc, msg = sys.exc_info()[0:2]
      raise Exception, "Saving '%s', %s: %s"%(filename, exc, msg)

def make_deps(ast):
   """Creates the dependencies object to save in the .syn file from the given
   AST. The dependencies are a list of (filename, timestamp) pairs, extracted
   from ast.files()"""
   deps = []
   for file in ast.files().values():
      filename = file.full_filename()
      try:
         info = os.stat(filename)
      except:
         # Ignore any file we can't stat
         continue
      time = info[stat.ST_MTIME]
      deps.append( (filename, time) )
   return deps

class AST:
   """Top-level Abstract Syntax Tree.
   The AST represents the whole AST for a file or group of files as a list of
   declarations and a types dictionary.
   """

   def __init__(self, files=None, declarations=None, typedict=None):
      """Constructor"""

      if files is None: files = {}
      if declarations is None: declarations = []
      if type(files) is not types.DictType: raise TypeError, "files parameter must be a dict of SourceFile objects"
      if type(declarations) != type([]): raise TypeError, "declarations parameter must be a list of declarations"
      if typedict is None: typedict = Type.Dictionary()
      elif not isinstance(typedict, Type.Dictionary): raise TypeError, "types must be an instance of Type.Dictionary"
      self.__files	    = files
      self.__declarations = list(declarations)
      self.__types	    = typedict

   def files(self):
      """The files this AST represents. Returns a dictionary mapping
      filename to SourceFile objects."""	
      return self.__files

   def declarations(self):
      """List of Declaration objects. These are the declarations at file scope"""
      return self.__declarations

   def types(self):
      """Dictionary of types. This is a Type.Dictionary object"""
      return self.__types

   def accept(self, visitor):
      """Accept the given visitor"""
      visitor.visitAST(self)

   def merge(self, other_ast):
      """Merges another AST. Files and declarations are appended to those in
      this AST, and types are merged by overwriting existing types -
      Unduplicator is responsible for sorting out the mess this may cause :)"""
      self.__types.merge(other_ast.types())
      self.__declarations.extend(other_ast.declarations())
      # Do a basic job of merging the SourceFiles, trying not to lose
      # anything (even though duplicates are created at this stage - they
      # will be cleaned up by the Linker)
      for filename, file in other_ast.files().items():
         if not self.__files.has_key(filename):
            self.__files[filename] = file
            continue
         myfile = self.__files[filename]
         myfile.set_is_main(myfile.is_main() or file.is_main())
         myfile.declarations().extend(file.declarations())
         myfile.includes().extend(file.includes())

class Include:
   """Information about an include directive in a SourceFile.
   If the include directive required a macro expansion to get the filename,
   the is_macro will return true. If the include directive was actually an
   include_next, then is_next will return true.
   """
   def __init__(self, target, is_macro, is_next):
      if not isinstance(target, SourceFile):
         raise TypeError, "target parameter must be a SourceFile"
      self.__target = target
      self.__is_macro = is_macro
      self.__is_next = is_next

   def target(self):
      return self.__target

   def set_target(self, target):
      self.__target = target

   def is_macro(self):
      return self.__is_macro

   def is_next(self):
      return self.__is_next

class MacroCall:
   """A class to support mapping from positions in a preprocessed file
   back to positions in the original file."""

   def __init__(self, name, start, end, diff):

      self.name = name
      self.start = start,
      self.end = end
      self.diff = diff

class SourceFile:
   """The information about a file that the AST was generated from.
   Contains filename, all declarations from this file (even nested ones) and
   includes (aka imports) from this file."""

   def __init__(self, filename, full_filename, language):
      """Constructor"""<commit_msg>fix typo<commit_after># $Id: AST.py,v 1.31 2004/01/13 07:37:03 stefan Exp $
#
# Copyright (C) 2000 <NAME>
# Copyright (C) 2000 <NAME>
# All rights reserved.
# Licensed to the public under the terms of the GNU LGPL (>= 2),
# see the file COPYING for details.
#

"""Abstract Syntax Tree classes.

This file contains classes which encapsulate nodes in the AST. The base class
is the Declaration class that encapsulates a named declaration. All names used
are scoped tuples.

Also defined in module scope are the constants DEFAULT, PUBLIC, PROTECTED and
PRIVATE.
"""

import Util, Type

import string, sys, cPickle, types, stat

# The version of the file format - this should be increased everytime
# incompatible changes are made to the AST or Type classes
FILE_VERSION = 5

# Accessibility constants
DEFAULT = 0
PUBLIC = 1
PROTECTED = 2
PRIVATE = 3

def ccmp(a,b):
   """Compares classes of two objects"""
   return cmp(type(a),type(b)) or cmp(a.__class__,b.__class__)

def load(filename):
   """Loads an AST object from the given filename"""
   try:
      file = open(filename, "r")
      unpickler = cPickle.Unpickler(file)
      version = unpickler.load()
      if version is not FILE_VERSION:
         file.close()
         raise Exception, 'Wrong file version'
      deps = unpickler.load() # ignored here
      ast = unpickler.load()
      file.close()
      return ast
   except:
      exc, msg = sys.exc_info()[0:2]
      if exc is Exception:
         raise Exception, "Loading AST from '%s': %s"%(filename, msg)
      raise Exception, "Loading AST from '%s', %s: %s"%(filename, exc, msg)

def load_deps(filename):
   """Loads a dependencies object from the given filename. The file will be
   an AST save file (usually *.syn), but this method only reads up to the
   dependencies object stored before the actual AST. The object returned is a
   list of (filename, timestamp) pairs."""
   try:
      file = open(filename, "r")
      unpickler = cPickle.Unpickler(file)
      version = unpickler.load()
      if version is not FILE_VERSION:
         file.close()
         raise Exception, 'Wrong file version'
      deps = unpickler.load()
      # stop here, before loading the (probably huge) AST object
      file.close()
      return deps
   except:
      exc, msg = sys.exc_info()[0:2]
      if exc is Exception:
         raise Exception, "Loading dependencies from '%s': %s"%(filename, msg)
      raise Exception, "Loading dependencies from '%s', %s: %s"%(filename, exc, msg)

def save(filename, ast):
   """Saves an AST object to the given filename"""
   try:
      deps = make_deps(ast)
      file = open(filename, "w")
      pickler = cPickle.Pickler(file, 1)
      pickler.dump(FILE_VERSION)
      pickler.dump(deps)
      pickler.dump(ast)
      file.close()
   except:
      exc, msg = sys.exc_info()[0:2]
      raise Exception, "Saving '%s', %s: %s"%(filename, exc, msg)

def make_deps(ast):
   """Creates the dependencies object to save in the .syn file from the given
   AST. The dependencies are a list of (filename, timestamp) pairs, extracted
   from ast.files()"""
   deps = []
   for file in ast.files().values():
      filename = file.full_filename()
      try:
         info = os.stat(filename)
      except:
         # Ignore any file we can't stat
         continue
      time = info[stat.ST_MTIME]
      deps.append( (filename, time) )
   return deps

class AST:
   """Top-level Abstract Syntax Tree.
   The AST represents the whole AST for a file or group of files as a list of
   declarations and a types dictionary.
   """

   def __init__(self, files=None, declarations=None, typedict=None):
      """Constructor"""

      if files is None: files = {}
      if declarations is None: declarations = []
      if type(files) is not types.DictType: raise TypeError, "files parameter must be a dict of SourceFile objects"
      if type(declarations) != type([]): raise TypeError, "declarations parameter must be a list of declarations"
      if typedict is None: typedict = Type.Dictionary()
      elif not isinstance(typedict, Type.Dictionary): raise TypeError, "types must be an instance of Type.Dictionary"
      self.__files	    = files
      self.__declarations = list(declarations)
      self.__types	    = typedict

   def files(self):
      """The files this AST represents. Returns a dictionary mapping
      filename to SourceFile objects."""	
      return self.__files

   def declarations(self):
      """List of Declaration objects. These are the declarations at file scope"""
      return self.__declarations

   def types(self):
      """Dictionary of types. This is a Type.Dictionary object"""
      return self.__types

   def accept(self, visitor):
      """Accept the given visitor"""
      visitor.visitAST(self)

   def merge(self, other_ast):
      """Merges another AST. Files and declarations are appended to those in
      this AST, and types are merged by overwriting existing types -
      Unduplicator is responsible for sorting out the mess this may cause :)"""
      self.__types.merge(other_ast.types())
      self.__declarations.extend(other_ast.declarations())
      # Do a basic job of merging the SourceFiles, trying not to lose
      # anything (even though duplicates are created at this stage - they
      # will be cleaned up by the Linker)
      for filename, file in other_ast.files().items():
         if not self.__files.has_key(filename):
            self.__files[filename] = file
            continue
         myfile = self.__files[filename]
         myfile.set_is_main(myfile.is_main() or file.is_main())
         myfile.declarations().extend(file.declarations())
         myfile.includes().extend(file.includes())

class Include:
   """Information about an include directive in a SourceFile.
   If the include directive required a macro expansion to get the filename,
   the is_macro will return true. If the include directive was actually an
   include_next, then is_next will return true.
   """
   def __init__(self, target, is_macro, is_next):
      if not isinstance(target, SourceFile):
         raise TypeError, "target parameter must be a SourceFile"
      self.__target = target
      self.__is_macro = is_macro
      self.__is_next = is_next

   def target(self):
      return self.__target

   def set_target(self, target):
      self.__target = target

   def is_macro(self):
      return self.__is_macro

   def is_next(self):
      return self.__is_next

class MacroCall:
   """A class to support mapping from positions in a preprocessed file
   back to positions in the original file."""

   def __init__(self, name, start, end, diff):

      self.name = name
      self.start = start
      self.end = end
      self.diff = diff

class SourceFile:
   """The information about a file that the AST was generated from.
   Contains filename, all declarations from this file (even nested ones) and
   includes (aka imports) from this file."""

   def __init__(self, filename, full_filename, language):
      """Constructor"""
{'fraction_non_alphanumeric': 0.09560815628119207, 'fraction_numerical': 0.005418508484243548, 'mean_word_length': 3.0008556759840275, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1822628', 'n_tokens_mistral': 4146, 'n_tokens_neox': 3945, 'n_words': 1552}