# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests
  #    treated as stand-alone types, bust must be added as an attribute
  #    instead, since they can't be part of the (extended grammar)
  def ContLineCleanup(self):
    if not self.tokens: return
    
    # Now search for the last real token, and skip all special
    # tokens like comments, directives, and end-of-line separators.
    # -------------------------------------------------------------
    i=len(self.tokens)-1
    t=self.tokens[i]
    while i>=0 and (t[2]==Token.tok_COMMENTLINE  or \
                    t[2]==Token.tok_PREDIRECTIVE or \
                    t[2]==Token.tok_DIRECTIVE    or \
                    t[2]==Token.tok_SEPARATOR        ):
      i=i-1
      t=self.tokens[i]

    # Now i is the last real token, i+1 is the separator token:
    # ---------------------------------------------------------
    i=i+1
    
    # Now convert all non-separator tokens into attributes:
    # -----------------------------------------------------
    if i>=0 and i<=len(self.tokens)-1 and \
           self.tokens[i][2]==Token.tok_SEPARATOR:
      # Now convert all commentlines and directives into
      # attributes
      j=i+1
      while j<len(self.tokens):
        if self.tokens[j][2]==Token.tok_COMMENTLINE:

# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


# Path: Scanner/Scanner.py
import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests

  #    treated as stand-alone types, bust must be added as an attribute
  #    instead, since they can't be part of the (extended grammar)
  def ContLineCleanup(self):
    if not self.tokens: return
    
    # Now search for the last real token, and skip all special
    # tokens like comments, directives, and end-of-line separators.
    # -------------------------------------------------------------
    i=len(self.tokens)-1
    t=self.tokens[i]
    while i>=0 and (t[2]==Token.tok_COMMENTLINE  or \
                    t[2]==Token.tok_PREDIRECTIVE or \
                    t[2]==Token.tok_DIRECTIVE    or \
                    t[2]==Token.tok_SEPARATOR        ):
      i=i-1
      t=self.tokens[i]

    # Now i is the last real token, i+1 is the separator token:
    # ---------------------------------------------------------
    i=i+1
    
    # Now convert all non-separator tokens into attributes:
    # -----------------------------------------------------
    if i>=0 and i<=len(self.tokens)-1 and \
           self.tokens[i][2]==Token.tok_SEPARATOR:
      # Now convert all commentlines and directives into
      # attributes
      j=i+1
      while j<len(self.tokens):
        if self.tokens[j][2]==Token.tok_COMMENTLINE:

          self.lPrefix.append( CommentLine(self.tokens[j][3]) )