# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests
    else:
      self.tokens.append( (line, column, tokentype, s) )

  # ---------------------------------------------------------------------------

  def Tokenise(self,line_num,col_num,text):
    # determine what a specific token is and add it to the token list
    if text[0]=='!':
      self.AppendAttribute(Comment(text, (line_num, col_num)))
      return col_num+len(text)

    if text[0]=='\'':
      if text[-1]==text[0] and (self.re_oddsquote.search(text) or text=="''"):
        tok_type = self.tok_QUOTE
      else:
        tok_type = self.tok_QUOTE_START

    elif text[0]=='"':
      if text[-1]==text[0] and (self.re_odddquote.search(text) or text=='""'):
        tok_type = self.tok_QUOTE
      else:
        tok_type = self.tok_QUOTE_START

    elif text[0]=='&':
      # non-leading cont marker: free form only
      # need to remove leading blanks in col_num calculation
      col_num = col_num + len(text)-1
      
      # A non-leading cont marker needs to be appended to the
      # previous tokens as a postfix.

# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


# Path: Scanner/Scanner.py
import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests

    else:
      self.tokens.append( (line, column, tokentype, s) )

  # ---------------------------------------------------------------------------

  def Tokenise(self,line_num,col_num,text):
    # determine what a specific token is and add it to the token list
    if text[0]=='!':
      self.AppendAttribute(Comment(text, (line_num, col_num)))
      return col_num+len(text)

    if text[0]=='\'':
      if text[-1]==text[0] and (self.re_oddsquote.search(text) or text=="''"):
        tok_type = self.tok_QUOTE
      else:
        tok_type = self.tok_QUOTE_START

    elif text[0]=='"':
      if text[-1]==text[0] and (self.re_odddquote.search(text) or text=='""'):
        tok_type = self.tok_QUOTE
      else:
        tok_type = self.tok_QUOTE_START

    elif text[0]=='&':
      # non-leading cont marker: free form only
      # need to remove leading blanks in col_num calculation
      col_num = col_num + len(text)-1
      
      # A non-leading cont marker needs to be appended to the
      # previous tokens as a postfix.

      self.AppendAttribute2PreviousToken(ContMarker(text[0],