# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests
      everything = self.re_all_tokens.findall(line)
      spacing    = self.re_all_tokens.split(line)

      for j in range(len(everything)):
          space_len  = len(spacing[j])
          space_text = lstrip(spacing[j])
          if space_text!='':
              # we have found something we don't understand!
              col_num = col_num+(space_len-len(space_text))
              self.AppendToken(line_num, col_num, self.tok_UNKNOWN, space_text)
              col_num = col_num+len(space_text)
          else:
              col_num = col_num+space_len

          if everything[j]=='&' and line_type!=self.CONT_SQUOTE  \
                 and line_type!=self.CONT_DQUOTE:
              line_type=self.CONT_STATEMENT

          # check for macro-token types (string literals, continuation
          # markers,statement separators (;) and trailing comments)
          col_num = self.Tokenise(line_num,col_num,everything[j])
          # if token was an unfinished quote we will need to set line_type
          if self.tokens[-1][2]==self.tok_QUOTE_START \
                 and line_type!=self.CONT_SQUOTE and line_type!=self.CONT_DQUOTE:
              if everything[j][0]=='\'':
                  line_type = self.CONT_SQUOTE
              else:
                  line_type = self.CONT_DQUOTE

      if remainder!='':

# Path: AOR/AttributeString.py
# class AttributeString(str):
#     def __init__(self, s):
#         str.__init__(self, s)
#         # If this object is initialised with an AttributeString,
#         # copy the attributes as well
#         if isinstance(s, AttributeString):
#             self.lPrefix  = s.GetPrefixAttributes()[:]
#             self.lPostfix = s.GetPostfixAttributes()[:]
#         else:
#             self.lPrefix  = []
#             self.lPostfix = []
#     # --------------------------------------------------------------------------
#     # Creates a new attribute string which has the same pre- and postfix
#     # attributes, but a new string.
#     def sCreateCopy(self, s):
#         new = AttributeString(s)
#         new.SetPrefixAttributes(self.GetPrefixAttributes())
#         new.SetPostfixAttributes(self.GetPostfixAttributes())
#         return new
#     # --------------------------------------------------------------------------
#     # Splits an AttributeString into two strings, the first one containing
#     # all prefix attributes, the second one all postfix ones. The parameter
#     # n specified the number of character to for the first string.
#     def tSplitString(self, n):
#         s1 = AttributeString(self[0:n])
#         s2 = AttributeString(self[n: ])
#         s1.SetPrefixAttributes(self.GetPrefixAttributes())
#         s2.SetPostfixAttributes(self.GetPostfixAttributes())
#         return s1,s2
#     # --------------------------------------------------------------------------
#     def SetPrefixAttributes(self,  l) : self.lPrefix = l[:]
#     # --------------------------------------------------------------------------
#     def SetPostfixAttributes(self, l) : self.lPostfix = l[:]
#     # --------------------------------------------------------------------------
#     def AppendPrefix(self,  o):
#         if type(o)==ListType:
#             self.lPrefix.extend(o)
#         else:
#             self.lPrefix.append(o)
#     # --------------------------------------------------------------------------
#     def AppendPostfix(self, o):
#         if type(o)==ListType:
#             self.lPostfix.extend(o)
#         else:
#             self.lPostfix.append(o)
#     # --------------------------------------------------------------------------
#     def GetPrefixAttributes(self ): return self.lPrefix
#     # --------------------------------------------------------------------------
#     def GetPostfixAttributes(self): return self.lPostfix
#     # --------------------------------------------------------------------------
#     def GetString(self): return str.__str__(self)
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def upper(self):
#         return self.sCreateCopy(str.__str__(self).upper())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def lower(self):
#         return self.sCreateCopy(str.__str__(self).lower())
#     # --------------------------------------------------------------------------
#     # We have to overwrite this method so that a new AttributeString is
#     # returned!
#     def capitalize(self):
#         return self.sCreateCopy(str.__str__(self).capitalize())
#     # --------------------------------------------------------------------------
#     #def ToList(self, stylesheet, l): return l.append(str.__str__(self))
#     def ToList(self, stylesheet, l): return l.append(self)
#     # --------------------------------------------------------------------------
#     #def __repr__(self): return str.__str__(self)
#     # Debug: output an attribute string with all attributes
#     def __repr__(self):
#         return "%s-%s-%s"%(`self.lPrefix`,str.__str__(self),`self.lPostfix`)
# 
# Path: AOR/SpecialStatements.py
# class Comment(BasicNonStatement):
#     def __init__(self, sComment, loc):
#         BasicNonStatement.__init__(self, sComment, loc)
# 
# class ContMarker(BasicNonStatement):
#     def __init__(self, sContMarker, loc):
#         BasicNonStatement.__init__(self, sContMarker, loc)
# 
# class CommentLine(BasicNonStatement):
#     def __init__(self, sComment, loc=None):
#         BasicNonStatement.__init__(self, sComment, loc)


# Path: Scanner/Scanner.py
import re
from   string                import expandtabs,rstrip,lstrip,strip, upper
from   Token                 import Token, Token2Name
from   AOR.AttributeString   import AttributeString
from   AOR.SpecialStatements import Comment, ContMarker, CommentLine
      from Tools.Project import Project
      from Tools.Project import Project
  from Test.ScannerTest import RunAllTests

      everything = self.re_all_tokens.findall(line)
      spacing    = self.re_all_tokens.split(line)

      for j in range(len(everything)):
          space_len  = len(spacing[j])
          space_text = lstrip(spacing[j])
          if space_text!='':
              # we have found something we don't understand!
              col_num = col_num+(space_len-len(space_text))
              self.AppendToken(line_num, col_num, self.tok_UNKNOWN, space_text)
              col_num = col_num+len(space_text)
          else:
              col_num = col_num+space_len

          if everything[j]=='&' and line_type!=self.CONT_SQUOTE  \
                 and line_type!=self.CONT_DQUOTE:
              line_type=self.CONT_STATEMENT

          # check for macro-token types (string literals, continuation
          # markers,statement separators (;) and trailing comments)
          col_num = self.Tokenise(line_num,col_num,everything[j])
          # if token was an unfinished quote we will need to set line_type
          if self.tokens[-1][2]==self.tok_QUOTE_START \
                 and line_type!=self.CONT_SQUOTE and line_type!=self.CONT_DQUOTE:
              if everything[j][0]=='\'':
                  line_type = self.CONT_SQUOTE
              else:
                  line_type = self.CONT_DQUOTE

      if remainder!='':

        self.AppendAttribute(Comment(remainder, (line_num, col_num)))