<commit_before>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

library csslib.parser;

import 'dart:math' as math;

import 'package:source_span/source_span.dart';

import "visitor.dart";
import 'src/messages.dart';
import 'src/options.dart';

part 'src/analyzer.dart';
part 'src/polyfill.dart';
part 'src/property.dart';
part 'src/token.dart';
part 'src/tokenizer_base.dart';
part 'src/tokenizer.dart';
part 'src/tokenkind.dart';


/** Used for parser lookup ahead (used for nested selectors Less support). */
class ParserState extends TokenizerState {
  final Token peekToken;
  final Token previousToken;

  ParserState(this.peekToken, this.previousToken, Tokenizer tokenizer)
      : super(tokenizer);
}

// TODO(jmesserly): this should not be global
void _createMessages({List<Message> errors, List<String> options}) {
  if (errors == null) errors = [];

  if (options == null) {
    options = ['--no-colors', 'memory'];
  }
  var opt = PreprocessorOptions.parse(options);
  messages = new Messages(options: opt, printHandler: errors.add);
}

/** CSS checked mode enabled. */
bool get isChecked => messages.options.checked;

// TODO(terry): Remove nested name parameter.
/** Parse and analyze the CSS file. */
StyleSheet compile(input, {List<Message> errors, List<String> options,
    bool nested: true,
    bool polyfill: false,
    List<StyleSheet> includes: null}) {

  if (includes == null) {
    includes = [];
  }

  var source = _inputAsString(input);

  _createMessages(errors: errors, options: options);

  var file = new SourceFile(source);

  var tree = new _Parser(file, source).parse();

  analyze([tree], errors: errors, options: options);

  if (polyfill) {
    var processCss = new PolyFill(messages, true);
    processCss.process(tree, includes: includes);
  }

  return tree;
}

/** Analyze the CSS file. */
void analyze(List<StyleSheet> styleSheets,
    {List<Message> errors, List<String> options}) {

  _createMessages(errors: errors, options: options);
  new Analyzer(styleSheets, messages).run();
}

/**
 * Parse the [input] CSS stylesheet into a tree. The [input] can be a [String],
 * or [List<int>] of bytes and returns a [StyleSheet] AST.  The optional
 * [errors] list will contain each error/warning as a [Message].
 */
StyleSheet parse(input, {List<Message> errors, List<String> options}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors, options: options);

  var file = new SourceFile(source);
  return new _Parser(file, source).parse();
}

/**
 * Parse the [input] CSS selector into a tree. The [input] can be a [String],
 * or [List<int>] of bytes and returns a [StyleSheet] AST.  The optional
 * [errors] list will contain each error/warning as a [Message].
 */
// TODO(jmesserly): should rename "parseSelector" and return Selector
StyleSheet selector(input, {List<Message> errors}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors);

  var file = new SourceFile(source);
  return (new _Parser(file, source)
      ..tokenizer.inSelector = true)
      .parseSelector();
}

SelectorGroup parseSelectorGroup(input, {List<Message> errors}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors);

  var file = new SourceFile(source);
  return (new _Parser(file, source)
      // TODO(jmesserly): this fix should be applied to the parser. It's tricky
      // because by the time the flag is set one token has already been fetched.
      ..tokenizer.inSelector = true)
      .processSelectorGroup();
}

String _inputAsString(input) {
  String source;

  if (input is String) {
    source = input;
  } else if (input is List<int>) {
    // TODO(terry): The parse function needs an "encoding" argument and will
    //              default to whatever encoding CSS defaults to.
    //
    // Here's some info about CSS encodings:
    // http://www.w3.org/International/questions/qa-css-charset.en.php
    //
    // As JMesserly suggests it will probably need a "preparser" html5lib
    // (encoding_parser.dart) that interprets the bytes as ASCII and scans for
    // @charset. But for now an "encoding" argument would work.  Often the
    // HTTP header will indicate the correct encoding.
    //
    // See encoding helpers at: package:html5lib/lib/src/char_encodings.dart
    // These helpers can decode in different formats given an encoding name
    // (mostly unicode, ascii, windows-1252 which is html5 default encoding).
    source = new String.fromCharCodes(input);
  } else {
    // TODO(terry): Support RandomAccessFile using console.
    throw new ArgumentError("'source' must be a String or "
        "List<int> (of bytes). RandomAccessFile not supported from this "
        "simple interface");
  }

  return source;
}

// TODO(terry): Consider removing this class when all usages can be eliminated
//               or replaced with compile API.
/** Public parsing interface for csslib. */
class Parser {
  final _Parser _parser;

  // TODO(jmesserly): having file and text is redundant.
  Parser(SourceFile file, String text, {int start: 0, String baseUrl}) :
    _parser = new _Parser(file, text, start: start, baseUrl: baseUrl);

  StyleSheet parse() => _parser.parse();
}

/** A simple recursive descent parser for CSS. */
class _Parser {
  final Tokenizer tokenizer;

  /** Base url of CSS file. */
  final String _baseUrl;

  /**
   * File containing the source being parsed, used to report errors with
   * source-span locations.
   */
  final SourceFile file;

  Token _previousToken;
  Token _peekToken;

  _Parser(SourceFile file, String text, {int start: 0, String baseUrl})
      : this.file = file,
        _baseUrl = baseUrl,
        tokenizer = new Tokenizer(file, text, true, start) {
    _peekToken = tokenizer.next();
  }

  /** Main entry point for parsing an entire CSS file. */
  StyleSheet parse() {
    List<TreeNode> productions = [];

    int start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE) && !_peekKind(TokenKind.RBRACE)) {
      // TODO(terry): Need to handle charset.
      var directive = processDirective();
      if (directive != null) {
        productions.add(directive);
        _maybeEat(TokenKind.SEMICOLON);
      } else {
        RuleSet ruleset = processRuleSet();
        if (ruleset != null) {
          productions.add(ruleset);
        } else {
          break;
        }
      }
    }

    checkEndOfFile();

    return new StyleSheet(productions, _makeSpan(start));
  }

  /** Main entry point for parsing a simple selector sequence. */
  StyleSheet parseSelector() {
    List<TreeNode> productions = [];

    int start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE) && !_peekKind(TokenKind.RBRACE)) {
      var selector = processSelector();
      if (selector != null) {
        productions.add(selector);
      }
    }

    checkEndOfFile();

    return new StyleSheet.selector(productions, _makeSpan(start));
  }

  /** Generate an error if [file] has not been completely consumed. */
  void checkEndOfFile() {
    if (!(_peekKind(TokenKind.END_OF_FILE) ||
        _peekKind(TokenKind.INCOMPLETE_COMMENT))) {
      _error('premature end of file unknown CSS', _peekToken.span);
    }
  }

  /** Guard to break out of parser when an unexpected end of file is found. */
  // TODO(jimhug): Failure to call this method can lead to inifinite parser
  //   loops.  Consider embracing exceptions for more errors to reduce
  //   the danger here.
  bool isPrematureEndOfFile() {
    if (_maybeEat(TokenKind.END_OF_FILE)) {
      _error('unexpected end of file', _peekToken.span);
      return true;
    } else {
      return false;
    }
  }

  ///////////////////////////////////////////////////////////////////
  // Basic support methods
  ///////////////////////////////////////////////////////////////////
  int _peek() {
    return _peekToken.kind;
  }

  Token _next({unicodeRange : false}) {
    _previousToken = _peekToken;
    _peekToken = tokenizer.next(unicodeRange: unicodeRange);
    return _previousToken;
  }

  bool _peekKind(int kind) {
    return _peekToken.kind == kind;
  }

  /* Is the next token a legal identifier?  This includes pseudo-keywords. */
  bool _peekIdentifier() {
    return TokenKind.isIdentifier(_peekToken.kind);
  }

  /** Marks the parser/tokenizer look ahead to support Less nested selectors. */
  ParserState get _mark =>
      new ParserState(_peekToken, _previousToken, tokenizer);

  /** Restores the parser/tokenizer state to state remembered by _mark. */
  void _restore(ParserState markedData) {
    tokenizer.restore(markedData);
    _peekToken = markedData.peekToken;
    _previousToken = markedData.previousToken;
  }

  bool _maybeEat(int kind, {unicodeRange : false}) {
    if (_peekToken.kind == kind) {
      _previousToken = _peekToken;
      _peekToken = tokenizer.next(unicodeRange: unicodeRange);
      return true;
    } else {
      return false;
    }
  }

  void _eat(int kind, {unicodeRange : false}) {
    if (!_maybeEat(kind, unicodeRange: unicodeRange)) {
      _errorExpected(TokenKind.kindToString(kind));
    }
  }

  void _eatSemicolon() {
    _eat(TokenKind.SEMICOLON);
  }

  void _errorExpected(String expected) {
    var tok = _next();
    var message;
    try {
      message = 'expected $expected, but found $tok';
    } catch (e) {
      message = 'parsing error expected $expected';
    }
    _error(message, tok.span);
  }

  void _error(String message, SourceSpan location) {
    if (location == null) {
      location = _peekToken.span;
    }
    messages.error(message, location);
  }

  void _warning(String message, SourceSpan location) {
    if (location == null) {
      location = _peekToken.span;
    }
    messages.warning(message, location);
  }

  SourceSpan _makeSpan(int start) {
    // TODO(terry): there are places where we are creating spans before we eat
    // the tokens, so using _previousToken.end is not always valid.
    var end = _previousToken != null && _previousToken.end >= start
        ? _previousToken.end : _peekToken.end;
    return file.span(start, end);
  }

  ///////////////////////////////////////////////////////////////////
  // Top level productions
  ///////////////////////////////////////////////////////////////////

  /**
   * The media_query_list production below replaces the media_list production
   * from CSS2 the new grammar is:
   *
   *   media_query_list
   *    : S* [media_query [ ',' S* media_query ]* ]?
   *   media_query
   *    : [ONLY | NOT]? S* media_type S* [ AND S* expression ]*
   *    | expression [ AND S* expression ]*
   *   media_type
   *    : IDENT
   *   expression
   *    : '(' S* media_feature S* [ ':' S* expr ]? ')' S*
   *   media_feature
   *    : IDENT
   */
  List<MediaQuery> processMediaQueryList() {
    var mediaQueries = [];

    bool firstTime = true;
    var mediaQuery;
    do {
      mediaQuery = processMediaQuery(firstTime == true);
      if (mediaQuery != null) {
        mediaQueries.add(mediaQuery);
        firstTime = false;
        continue;
      }

      // Any more more media types separated by comma.
      if (!_maybeEat(TokenKind.COMMA)) break;

      // Yep more media types start again.
      firstTime = true;
    } while ((!firstTime && mediaQuery != null) || firstTime);

    return mediaQueries;
  }

  MediaQuery processMediaQuery([bool startQuery = true]) {
    // Grammar: [ONLY | NOT]? S* media_type S*
    //          [ AND S* MediaExpr ]* | MediaExpr [ AND S* MediaExpr ]*

    int start = _peekToken.start;

    // Is it a unary media operator?
    var op = _peekToken.text;
    var opLen = op.length;
    var unaryOp = TokenKind.matchMediaOperator(op, 0, opLen);
    if (unaryOp != -1) {
      if (isChecked) {
        if (startQuery &&
            unaryOp != TokenKind.MEDIA_OP_NOT ||
            unaryOp != TokenKind.MEDIA_OP_ONLY) {
          _warning("Only the unary operators NOT and ONLY allowed",
              _makeSpan(start));
        }
        if (!startQuery && unaryOp != TokenKind.MEDIA_OP_AND) {
          _warning("Only the binary AND operator allowed", _makeSpan(start));
        }
      }
      _next();
      start = _peekToken.start;
    }

    var type;
    if (startQuery && unaryOp != TokenKind.MEDIA_OP_AND) {
      // Get the media type.
      if (_peekIdentifier()) type = identifier();
    }

    var exprs = [];

    if (unaryOp == -1 || unaryOp == TokenKind.MEDIA_OP_AND) {
      var andOp = false;
      while (true) {
        var expr = processMediaExpression(andOp);
        if (expr == null) break;

        exprs.add(expr);
        op = _peekToken.text;
        opLen = op.length;
        andOp = TokenKind.matchMediaOperator(op, 0, opLen) ==
            TokenKind.MEDIA_OP_AND;
        if (!andOp) break;
        _next();
      }
    }

    if (unaryOp != -1 || type != null || exprs.length > 0) {
      return new MediaQuery(unaryOp, type, exprs, _makeSpan(start));
    }
  }

  MediaExpression processMediaExpression([bool andOperator = false]) {
    int start = _peekToken.start;

    // Grammar: '(' S* media_feature S* [ ':' S* expr ]? ')' S*
    if (_maybeEat(TokenKind.LPAREN)) {
      if (_peekIdentifier()) {
        var feature = identifier();           // Media feature.
        while (_maybeEat(TokenKind.COLON)) {
          int startExpr = _peekToken.start;
          var exprs = processExpr();
          if (_maybeEat(TokenKind.RPAREN)) {
            return new MediaExpression(andOperator, feature, exprs,
                _makeSpan(startExpr));
          } else if (isChecked) {
            _warning("Missing parenthesis around media expression",
                _makeSpan(start));
            return null;
          }
        }
      } else if (isChecked) {
        _warning("Missing media feature in media expression", _makeSpan(start));
        return null;
      }
    }
  }

  /**
   * Directive grammar:
   *
   *  import:             '@import' [string | URI] media_list?
   *  media:              '@media' media_query_list '{' ruleset '}'
   *  page:               '@page' [':' IDENT]? '{' declarations '}'
   *  stylet:             '@stylet' IDENT '{' ruleset '}'
   *  media_query_list:   IDENT [',' IDENT]
   *  keyframes:          '@-webkit-keyframes ...' (see grammar below).
   *  font_face:          '@font-face' '{' declarations '}'
   *  namespace:          '@namespace name url("xmlns")
   *  host:               '@host '{' ruleset '}'
   *  mixin:              '@mixin name [(args,...)] '{' declarations/ruleset '}'
   *  include:            '@include name [(@arg,@arg1)]
   *                      '@include name [(@arg...)]
   *  content             '@content'
   */
  processDirective() {
    int start = _peekToken.start;

    var tokId = processVariableOrDirective();
    if (tokId is VarDefinitionDirective) return tokId;
    switch (tokId) {
      case TokenKind.DIRECTIVE_IMPORT:
        _next();

        // @import "uri_string" or @import url("uri_string") are identical; only
        // a url can follow an @import.
        String importStr;
        if (_peekIdentifier()) {
          var func = processFunction(identifier());
          if (func is UriTerm) {
            importStr = func.text;
          }
        } else {
          importStr = processQuotedString(false);
        }

        // Any medias?
        var medias = processMediaQueryList();

        if (importStr == null) {
          _error('missing import string', _peekToken.span);
        }

        return new ImportDirective(importStr.trim(), medias, _makeSpan(start));

      case TokenKind.DIRECTIVE_MEDIA:
        _next();

        // Any medias?
        var media = processMediaQueryList();

        List<TreeNode> rulesets = [];
        if (_maybeEat(TokenKind.LBRACE)) {
          while (!_maybeEat(TokenKind.END_OF_FILE)) {
            RuleSet ruleset = processRuleSet();
            if (ruleset == null) break;
            rulesets.add(ruleset);
          }

          if (!_maybeEat(TokenKind.RBRACE)) {
            _error('expected } after ruleset for @media', _peekToken.span);
          }
        } else {
          _error('expected { after media before ruleset', _peekToken.span);
        }
        return new MediaDirective(media, rulesets, _makeSpan(start));

      case TokenKind.DIRECTIVE_HOST:
        _next();

        List<TreeNode> rulesets = [];
        if (_maybeEat(TokenKind.LBRACE)) {
          while (!_maybeEat(TokenKind.END_OF_FILE)) {
            RuleSet ruleset = processRuleSet();
            if (ruleset == null) break;
            rulesets.add(ruleset);
          }

          if (!_maybeEat(TokenKind.RBRACE)) {
            _error('expected } after ruleset for @host', _peekToken.span);
          }
        } else {
          _error('expected { after host before ruleset', _peekToken.span);
        }
        return new HostDirective(rulesets, _makeSpan(start));

      case TokenKind.DIRECTIVE_PAGE:
        /*
         * @page S* IDENT? pseudo_page?
         *      S* '{' S*
         *      [ declaration | margin ]?
         *      [ ';' S* [ declaration | margin ]? ]* '}' S*
         *
         * pseudo_page :
         *      ':' [ "left" | "right" | "first" ]
         *
         * margin :
         *      margin_sym S* '{' declaration [ ';' S* declaration? ]* '}' S*
         *
         * margin_sym : @top-left-corner, @top-left, @bottom-left, etc.
         *
         * See http://www.w3.org/TR/css3-page/#CSS21
         */
        _next();

        // Page name
        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        // Any pseudo page?
        var pseudoPage;
        if (_maybeEat(TokenKind.COLON)) {
          if (_peekIdentifier()) {
            pseudoPage = identifier();
            // TODO(terry): Normalize pseudoPage to lowercase.
            if (isChecked &&
                !(pseudoPage.name == 'left' ||
                  pseudoPage.name == 'right' ||
                  pseudoPage.name == 'first')) {
              _warning("Pseudo page must be left, top or first",
                  pseudoPage.span);
              return null;
            }
          }
        }

        String pseudoName = pseudoPage is Identifier ? pseudoPage.name : '';
        String ident = name is Identifier ? name.name : '';
        return new PageDirective(ident, pseudoName,
            processMarginsDeclarations(), _makeSpan(start));

      case TokenKind.DIRECTIVE_CHARSET:
        // @charset S* STRING S* ';'
        _next();

        var charEncoding = processQuotedString(false);
        if (isChecked && charEncoding == null) {
          // Missing character encoding.
          _warning('missing character encoding string', _makeSpan(start));
        }

        return new CharsetDirective(charEncoding, _makeSpan(start));

      // TODO(terry): Workaround Dart2js bug continue not implemented in switch
      //              see https://code.google.com/p/dart/issues/detail?id=8270
      /*
      case TokenKind.DIRECTIVE_MS_KEYFRAMES:
        // TODO(terry): For now only IE 10 (are base level) supports @keyframes,
        // -moz- has only been optional since Oct 2012 release of Firefox, not
        // all versions of webkit support @keyframes and opera doesn't yet
        // support w/o -o- prefix.  Add more warnings for other prefixes when
        // they become optional.
        if (isChecked) {
          _warning('@-ms-keyframes should be @keyframes', _makeSpan(start));
        }
        continue keyframeDirective;

      keyframeDirective:
      */
      case TokenKind.DIRECTIVE_KEYFRAMES:
      case TokenKind.DIRECTIVE_WEB_KIT_KEYFRAMES:
      case TokenKind.DIRECTIVE_MOZ_KEYFRAMES:
      case TokenKind.DIRECTIVE_O_KEYFRAMES:
      // TODO(terry): Remove workaround when bug 8270 is fixed.
      case TokenKind.DIRECTIVE_MS_KEYFRAMES:
        if (tokId == TokenKind.DIRECTIVE_MS_KEYFRAMES && isChecked) {
          _warning('@-ms-keyframes should be @keyframes', _makeSpan(start));
        }
      // TODO(terry): End of workaround.

        /*  Key frames grammar:
         *
         *  @[browser]? keyframes [IDENT|STRING] '{' keyframes-blocks '}';
         *
         *  browser: [-webkit-, -moz-, -ms-, -o-]
         *
         *  keyframes-blocks:
         *    [keyframe-selectors '{' declarations '}']* ;
         *
         *  keyframe-selectors:
         *    ['from'|'to'|PERCENTAGE] [',' ['from'|'to'|PERCENTAGE] ]* ;
         */
        _next();

        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        _eat(TokenKind.LBRACE);

        var keyframe = new KeyFrameDirective(tokId, name, _makeSpan(start));

        do {
          Expressions selectors = new Expressions(_makeSpan(start));

          do {
            var term = processTerm();

            // TODO(terry): Only allow from, to and PERCENTAGE ...

            selectors.add(term);
          } while (_maybeEat(TokenKind.COMMA));

          keyframe.add(new KeyFrameBlock(selectors, processDeclarations(),
              _makeSpan(start)));

        } while (!_maybeEat(TokenKind.RBRACE) && !isPrematureEndOfFile());

        return keyframe;

      case TokenKind.DIRECTIVE_FONTFACE:
        _next();
        return new FontFaceDirective(processDeclarations(), _makeSpan(start));

      case TokenKind.DIRECTIVE_STYLET:
        /* Stylet grammar:
         *
         *  @stylet IDENT '{'
         *    ruleset
         *  '}'
         */
        _next();

        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        _eat(TokenKind.LBRACE);

        List<TreeNode> productions = [];

        start = _peekToken.start;
        while (!_maybeEat(TokenKind.END_OF_FILE)) {
          RuleSet ruleset = processRuleSet();
          if (ruleset == null) {
            break;
          }
          productions.add(ruleset);
        }

        _eat(TokenKind.RBRACE);

        return new StyletDirective(name, productions, _makeSpan(start));

      case TokenKind.DIRECTIVE_NAMESPACE:
        /* Namespace grammar:
         *
         * @namespace S* [namespace_prefix S*]? [STRING|URI] S* ';' S*
         * namespace_prefix : IDENT
         *
         */
        _next();

        var prefix;
        if (_peekIdentifier()) {
          prefix = identifier();
        }

        // The namespace URI can be either a quoted string url("uri_string")
        // are identical.
        String namespaceUri;
        if (_peekIdentifier()) {
          var func = processFunction(identifier());
          if (func is UriTerm) {
            namespaceUri = func.text;
          }
        } else {
          if (prefix != null && prefix.name == 'url') {
            var func = processFunction(prefix);
            if (func is UriTerm) {
              // @namespace url("");
              namespaceUri = func.text;
              prefix = null;
            }
          } else {
            namespaceUri = processQuotedString(false);
          }
        }

        return new NamespaceDirective(prefix != null ? prefix.name : '',
            namespaceUri, _makeSpan(start));

      case TokenKind.DIRECTIVE_MIXIN:
        return processMixin(start);

      case TokenKind.DIRECTIVE_INCLUDE:
        return processInclude( _makeSpan(start));

      case TokenKind.DIRECTIVE_CONTENT:
        // TODO(terry): TBD
        _warning("@content not implemented.", _makeSpan(start));
        return null;
    }
    return null;
  }

  /**
   * Parse the mixin beginning token offset [start]. Returns a [MixinDefinition]
   * node.
   *
   * Mixin grammar:
   *
   *  @mixin IDENT [(args,...)] '{'
   *    [ruleset | property | directive]*
   *  '}'
   */
  MixinDefinition processMixin(int start) {
    _next();

    var name = identifier();

    List<VarDefinitionDirective> params = [];
    // Any parameters?
    if (_maybeEat(TokenKind.LPAREN)) {
      var mustHaveParam = false;
      var keepGoing = true;
      while (keepGoing) {
        var varDef = processVariableOrDirective(mixinParameter: true);
        if (varDef is VarDefinitionDirective || varDef is VarDefinition) {
          params.add(varDef);
        } else if (mustHaveParam) {
          _warning("Expecting parameter", _makeSpan(_peekToken.start));
          keepGoing = false;
        }
        if (_maybeEat(TokenKind.COMMA)) {
          mustHaveParam = true;
          continue;
        }
        keepGoing = !_maybeEat(TokenKind.RPAREN);
      }
    }

    _eat(TokenKind.LBRACE);

    List<TreeNode> productions = [];
    List<TreeNode> declarations = [];
    var mixinDirective;

    start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE)) {
      var directive = processDirective();
      if (directive != null) {
        productions.add(directive);
        continue;
      }

      var declGroup = processDeclarations(checkBrace: false);
      var decls = [];
      if (declGroup.declarations.any((decl) {
        return decl is Declaration &&
            decl is! IncludeMixinAtDeclaration;
      })) {
        var newDecls = [];
        productions.forEach((include) {
          // If declGroup has items that are declarations then we assume
          // this mixin is a declaration mixin not a top-level mixin.
          if (include is IncludeDirective) {
            newDecls.add(new IncludeMixinAtDeclaration(include,
                include.span));
          } else {
            _warning("Error mixing of top-level vs declarations mixins",
                _makeSpan(include));
          }
        });
        declGroup.declarations.insertAll(0, newDecls);
        productions = [];
      } else {
        // Declarations are just @includes make it a list of productions
        // not a declaration group (anything else is a ruleset).  Make it a
        // list of productions, not a declaration group.
        for (var decl in declGroup.declarations) {
          productions.add(decl is IncludeMixinAtDeclaration ?
              decl.include : decl);
        };
        declGroup.declarations.clear();
      }

      if (declGroup.declarations.isNotEmpty) {
        if (productions.isEmpty) {
          mixinDirective = new MixinDeclarationDirective(name.name, params,
              false, declGroup, _makeSpan(start));
          break;
        } else {
          for (var decl in declGroup.declarations) {
            productions.add(decl is IncludeMixinAtDeclaration ?
                decl.include : decl);
          }
        }
      } else {
        mixinDirective = new MixinRulesetDirective(name.name, params,
            false, productions, _makeSpan(start));
        break;
      }
    }

    if (productions.isNotEmpty) {
      mixinDirective = new MixinRulesetDirective(name.name, params,
          false, productions, _makeSpan(start));
    }

    _eat(TokenKind.RBRACE);

    return mixinDirective;
  }

  /**
   * Returns a VarDefinitionDirective or VarDefinition if a varaible otherwise
   * return the token id of a directive or -1 if neither.
   */
  processVariableOrDirective({bool mixinParameter: false}) {
    int start = _peekToken.start;

    var tokId = _peek();
    // Handle case for @ directive (where there's a whitespace between the @
    // sign and the directive name.  Technically, it's not valid grammar but
    // a number of CSS tests test for whitespace between @ and name.
    if (tokId == TokenKind.AT) {
      Token tok = _next();
      tokId = _peek();
      if (_peekIdentifier()) {
        // Is it a directive?
        var directive = _peekToken.text;
        var directiveLen = directive.length;
        tokId = TokenKind.matchDirectives(directive, 0, directiveLen);
        if (tokId == -1) {
          tokId = TokenKind.matchMarginDirectives(directive, 0, directiveLen);
        }
      }

      if (tokId == -1) {
        if (messages.options.lessSupport) {
          // Less compatibility:
          //    @name: value;      =>    var-name: value;       (VarDefinition)
          //    property: @name;   =>    property: var(name);   (VarUsage)
          var name;
          if (_peekIdentifier()) {
            name = identifier();
          }

          Expressions exprs;
          if (mixinParameter && _maybeEat(TokenKind.COLON)) {
            exprs = processExpr();
          } else if (!mixinParameter) {
            _eat(TokenKind.COLON);
            exprs = processExpr();
          }

          var span = _makeSpan(start);
          return new VarDefinitionDirective(
              new VarDefinition(name, exprs, span), span);
        } else if (isChecked) {
          _error('unexpected directive @$_peekToken', _peekToken.span);
        }
      }
    } else if (mixinParameter && _peekToken.kind == TokenKind.VAR_DEFINITION) {
      _next();
      var definedName;
      if (_peekIdentifier()) definedName = identifier();

      Expressions exprs;
      if (_maybeEat(TokenKind.COLON)) {
        exprs = processExpr();
      }

      return new VarDefinition(definedName, exprs, _makeSpan(start));
    }

    return tokId;
  }

  IncludeDirective processInclude(SourceSpan span, {bool eatSemiColon: true}) {
    /* Stylet grammar:
    *
     *  @include IDENT [(args,...)];
     */
    _next();

    var name;
    if (_peekIdentifier()) {
      name = identifier();
    }

    var params = [];

    // Any parameters?  Parameters can be multiple terms per argument e.g.,
    // 3px solid yellow, green is two parameters:
    //    1. 3px solid yellow
    //    2. green
    // the first has 3 terms and the second has 1 term.
    if (_maybeEat(TokenKind.LPAREN)) {
      var terms = [];
      var expr;
      var keepGoing = true;
      while (keepGoing && (expr = processTerm()) != null) {
        // VarUsage is returns as a list
        terms.add(expr is List ? expr[0] : expr);
        keepGoing = !_peekKind(TokenKind.RPAREN);
        if (keepGoing) {
          if (_maybeEat(TokenKind.COMMA)) {
            params.add(terms);
            terms = [];
          }
        }
      }
      params.add(terms);
      _maybeEat(TokenKind.RPAREN);
    }

    if (eatSemiColon) {
      _eat(TokenKind.SEMICOLON);
    }

    return new IncludeDirective(name.name, params, span);
  }

  RuleSet processRuleSet([SelectorGroup selectorGroup]) {
    if (selectorGroup == null) {
      selectorGroup = processSelectorGroup();
    }
    if (selectorGroup != null) {
      return new RuleSet(selectorGroup, processDeclarations(),
          selectorGroup.span);
    }
  }

  /**
   * Look ahead to see if what should be a declaration is really a selector.
   * If it's a selector than it's a nested selector.  This support's Less'
   * nested selector syntax (requires a look ahead). E.g.,
   *
   *    div {
   *      width : 20px;
   *      span {
   *        color: red;
   *      }
   *    }
   *
   * Two tag name selectors div and span equivalent to:
   *
   *    div {
   *      width: 20px;
   *    }
   *    div span {
   *      color: red;
   *    }
   *
   * Return [:null:] if no selector or [SelectorGroup] if a selector was parsed.
   */
  SelectorGroup _nestedSelector() {
    Messages oldMessages = messages;
    _createMessages();

    var markedData = _mark;

    // Look a head do we have a nested selector instead of a declaration?
    SelectorGroup selGroup = processSelectorGroup();

    var nestedSelector = selGroup != null && _peekKind(TokenKind.LBRACE) &&
        messages.messages.isEmpty;

    if (!nestedSelector) {
      // Not a selector so restore the world.
      _restore(markedData);
      messages = oldMessages;
      return null;
    } else {
      // Remember any messages from look ahead.
      oldMessages.mergeMessages(messages);
      messages = oldMessages;
      return selGroup;
    }
  }

  DeclarationGroup processDeclarations({bool checkBrace: true}) {
    int start = _peekToken.start;

    if (checkBrace) _eat(TokenKind.LBRACE);

    List decls = [];
    List dartStyles = [];             // List of latest styles exposed to Dart.

    do {
      var selectorGroup = _nestedSelector();
      while (selectorGroup != null) {
        // Nested selector so process as a ruleset.
        var ruleset = processRuleSet(selectorGroup);
        decls.add(ruleset);
        selectorGroup = _nestedSelector();
      }

      Declaration decl = processDeclaration(dartStyles);
      if (decl != null) {
        if (decl.hasDartStyle) {
          var newDartStyle = decl.dartStyle;

          // Replace or add latest Dart style.
          bool replaced = false;
          for (var i = 0; i < dartStyles.length; i++) {
            var dartStyle = dartStyles[i];
            if (dartStyle.isSame(newDartStyle)) {
              dartStyles[i] = newDartStyle;
              replaced = true;
              break;
            }
          }
          if (!replaced) {
            dartStyles.add(newDartStyle);
          }
        }
        decls.add(decl);
      }
    } while (_maybeEat(TokenKind.SEMICOLON));

    if (checkBrace) _eat(TokenKind.RBRACE);

    // Fixup declaration to only have dartStyle that are live for this set of
    // declarations.
    for (var decl in decls) {
      if (decl is Declaration) {
        if (decl.hasDartStyle && dartStyles.indexOf(decl.dartStyle) < 0) {
          // Dart style not live, ignore these styles in this Declarations.
          decl.dartStyle = null;
        }
      }
    }

    return new DeclarationGroup(decls, _makeSpan(start));
  }

  List<DeclarationGroup> processMarginsDeclarations() {
    List groups = [];

    int start = _peekToken.start;

    _eat(TokenKind.LBRACE);

    List<Declaration> decls = [];
    List dartStyles = [];             // List of latest styles exposed to Dart.

    do {
      switch (_peek()) {
        case TokenKind.MARGIN_DIRECTIVE_TOPLEFTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_TOPLEFT:
        case TokenKind.MARGIN_DIRECTIVE_TOPCENTER:
        case TokenKind.MARGIN_DIRECTIVE_TOPRIGHT:
        case TokenKind.MARGIN_DIRECTIVE_TOPRIGHTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMLEFTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMLEFT:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMCENTER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMRIGHT:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMRIGHTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_LEFTTOP:
        case TokenKind.MARGIN_DIRECTIVE_LEFTMIDDLE:
        case TokenKind.MARGIN_DIRECTIVE_LEFTBOTTOM:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTTOP:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTMIDDLE:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTBOTTOM:
          // Margin syms processed.
          //   margin :
          //      margin_sym S* '{' declaration [ ';' S* declaration? ]* '}' S*
          //
          //      margin_sym : @top-left-corner, @top-left, @bottom-left, etc.
          var marginSym = _peek();

          _next();

          var declGroup = processDeclarations();
          if (declGroup != null) {
            groups.add(new MarginGroup(marginSym, declGroup.declarations,
                _makeSpan(start)));
          }
          break;
        default:
          Declaration decl = processDeclaration(dartStyles);
          if (decl != null) {
            if (decl.hasDartStyle) {
              var newDartStyle = decl.dartStyle;

              // Replace or add latest Dart style.
              bool replaced = false;
              for (var i = 0; i < dartStyles.length; i++) {
                var dartStyle = dartStyles[i];
                if (dartStyle.isSame(newDartStyle)) {
                  dartStyles[i] = newDartStyle;
                  replaced = true;
                  break;
                }
              }
              if (!replaced) {
                dartStyles.add(newDartStyle);
              }
            }
            decls.add(decl);
          }
          _maybeEat(TokenKind.SEMICOLON);
          break;
      }
    } while (!_maybeEat(TokenKind.RBRACE) && !isPrematureEndOfFile());

    // Fixup declaration to only have dartStyle that are live for this set of
    // declarations.
    for (var decl in decls) {
      if (decl.hasDartStyle && dartStyles.indexOf(decl.dartStyle) < 0) {
        // Dart style not live, ignore these styles in this Declarations.
        decl.dartStyle = null;
      }
    }

    if (decls.length > 0) {
      groups.add(new DeclarationGroup(decls, _makeSpan(start)));
    }

    return groups;
  }

  SelectorGroup processSelectorGroup() {
    List<Selector> selectors = [];
    int start = _peekToken.start;

    do {
      Selector selector = processSelector();
      if (selector != null) {
        selectors.add(selector);
      }
    } while (_maybeEat(TokenKind.COMMA));

    if (selectors.length > 0) {
      return new SelectorGroup(selectors, _makeSpan(start));
    }
  }

  /**
   * Return list of selectors
   */
  Selector processSelector() {
    var simpleSequences = <SimpleSelectorSequence>[];
    var start = _peekToken.start;
    while (true) {
      // First item is never descendant make sure it's COMBINATOR_NONE.
      var selectorItem = simpleSelectorSequence(simpleSequences.length == 0);
      if (selectorItem != null) {
        simpleSequences.add(selectorItem);
      } else {
        break;
      }
    }

    if (simpleSequences.length > 0) {
      return new Selector(simpleSequences, _makeSpan(start));
    }
  }

  simpleSelectorSequence(bool forceCombinatorNone) {
    var start = _peekToken.start;
    var combinatorType = TokenKind.COMBINATOR_NONE;
    var thisOperator = false;

    switch (_peek()) {
      case TokenKind.PLUS:
        _eat(TokenKind.PLUS);
        combinatorType = TokenKind.COMBINATOR_PLUS;
        break;
      case TokenKind.GREATER:
        _eat(TokenKind.GREATER);
        combinatorType = TokenKind.COMBINATOR_GREATER;
        break;
      case TokenKind.TILDE:
        _eat(TokenKind.TILDE);
        combinatorType = TokenKind.COMBINATOR_TILDE;
        break;
      case TokenKind.AMPERSAND:
        _eat(TokenKind.AMPERSAND);
        thisOperator = true;
        break;
      }

    // Check if WHITESPACE existed between tokens if so we're descendent.
    if (combinatorType == TokenKind.COMBINATOR_NONE && !forceCombinatorNone) {
      if (this._previousToken != null &&
          this._previousToken.end != this._peekToken.start) {
        combinatorType = TokenKind.COMBINATOR_DESCENDANT;
      }
    }

    var span = _makeSpan(start);
    var simpleSel = thisOperator ?
        new ElementSelector(new ThisOperator(span), span) : simpleSelector();
    if (simpleSel == null &&
        (combinatorType == TokenKind.COMBINATOR_PLUS ||
        combinatorType == TokenKind.COMBINATOR_GREATER ||
        combinatorType == TokenKind.COMBINATOR_TILDE)) {
      // For "+ &", "~ &" or "> &" a selector sequence with no name is needed
      // so that the & will have a combinator too.  This is needed to
      // disambiguate selector expressions:
      //    .foo&:hover     combinator before & is NONE
      //    .foo &          combinator before & is DESCDENDANT
      //    .foo > &        combinator before & is GREATER
      simpleSel =  new ElementSelector(new Identifier("", span), span);
    }
    if (simpleSel != null) {
      return new SimpleSelectorSequence(simpleSel, span, combinatorType);
    }
  }

  /**
   * Simple selector grammar:
   *
   *    simple_selector_sequence
   *       : [ type_selector | universal ]
   *         [ HASH | class | attrib | pseudo | negation ]*
   *       | [ HASH | class | attrib | pseudo | negation ]+
   *    type_selector
   *       : [ namespace_prefix ]? element_name
   *    namespace_prefix
   *       : [ IDENT | '*' ]? '|'
   *    element_name
   *       : IDENT
   *    universal
   *       : [ namespace_prefix ]? '*'
   *    class
   *       : '.' IDENT
   */
  simpleSelector() {
    // TODO(terry): Nathan makes a good point parsing of namespace and element
    //              are essentially the same (asterisk or identifier) other
    //              than the error message for element.  Should consolidate the
    //              code.
    // TODO(terry): Need to handle attribute namespace too.
    var first;
    int start = _peekToken.start;
    switch (_peek()) {
      case TokenKind.ASTERISK:
        // Mark as universal namespace.
        var tok = _next();
        first = new Wildcard(_makeSpan(tok.start));
        break;
      case TokenKind.IDENTIFIER:
        first = identifier();
        break;
      default:
        // Expecting simple selector.
        // TODO(terry): Could be a synthesized token like value, etc.
        if (TokenKind.isKindIdentifier(_peek())) {
          first = identifier();
        } else if (_peekKind(TokenKind.SEMICOLON)) {
          // Can't be a selector if we found a semi-colon.
          return null;
        }<commit_msg>Update my name in test data and comments.<commit_after>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

library csslib.parser;

import 'dart:math' as math;

import 'package:source_span/source_span.dart';

import "visitor.dart";
import 'src/messages.dart';
import 'src/options.dart';

part 'src/analyzer.dart';
part 'src/polyfill.dart';
part 'src/property.dart';
part 'src/token.dart';
part 'src/tokenizer_base.dart';
part 'src/tokenizer.dart';
part 'src/tokenkind.dart';


/** Used for parser lookup ahead (used for nested selectors Less support). */
class ParserState extends TokenizerState {
  final Token peekToken;
  final Token previousToken;

  ParserState(this.peekToken, this.previousToken, Tokenizer tokenizer)
      : super(tokenizer);
}

// TODO(jmesserly): this should not be global
void _createMessages({List<Message> errors, List<String> options}) {
  if (errors == null) errors = [];

  if (options == null) {
    options = ['--no-colors', 'memory'];
  }
  var opt = PreprocessorOptions.parse(options);
  messages = new Messages(options: opt, printHandler: errors.add);
}

/** CSS checked mode enabled. */
bool get isChecked => messages.options.checked;

// TODO(terry): Remove nested name parameter.
/** Parse and analyze the CSS file. */
StyleSheet compile(input, {List<Message> errors, List<String> options,
    bool nested: true,
    bool polyfill: false,
    List<StyleSheet> includes: null}) {

  if (includes == null) {
    includes = [];
  }

  var source = _inputAsString(input);

  _createMessages(errors: errors, options: options);

  var file = new SourceFile(source);

  var tree = new _Parser(file, source).parse();

  analyze([tree], errors: errors, options: options);

  if (polyfill) {
    var processCss = new PolyFill(messages, true);
    processCss.process(tree, includes: includes);
  }

  return tree;
}

/** Analyze the CSS file. */
void analyze(List<StyleSheet> styleSheets,
    {List<Message> errors, List<String> options}) {

  _createMessages(errors: errors, options: options);
  new Analyzer(styleSheets, messages).run();
}

/**
 * Parse the [input] CSS stylesheet into a tree. The [input] can be a [String],
 * or [List<int>] of bytes and returns a [StyleSheet] AST.  The optional
 * [errors] list will contain each error/warning as a [Message].
 */
StyleSheet parse(input, {List<Message> errors, List<String> options}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors, options: options);

  var file = new SourceFile(source);
  return new _Parser(file, source).parse();
}

/**
 * Parse the [input] CSS selector into a tree. The [input] can be a [String],
 * or [List<int>] of bytes and returns a [StyleSheet] AST.  The optional
 * [errors] list will contain each error/warning as a [Message].
 */
// TODO(jmesserly): should rename "parseSelector" and return Selector
StyleSheet selector(input, {List<Message> errors}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors);

  var file = new SourceFile(source);
  return (new _Parser(file, source)
      ..tokenizer.inSelector = true)
      .parseSelector();
}

SelectorGroup parseSelectorGroup(input, {List<Message> errors}) {
  var source = _inputAsString(input);

  _createMessages(errors: errors);

  var file = new SourceFile(source);
  return (new _Parser(file, source)
      // TODO(jmesserly): this fix should be applied to the parser. It's tricky
      // because by the time the flag is set one token has already been fetched.
      ..tokenizer.inSelector = true)
      .processSelectorGroup();
}

String _inputAsString(input) {
  String source;

  if (input is String) {
    source = input;
  } else if (input is List<int>) {
    // TODO(terry): The parse function needs an "encoding" argument and will
    //              default to whatever encoding CSS defaults to.
    //
    // Here's some info about CSS encodings:
    // http://www.w3.org/International/questions/qa-css-charset.en.php
    //
    // As JMesserly suggests it will probably need a "preparser" html5lib
    // (encoding_parser.dart) that interprets the bytes as ASCII and scans for
    // @charset. But for now an "encoding" argument would work.  Often the
    // HTTP header will indicate the correct encoding.
    //
    // See encoding helpers at: package:html5lib/lib/src/char_encodings.dart
    // These helpers can decode in different formats given an encoding name
    // (mostly unicode, ascii, windows-1252 which is html5 default encoding).
    source = new String.fromCharCodes(input);
  } else {
    // TODO(terry): Support RandomAccessFile using console.
    throw new ArgumentError("'source' must be a String or "
        "List<int> (of bytes). RandomAccessFile not supported from this "
        "simple interface");
  }

  return source;
}

// TODO(terry): Consider removing this class when all usages can be eliminated
//               or replaced with compile API.
/** Public parsing interface for csslib. */
class Parser {
  final _Parser _parser;

  // TODO(jmesserly): having file and text is redundant.
  Parser(SourceFile file, String text, {int start: 0, String baseUrl}) :
    _parser = new _Parser(file, text, start: start, baseUrl: baseUrl);

  StyleSheet parse() => _parser.parse();
}

/** A simple recursive descent parser for CSS. */
class _Parser {
  final Tokenizer tokenizer;

  /** Base url of CSS file. */
  final String _baseUrl;

  /**
   * File containing the source being parsed, used to report errors with
   * source-span locations.
   */
  final SourceFile file;

  Token _previousToken;
  Token _peekToken;

  _Parser(SourceFile file, String text, {int start: 0, String baseUrl})
      : this.file = file,
        _baseUrl = baseUrl,
        tokenizer = new Tokenizer(file, text, true, start) {
    _peekToken = tokenizer.next();
  }

  /** Main entry point for parsing an entire CSS file. */
  StyleSheet parse() {
    List<TreeNode> productions = [];

    int start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE) && !_peekKind(TokenKind.RBRACE)) {
      // TODO(terry): Need to handle charset.
      var directive = processDirective();
      if (directive != null) {
        productions.add(directive);
        _maybeEat(TokenKind.SEMICOLON);
      } else {
        RuleSet ruleset = processRuleSet();
        if (ruleset != null) {
          productions.add(ruleset);
        } else {
          break;
        }
      }
    }

    checkEndOfFile();

    return new StyleSheet(productions, _makeSpan(start));
  }

  /** Main entry point for parsing a simple selector sequence. */
  StyleSheet parseSelector() {
    List<TreeNode> productions = [];

    int start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE) && !_peekKind(TokenKind.RBRACE)) {
      var selector = processSelector();
      if (selector != null) {
        productions.add(selector);
      }
    }

    checkEndOfFile();

    return new StyleSheet.selector(productions, _makeSpan(start));
  }

  /** Generate an error if [file] has not been completely consumed. */
  void checkEndOfFile() {
    if (!(_peekKind(TokenKind.END_OF_FILE) ||
        _peekKind(TokenKind.INCOMPLETE_COMMENT))) {
      _error('premature end of file unknown CSS', _peekToken.span);
    }
  }

  /** Guard to break out of parser when an unexpected end of file is found. */
  // TODO(jimhug): Failure to call this method can lead to inifinite parser
  //   loops.  Consider embracing exceptions for more errors to reduce
  //   the danger here.
  bool isPrematureEndOfFile() {
    if (_maybeEat(TokenKind.END_OF_FILE)) {
      _error('unexpected end of file', _peekToken.span);
      return true;
    } else {
      return false;
    }
  }

  ///////////////////////////////////////////////////////////////////
  // Basic support methods
  ///////////////////////////////////////////////////////////////////
  int _peek() {
    return _peekToken.kind;
  }

  Token _next({unicodeRange : false}) {
    _previousToken = _peekToken;
    _peekToken = tokenizer.next(unicodeRange: unicodeRange);
    return _previousToken;
  }

  bool _peekKind(int kind) {
    return _peekToken.kind == kind;
  }

  /* Is the next token a legal identifier?  This includes pseudo-keywords. */
  bool _peekIdentifier() {
    return TokenKind.isIdentifier(_peekToken.kind);
  }

  /** Marks the parser/tokenizer look ahead to support Less nested selectors. */
  ParserState get _mark =>
      new ParserState(_peekToken, _previousToken, tokenizer);

  /** Restores the parser/tokenizer state to state remembered by _mark. */
  void _restore(ParserState markedData) {
    tokenizer.restore(markedData);
    _peekToken = markedData.peekToken;
    _previousToken = markedData.previousToken;
  }

  bool _maybeEat(int kind, {unicodeRange : false}) {
    if (_peekToken.kind == kind) {
      _previousToken = _peekToken;
      _peekToken = tokenizer.next(unicodeRange: unicodeRange);
      return true;
    } else {
      return false;
    }
  }

  void _eat(int kind, {unicodeRange : false}) {
    if (!_maybeEat(kind, unicodeRange: unicodeRange)) {
      _errorExpected(TokenKind.kindToString(kind));
    }
  }

  void _eatSemicolon() {
    _eat(TokenKind.SEMICOLON);
  }

  void _errorExpected(String expected) {
    var tok = _next();
    var message;
    try {
      message = 'expected $expected, but found $tok';
    } catch (e) {
      message = 'parsing error expected $expected';
    }
    _error(message, tok.span);
  }

  void _error(String message, SourceSpan location) {
    if (location == null) {
      location = _peekToken.span;
    }
    messages.error(message, location);
  }

  void _warning(String message, SourceSpan location) {
    if (location == null) {
      location = _peekToken.span;
    }
    messages.warning(message, location);
  }

  SourceSpan _makeSpan(int start) {
    // TODO(terry): there are places where we are creating spans before we eat
    // the tokens, so using _previousToken.end is not always valid.
    var end = _previousToken != null && _previousToken.end >= start
        ? _previousToken.end : _peekToken.end;
    return file.span(start, end);
  }

  ///////////////////////////////////////////////////////////////////
  // Top level productions
  ///////////////////////////////////////////////////////////////////

  /**
   * The media_query_list production below replaces the media_list production
   * from CSS2 the new grammar is:
   *
   *   media_query_list
   *    : S* [media_query [ ',' S* media_query ]* ]?
   *   media_query
   *    : [ONLY | NOT]? S* media_type S* [ AND S* expression ]*
   *    | expression [ AND S* expression ]*
   *   media_type
   *    : IDENT
   *   expression
   *    : '(' S* media_feature S* [ ':' S* expr ]? ')' S*
   *   media_feature
   *    : IDENT
   */
  List<MediaQuery> processMediaQueryList() {
    var mediaQueries = [];

    bool firstTime = true;
    var mediaQuery;
    do {
      mediaQuery = processMediaQuery(firstTime == true);
      if (mediaQuery != null) {
        mediaQueries.add(mediaQuery);
        firstTime = false;
        continue;
      }

      // Any more more media types separated by comma.
      if (!_maybeEat(TokenKind.COMMA)) break;

      // Yep more media types start again.
      firstTime = true;
    } while ((!firstTime && mediaQuery != null) || firstTime);

    return mediaQueries;
  }

  MediaQuery processMediaQuery([bool startQuery = true]) {
    // Grammar: [ONLY | NOT]? S* media_type S*
    //          [ AND S* MediaExpr ]* | MediaExpr [ AND S* MediaExpr ]*

    int start = _peekToken.start;

    // Is it a unary media operator?
    var op = _peekToken.text;
    var opLen = op.length;
    var unaryOp = TokenKind.matchMediaOperator(op, 0, opLen);
    if (unaryOp != -1) {
      if (isChecked) {
        if (startQuery &&
            unaryOp != TokenKind.MEDIA_OP_NOT ||
            unaryOp != TokenKind.MEDIA_OP_ONLY) {
          _warning("Only the unary operators NOT and ONLY allowed",
              _makeSpan(start));
        }
        if (!startQuery && unaryOp != TokenKind.MEDIA_OP_AND) {
          _warning("Only the binary AND operator allowed", _makeSpan(start));
        }
      }
      _next();
      start = _peekToken.start;
    }

    var type;
    if (startQuery && unaryOp != TokenKind.MEDIA_OP_AND) {
      // Get the media type.
      if (_peekIdentifier()) type = identifier();
    }

    var exprs = [];

    if (unaryOp == -1 || unaryOp == TokenKind.MEDIA_OP_AND) {
      var andOp = false;
      while (true) {
        var expr = processMediaExpression(andOp);
        if (expr == null) break;

        exprs.add(expr);
        op = _peekToken.text;
        opLen = op.length;
        andOp = TokenKind.matchMediaOperator(op, 0, opLen) ==
            TokenKind.MEDIA_OP_AND;
        if (!andOp) break;
        _next();
      }
    }

    if (unaryOp != -1 || type != null || exprs.length > 0) {
      return new MediaQuery(unaryOp, type, exprs, _makeSpan(start));
    }
  }

  MediaExpression processMediaExpression([bool andOperator = false]) {
    int start = _peekToken.start;

    // Grammar: '(' S* media_feature S* [ ':' S* expr ]? ')' S*
    if (_maybeEat(TokenKind.LPAREN)) {
      if (_peekIdentifier()) {
        var feature = identifier();           // Media feature.
        while (_maybeEat(TokenKind.COLON)) {
          int startExpr = _peekToken.start;
          var exprs = processExpr();
          if (_maybeEat(TokenKind.RPAREN)) {
            return new MediaExpression(andOperator, feature, exprs,
                _makeSpan(startExpr));
          } else if (isChecked) {
            _warning("Missing parenthesis around media expression",
                _makeSpan(start));
            return null;
          }
        }
      } else if (isChecked) {
        _warning("Missing media feature in media expression", _makeSpan(start));
        return null;
      }
    }
  }

  /**
   * Directive grammar:
   *
   *  import:             '@import' [string | URI] media_list?
   *  media:              '@media' media_query_list '{' ruleset '}'
   *  page:               '@page' [':' IDENT]? '{' declarations '}'
   *  stylet:             '@stylet' IDENT '{' ruleset '}'
   *  media_query_list:   IDENT [',' IDENT]
   *  keyframes:          '@-webkit-keyframes ...' (see grammar below).
   *  font_face:          '@font-face' '{' declarations '}'
   *  namespace:          '@namespace name url("xmlns")
   *  host:               '@host '{' ruleset '}'
   *  mixin:              '@mixin name [(args,...)] '{' declarations/ruleset '}'
   *  include:            '@include name [(@arg,@arg1)]
   *                      '@include name [(@arg...)]
   *  content             '@content'
   */
  processDirective() {
    int start = _peekToken.start;

    var tokId = processVariableOrDirective();
    if (tokId is VarDefinitionDirective) return tokId;
    switch (tokId) {
      case TokenKind.DIRECTIVE_IMPORT:
        _next();

        // @import "uri_string" or @import url("uri_string") are identical; only
        // a url can follow an @import.
        String importStr;
        if (_peekIdentifier()) {
          var func = processFunction(identifier());
          if (func is UriTerm) {
            importStr = func.text;
          }
        } else {
          importStr = processQuotedString(false);
        }

        // Any medias?
        var medias = processMediaQueryList();

        if (importStr == null) {
          _error('missing import string', _peekToken.span);
        }

        return new ImportDirective(importStr.trim(), medias, _makeSpan(start));

      case TokenKind.DIRECTIVE_MEDIA:
        _next();

        // Any medias?
        var media = processMediaQueryList();

        List<TreeNode> rulesets = [];
        if (_maybeEat(TokenKind.LBRACE)) {
          while (!_maybeEat(TokenKind.END_OF_FILE)) {
            RuleSet ruleset = processRuleSet();
            if (ruleset == null) break;
            rulesets.add(ruleset);
          }

          if (!_maybeEat(TokenKind.RBRACE)) {
            _error('expected } after ruleset for @media', _peekToken.span);
          }
        } else {
          _error('expected { after media before ruleset', _peekToken.span);
        }
        return new MediaDirective(media, rulesets, _makeSpan(start));

      case TokenKind.DIRECTIVE_HOST:
        _next();

        List<TreeNode> rulesets = [];
        if (_maybeEat(TokenKind.LBRACE)) {
          while (!_maybeEat(TokenKind.END_OF_FILE)) {
            RuleSet ruleset = processRuleSet();
            if (ruleset == null) break;
            rulesets.add(ruleset);
          }

          if (!_maybeEat(TokenKind.RBRACE)) {
            _error('expected } after ruleset for @host', _peekToken.span);
          }
        } else {
          _error('expected { after host before ruleset', _peekToken.span);
        }
        return new HostDirective(rulesets, _makeSpan(start));

      case TokenKind.DIRECTIVE_PAGE:
        /*
         * @page S* IDENT? pseudo_page?
         *      S* '{' S*
         *      [ declaration | margin ]?
         *      [ ';' S* [ declaration | margin ]? ]* '}' S*
         *
         * pseudo_page :
         *      ':' [ "left" | "right" | "first" ]
         *
         * margin :
         *      margin_sym S* '{' declaration [ ';' S* declaration? ]* '}' S*
         *
         * margin_sym : @top-left-corner, @top-left, @bottom-left, etc.
         *
         * See http://www.w3.org/TR/css3-page/#CSS21
         */
        _next();

        // Page name
        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        // Any pseudo page?
        var pseudoPage;
        if (_maybeEat(TokenKind.COLON)) {
          if (_peekIdentifier()) {
            pseudoPage = identifier();
            // TODO(terry): Normalize pseudoPage to lowercase.
            if (isChecked &&
                !(pseudoPage.name == 'left' ||
                  pseudoPage.name == 'right' ||
                  pseudoPage.name == 'first')) {
              _warning("Pseudo page must be left, top or first",
                  pseudoPage.span);
              return null;
            }
          }
        }

        String pseudoName = pseudoPage is Identifier ? pseudoPage.name : '';
        String ident = name is Identifier ? name.name : '';
        return new PageDirective(ident, pseudoName,
            processMarginsDeclarations(), _makeSpan(start));

      case TokenKind.DIRECTIVE_CHARSET:
        // @charset S* STRING S* ';'
        _next();

        var charEncoding = processQuotedString(false);
        if (isChecked && charEncoding == null) {
          // Missing character encoding.
          _warning('missing character encoding string', _makeSpan(start));
        }

        return new CharsetDirective(charEncoding, _makeSpan(start));

      // TODO(terry): Workaround Dart2js bug continue not implemented in switch
      //              see https://code.google.com/p/dart/issues/detail?id=8270
      /*
      case TokenKind.DIRECTIVE_MS_KEYFRAMES:
        // TODO(terry): For now only IE 10 (are base level) supports @keyframes,
        // -moz- has only been optional since Oct 2012 release of Firefox, not
        // all versions of webkit support @keyframes and opera doesn't yet
        // support w/o -o- prefix.  Add more warnings for other prefixes when
        // they become optional.
        if (isChecked) {
          _warning('@-ms-keyframes should be @keyframes', _makeSpan(start));
        }
        continue keyframeDirective;

      keyframeDirective:
      */
      case TokenKind.DIRECTIVE_KEYFRAMES:
      case TokenKind.DIRECTIVE_WEB_KIT_KEYFRAMES:
      case TokenKind.DIRECTIVE_MOZ_KEYFRAMES:
      case TokenKind.DIRECTIVE_O_KEYFRAMES:
      // TODO(terry): Remove workaround when bug 8270 is fixed.
      case TokenKind.DIRECTIVE_MS_KEYFRAMES:
        if (tokId == TokenKind.DIRECTIVE_MS_KEYFRAMES && isChecked) {
          _warning('@-ms-keyframes should be @keyframes', _makeSpan(start));
        }
      // TODO(terry): End of workaround.

        /*  Key frames grammar:
         *
         *  @[browser]? keyframes [IDENT|STRING] '{' keyframes-blocks '}';
         *
         *  browser: [-webkit-, -moz-, -ms-, -o-]
         *
         *  keyframes-blocks:
         *    [keyframe-selectors '{' declarations '}']* ;
         *
         *  keyframe-selectors:
         *    ['from'|'to'|PERCENTAGE] [',' ['from'|'to'|PERCENTAGE] ]* ;
         */
        _next();

        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        _eat(TokenKind.LBRACE);

        var keyframe = new KeyFrameDirective(tokId, name, _makeSpan(start));

        do {
          Expressions selectors = new Expressions(_makeSpan(start));

          do {
            var term = processTerm();

            // TODO(terry): Only allow from, to and PERCENTAGE ...

            selectors.add(term);
          } while (_maybeEat(TokenKind.COMMA));

          keyframe.add(new KeyFrameBlock(selectors, processDeclarations(),
              _makeSpan(start)));

        } while (!_maybeEat(TokenKind.RBRACE) && !isPrematureEndOfFile());

        return keyframe;

      case TokenKind.DIRECTIVE_FONTFACE:
        _next();
        return new FontFaceDirective(processDeclarations(), _makeSpan(start));

      case TokenKind.DIRECTIVE_STYLET:
        /* Stylet grammar:
         *
         *  @stylet IDENT '{'
         *    ruleset
         *  '}'
         */
        _next();

        var name;
        if (_peekIdentifier()) {
          name = identifier();
        }

        _eat(TokenKind.LBRACE);

        List<TreeNode> productions = [];

        start = _peekToken.start;
        while (!_maybeEat(TokenKind.END_OF_FILE)) {
          RuleSet ruleset = processRuleSet();
          if (ruleset == null) {
            break;
          }
          productions.add(ruleset);
        }

        _eat(TokenKind.RBRACE);

        return new StyletDirective(name, productions, _makeSpan(start));

      case TokenKind.DIRECTIVE_NAMESPACE:
        /* Namespace grammar:
         *
         * @namespace S* [namespace_prefix S*]? [STRING|URI] S* ';' S*
         * namespace_prefix : IDENT
         *
         */
        _next();

        var prefix;
        if (_peekIdentifier()) {
          prefix = identifier();
        }

        // The namespace URI can be either a quoted string url("uri_string")
        // are identical.
        String namespaceUri;
        if (_peekIdentifier()) {
          var func = processFunction(identifier());
          if (func is UriTerm) {
            namespaceUri = func.text;
          }
        } else {
          if (prefix != null && prefix.name == 'url') {
            var func = processFunction(prefix);
            if (func is UriTerm) {
              // @namespace url("");
              namespaceUri = func.text;
              prefix = null;
            }
          } else {
            namespaceUri = processQuotedString(false);
          }
        }

        return new NamespaceDirective(prefix != null ? prefix.name : '',
            namespaceUri, _makeSpan(start));

      case TokenKind.DIRECTIVE_MIXIN:
        return processMixin(start);

      case TokenKind.DIRECTIVE_INCLUDE:
        return processInclude( _makeSpan(start));

      case TokenKind.DIRECTIVE_CONTENT:
        // TODO(terry): TBD
        _warning("@content not implemented.", _makeSpan(start));
        return null;
    }
    return null;
  }

  /**
   * Parse the mixin beginning token offset [start]. Returns a [MixinDefinition]
   * node.
   *
   * Mixin grammar:
   *
   *  @mixin IDENT [(args,...)] '{'
   *    [ruleset | property | directive]*
   *  '}'
   */
  MixinDefinition processMixin(int start) {
    _next();

    var name = identifier();

    List<VarDefinitionDirective> params = [];
    // Any parameters?
    if (_maybeEat(TokenKind.LPAREN)) {
      var mustHaveParam = false;
      var keepGoing = true;
      while (keepGoing) {
        var varDef = processVariableOrDirective(mixinParameter: true);
        if (varDef is VarDefinitionDirective || varDef is VarDefinition) {
          params.add(varDef);
        } else if (mustHaveParam) {
          _warning("Expecting parameter", _makeSpan(_peekToken.start));
          keepGoing = false;
        }
        if (_maybeEat(TokenKind.COMMA)) {
          mustHaveParam = true;
          continue;
        }
        keepGoing = !_maybeEat(TokenKind.RPAREN);
      }
    }

    _eat(TokenKind.LBRACE);

    List<TreeNode> productions = [];
    List<TreeNode> declarations = [];
    var mixinDirective;

    start = _peekToken.start;
    while (!_maybeEat(TokenKind.END_OF_FILE)) {
      var directive = processDirective();
      if (directive != null) {
        productions.add(directive);
        continue;
      }

      var declGroup = processDeclarations(checkBrace: false);
      var decls = [];
      if (declGroup.declarations.any((decl) {
        return decl is Declaration &&
            decl is! IncludeMixinAtDeclaration;
      })) {
        var newDecls = [];
        productions.forEach((include) {
          // If declGroup has items that are declarations then we assume
          // this mixin is a declaration mixin not a top-level mixin.
          if (include is IncludeDirective) {
            newDecls.add(new IncludeMixinAtDeclaration(include,
                include.span));
          } else {
            _warning("Error mixing of top-level vs declarations mixins",
                _makeSpan(include));
          }
        });
        declGroup.declarations.insertAll(0, newDecls);
        productions = [];
      } else {
        // Declarations are just @includes make it a list of productions
        // not a declaration group (anything else is a ruleset).  Make it a
        // list of productions, not a declaration group.
        for (var decl in declGroup.declarations) {
          productions.add(decl is IncludeMixinAtDeclaration ?
              decl.include : decl);
        };
        declGroup.declarations.clear();
      }

      if (declGroup.declarations.isNotEmpty) {
        if (productions.isEmpty) {
          mixinDirective = new MixinDeclarationDirective(name.name, params,
              false, declGroup, _makeSpan(start));
          break;
        } else {
          for (var decl in declGroup.declarations) {
            productions.add(decl is IncludeMixinAtDeclaration ?
                decl.include : decl);
          }
        }
      } else {
        mixinDirective = new MixinRulesetDirective(name.name, params,
            false, productions, _makeSpan(start));
        break;
      }
    }

    if (productions.isNotEmpty) {
      mixinDirective = new MixinRulesetDirective(name.name, params,
          false, productions, _makeSpan(start));
    }

    _eat(TokenKind.RBRACE);

    return mixinDirective;
  }

  /**
   * Returns a VarDefinitionDirective or VarDefinition if a varaible otherwise
   * return the token id of a directive or -1 if neither.
   */
  processVariableOrDirective({bool mixinParameter: false}) {
    int start = _peekToken.start;

    var tokId = _peek();
    // Handle case for @ directive (where there's a whitespace between the @
    // sign and the directive name.  Technically, it's not valid grammar but
    // a number of CSS tests test for whitespace between @ and name.
    if (tokId == TokenKind.AT) {
      Token tok = _next();
      tokId = _peek();
      if (_peekIdentifier()) {
        // Is it a directive?
        var directive = _peekToken.text;
        var directiveLen = directive.length;
        tokId = TokenKind.matchDirectives(directive, 0, directiveLen);
        if (tokId == -1) {
          tokId = TokenKind.matchMarginDirectives(directive, 0, directiveLen);
        }
      }

      if (tokId == -1) {
        if (messages.options.lessSupport) {
          // Less compatibility:
          //    @name: value;      =>    var-name: value;       (VarDefinition)
          //    property: @name;   =>    property: var(name);   (VarUsage)
          var name;
          if (_peekIdentifier()) {
            name = identifier();
          }

          Expressions exprs;
          if (mixinParameter && _maybeEat(TokenKind.COLON)) {
            exprs = processExpr();
          } else if (!mixinParameter) {
            _eat(TokenKind.COLON);
            exprs = processExpr();
          }

          var span = _makeSpan(start);
          return new VarDefinitionDirective(
              new VarDefinition(name, exprs, span), span);
        } else if (isChecked) {
          _error('unexpected directive @$_peekToken', _peekToken.span);
        }
      }
    } else if (mixinParameter && _peekToken.kind == TokenKind.VAR_DEFINITION) {
      _next();
      var definedName;
      if (_peekIdentifier()) definedName = identifier();

      Expressions exprs;
      if (_maybeEat(TokenKind.COLON)) {
        exprs = processExpr();
      }

      return new VarDefinition(definedName, exprs, _makeSpan(start));
    }

    return tokId;
  }

  IncludeDirective processInclude(SourceSpan span, {bool eatSemiColon: true}) {
    /* Stylet grammar:
    *
     *  @include IDENT [(args,...)];
     */
    _next();

    var name;
    if (_peekIdentifier()) {
      name = identifier();
    }

    var params = [];

    // Any parameters?  Parameters can be multiple terms per argument e.g.,
    // 3px solid yellow, green is two parameters:
    //    1. 3px solid yellow
    //    2. green
    // the first has 3 terms and the second has 1 term.
    if (_maybeEat(TokenKind.LPAREN)) {
      var terms = [];
      var expr;
      var keepGoing = true;
      while (keepGoing && (expr = processTerm()) != null) {
        // VarUsage is returns as a list
        terms.add(expr is List ? expr[0] : expr);
        keepGoing = !_peekKind(TokenKind.RPAREN);
        if (keepGoing) {
          if (_maybeEat(TokenKind.COMMA)) {
            params.add(terms);
            terms = [];
          }
        }
      }
      params.add(terms);
      _maybeEat(TokenKind.RPAREN);
    }

    if (eatSemiColon) {
      _eat(TokenKind.SEMICOLON);
    }

    return new IncludeDirective(name.name, params, span);
  }

  RuleSet processRuleSet([SelectorGroup selectorGroup]) {
    if (selectorGroup == null) {
      selectorGroup = processSelectorGroup();
    }
    if (selectorGroup != null) {
      return new RuleSet(selectorGroup, processDeclarations(),
          selectorGroup.span);
    }
  }

  /**
   * Look ahead to see if what should be a declaration is really a selector.
   * If it's a selector than it's a nested selector.  This support's Less'
   * nested selector syntax (requires a look ahead). E.g.,
   *
   *    div {
   *      width : 20px;
   *      span {
   *        color: red;
   *      }
   *    }
   *
   * Two tag name selectors div and span equivalent to:
   *
   *    div {
   *      width: 20px;
   *    }
   *    div span {
   *      color: red;
   *    }
   *
   * Return [:null:] if no selector or [SelectorGroup] if a selector was parsed.
   */
  SelectorGroup _nestedSelector() {
    Messages oldMessages = messages;
    _createMessages();

    var markedData = _mark;

    // Look a head do we have a nested selector instead of a declaration?
    SelectorGroup selGroup = processSelectorGroup();

    var nestedSelector = selGroup != null && _peekKind(TokenKind.LBRACE) &&
        messages.messages.isEmpty;

    if (!nestedSelector) {
      // Not a selector so restore the world.
      _restore(markedData);
      messages = oldMessages;
      return null;
    } else {
      // Remember any messages from look ahead.
      oldMessages.mergeMessages(messages);
      messages = oldMessages;
      return selGroup;
    }
  }

  DeclarationGroup processDeclarations({bool checkBrace: true}) {
    int start = _peekToken.start;

    if (checkBrace) _eat(TokenKind.LBRACE);

    List decls = [];
    List dartStyles = [];             // List of latest styles exposed to Dart.

    do {
      var selectorGroup = _nestedSelector();
      while (selectorGroup != null) {
        // Nested selector so process as a ruleset.
        var ruleset = processRuleSet(selectorGroup);
        decls.add(ruleset);
        selectorGroup = _nestedSelector();
      }

      Declaration decl = processDeclaration(dartStyles);
      if (decl != null) {
        if (decl.hasDartStyle) {
          var newDartStyle = decl.dartStyle;

          // Replace or add latest Dart style.
          bool replaced = false;
          for (var i = 0; i < dartStyles.length; i++) {
            var dartStyle = dartStyles[i];
            if (dartStyle.isSame(newDartStyle)) {
              dartStyles[i] = newDartStyle;
              replaced = true;
              break;
            }
          }
          if (!replaced) {
            dartStyles.add(newDartStyle);
          }
        }
        decls.add(decl);
      }
    } while (_maybeEat(TokenKind.SEMICOLON));

    if (checkBrace) _eat(TokenKind.RBRACE);

    // Fixup declaration to only have dartStyle that are live for this set of
    // declarations.
    for (var decl in decls) {
      if (decl is Declaration) {
        if (decl.hasDartStyle && dartStyles.indexOf(decl.dartStyle) < 0) {
          // Dart style not live, ignore these styles in this Declarations.
          decl.dartStyle = null;
        }
      }
    }

    return new DeclarationGroup(decls, _makeSpan(start));
  }

  List<DeclarationGroup> processMarginsDeclarations() {
    List groups = [];

    int start = _peekToken.start;

    _eat(TokenKind.LBRACE);

    List<Declaration> decls = [];
    List dartStyles = [];             // List of latest styles exposed to Dart.

    do {
      switch (_peek()) {
        case TokenKind.MARGIN_DIRECTIVE_TOPLEFTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_TOPLEFT:
        case TokenKind.MARGIN_DIRECTIVE_TOPCENTER:
        case TokenKind.MARGIN_DIRECTIVE_TOPRIGHT:
        case TokenKind.MARGIN_DIRECTIVE_TOPRIGHTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMLEFTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMLEFT:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMCENTER:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMRIGHT:
        case TokenKind.MARGIN_DIRECTIVE_BOTTOMRIGHTCORNER:
        case TokenKind.MARGIN_DIRECTIVE_LEFTTOP:
        case TokenKind.MARGIN_DIRECTIVE_LEFTMIDDLE:
        case TokenKind.MARGIN_DIRECTIVE_LEFTBOTTOM:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTTOP:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTMIDDLE:
        case TokenKind.MARGIN_DIRECTIVE_RIGHTBOTTOM:
          // Margin syms processed.
          //   margin :
          //      margin_sym S* '{' declaration [ ';' S* declaration? ]* '}' S*
          //
          //      margin_sym : @top-left-corner, @top-left, @bottom-left, etc.
          var marginSym = _peek();

          _next();

          var declGroup = processDeclarations();
          if (declGroup != null) {
            groups.add(new MarginGroup(marginSym, declGroup.declarations,
                _makeSpan(start)));
          }
          break;
        default:
          Declaration decl = processDeclaration(dartStyles);
          if (decl != null) {
            if (decl.hasDartStyle) {
              var newDartStyle = decl.dartStyle;

              // Replace or add latest Dart style.
              bool replaced = false;
              for (var i = 0; i < dartStyles.length; i++) {
                var dartStyle = dartStyles[i];
                if (dartStyle.isSame(newDartStyle)) {
                  dartStyles[i] = newDartStyle;
                  replaced = true;
                  break;
                }
              }
              if (!replaced) {
                dartStyles.add(newDartStyle);
              }
            }
            decls.add(decl);
          }
          _maybeEat(TokenKind.SEMICOLON);
          break;
      }
    } while (!_maybeEat(TokenKind.RBRACE) && !isPrematureEndOfFile());

    // Fixup declaration to only have dartStyle that are live for this set of
    // declarations.
    for (var decl in decls) {
      if (decl.hasDartStyle && dartStyles.indexOf(decl.dartStyle) < 0) {
        // Dart style not live, ignore these styles in this Declarations.
        decl.dartStyle = null;
      }
    }

    if (decls.length > 0) {
      groups.add(new DeclarationGroup(decls, _makeSpan(start)));
    }

    return groups;
  }

  SelectorGroup processSelectorGroup() {
    List<Selector> selectors = [];
    int start = _peekToken.start;

    do {
      Selector selector = processSelector();
      if (selector != null) {
        selectors.add(selector);
      }
    } while (_maybeEat(TokenKind.COMMA));

    if (selectors.length > 0) {
      return new SelectorGroup(selectors, _makeSpan(start));
    }
  }

  /**
   * Return list of selectors
   */
  Selector processSelector() {
    var simpleSequences = <SimpleSelectorSequence>[];
    var start = _peekToken.start;
    while (true) {
      // First item is never descendant make sure it's COMBINATOR_NONE.
      var selectorItem = simpleSelectorSequence(simpleSequences.length == 0);
      if (selectorItem != null) {
        simpleSequences.add(selectorItem);
      } else {
        break;
      }
    }

    if (simpleSequences.length > 0) {
      return new Selector(simpleSequences, _makeSpan(start));
    }
  }

  simpleSelectorSequence(bool forceCombinatorNone) {
    var start = _peekToken.start;
    var combinatorType = TokenKind.COMBINATOR_NONE;
    var thisOperator = false;

    switch (_peek()) {
      case TokenKind.PLUS:
        _eat(TokenKind.PLUS);
        combinatorType = TokenKind.COMBINATOR_PLUS;
        break;
      case TokenKind.GREATER:
        _eat(TokenKind.GREATER);
        combinatorType = TokenKind.COMBINATOR_GREATER;
        break;
      case TokenKind.TILDE:
        _eat(TokenKind.TILDE);
        combinatorType = TokenKind.COMBINATOR_TILDE;
        break;
      case TokenKind.AMPERSAND:
        _eat(TokenKind.AMPERSAND);
        thisOperator = true;
        break;
      }

    // Check if WHITESPACE existed between tokens if so we're descendent.
    if (combinatorType == TokenKind.COMBINATOR_NONE && !forceCombinatorNone) {
      if (this._previousToken != null &&
          this._previousToken.end != this._peekToken.start) {
        combinatorType = TokenKind.COMBINATOR_DESCENDANT;
      }
    }

    var span = _makeSpan(start);
    var simpleSel = thisOperator ?
        new ElementSelector(new ThisOperator(span), span) : simpleSelector();
    if (simpleSel == null &&
        (combinatorType == TokenKind.COMBINATOR_PLUS ||
        combinatorType == TokenKind.COMBINATOR_GREATER ||
        combinatorType == TokenKind.COMBINATOR_TILDE)) {
      // For "+ &", "~ &" or "> &" a selector sequence with no name is needed
      // so that the & will have a combinator too.  This is needed to
      // disambiguate selector expressions:
      //    .foo&:hover     combinator before & is NONE
      //    .foo &          combinator before & is DESCDENDANT
      //    .foo > &        combinator before & is GREATER
      simpleSel =  new ElementSelector(new Identifier("", span), span);
    }
    if (simpleSel != null) {
      return new SimpleSelectorSequence(simpleSel, span, combinatorType);
    }
  }

  /**
   * Simple selector grammar:
   *
   *    simple_selector_sequence
   *       : [ type_selector | universal ]
   *         [ HASH | class | attrib | pseudo | negation ]*
   *       | [ HASH | class | attrib | pseudo | negation ]+
   *    type_selector
   *       : [ namespace_prefix ]? element_name
   *    namespace_prefix
   *       : [ IDENT | '*' ]? '|'
   *    element_name
   *       : IDENT
   *    universal
   *       : [ namespace_prefix ]? '*'
   *    class
   *       : '.' IDENT
   */
  simpleSelector() {
    // TODO(terry): Natalie makes a good point parsing of namespace and element
    //              are essentially the same (asterisk or identifier) other
    //              than the error message for element.  Should consolidate the
    //              code.
    // TODO(terry): Need to handle attribute namespace too.
    var first;
    int start = _peekToken.start;
    switch (_peek()) {
      case TokenKind.ASTERISK:
        // Mark as universal namespace.
        var tok = _next();
        first = new Wildcard(_makeSpan(tok.start));
        break;
      case TokenKind.IDENTIFIER:
        first = identifier();
        break;
      default:
        // Expecting simple selector.
        // TODO(terry): Could be a synthesized token like value, etc.
        if (TokenKind.isKindIdentifier(_peek())) {
          first = identifier();
        } else if (_peekKind(TokenKind.SEMICOLON)) {
          // Can't be a selector if we found a semi-colon.
          return null;
        }
{'fraction_non_alphanumeric': 0.1169507055158512, 'fraction_numerical': 0.0016126076598863845, 'mean_word_length': 2.1506100611985683, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 85, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6367942', 'n_tokens_mistral': 24671, 'n_tokens_neox': 23744, 'n_words': 6219}