<commit_before>require 'parslet'

module Channel9
  module Script
    class Parser < Parslet::Parser
      rule(:ws) { match('[ \t]').repeat(1) }
      rule(:ws?) { ws.maybe }
      rule(:br) { str("\n") }

      rule(:line_comment_text) { (br.absnt? >> any).repeat }
      rule(:multiline_comment_text) { (str('*/').absnt? >> any).repeat }

      rule(:comment) { 
        str('#') >> line_comment_text.as(:text) |
        str('//') >> line_comment_text.as(:text) |
        str('/*') >> multiline_comment_text.as(:text) >> str('*/')
      }

      # Empty whitespace is any whitespace that has no content of any sort (
      # so unlike iws, it doesn't include comments).
      # Basically, it's whitespace before something really starts.
      rule(:ews) { (ws | br).repeat(1) }
      rule(:ews?) { ews.maybe }

      # Line whitespace is any kind of whitespace on the same line (including comments).
      # Use for stuff that shouldn't go to the next line, but still expects something
      # more to complete the expression (eg. a >> lws >> '+' >> iws >> b)
      rule(:lws) { (ws | comment.as(:inline_doc)).repeat(1) }
      rule(:lws?) { lws.maybe }

      # Inner whitespace is space between elements of the same line.
      # It can include spaces, line breaks, and comments.
      rule(:iws) { (ws | br | comment.as(:inline_doc)).repeat(1) }
      rule(:iws?) { iws.maybe }

      # End of line can be any amount of whitespace, including comments,
      # and ending in either a \n or semicolon
      rule(:eol) { (ws | comment.as(:inline_doc)).repeat >> (br | str(';')) }

      rule(:symbol) {
        match('[a-zA-Z_]') >> match('[a-zA-Z0-9_]').repeat
      }

      rule(:local_var) {
        symbol.as(:local_var)
      }
      rule(:declare_var) {
        (str("var") >> lws >> symbol.as(:declare_var) >> lws? >> str("=") >> lws? >> expression.as(:assign)) |
        (str("var") >> lws >> symbol.as(:declare_var))
      }
      rule(:special_var) {
        str('$') >> symbol.as(:special_var)
      }

      rule(:variable) { # Note: order IS important here.
        (declare_var | local_var | special_var)
      }

      rule(:nil_const) {
        str("nil").as(:nil)
      }
      rule(:undef_const) {
        str("undef").as(:undef)
      }
      rule(:true_const) {
        str("true").as(:true)
      }
      rule(:false_const) {
        str("false").as(:false)
      }

      rule(:integer_const) {
        match('[0-9]').repeat(1).as(:integer)
      }

      rule(:string_const) {
        ((str('"') >> (str('"').absnt? >> any).repeat.as(:string) >> str('"')) |
         (str("'") >> (str("'").absnt? >> any).repeat.as(:string) >> str("'")) |
         (str(":") >> (ws.absnt? >> any).repeat.as(:string))
        )
      }

      rule(:list_const) {
        (str("[") >> iws? >> (expression >> iws? >> str(",").maybe).repeat >> iws? >> str("]")).as(:list)
      }

      rule(:argdef) {
        local_var >> (iws? >> str(',') >> iws? >> local_var).repeat
      }

      rule(:argdef_list) {
        (str('(') >> iws? >> argdef.as(:args) >> iws? >> str(",") >> iws? >> str('&') >> symbol.as(:msg_var) >> iws? >> str(')')) |
        (str('(') >> iws? >> str('&') >> symbol.as(:msg_var) >> iws? >> str(')')) |
        (str('(') >> iws? >> argdef.maybe.as(:args) >> iws? >> str(')'))
      }

      rule(:const) {
        nil_const | undef_const | true_const | false_const | integer_const | string_const | list_const<commit_msg>Handle empty strings.<commit_after>require 'parslet'

module Channel9
  module Script
    class Parser < Parslet::Parser
      rule(:ws) { match('[ \t]').repeat(1) }
      rule(:ws?) { ws.maybe }
      rule(:br) { str("\n") }

      rule(:line_comment_text) { (br.absnt? >> any).repeat }
      rule(:multiline_comment_text) { (str('*/').absnt? >> any).repeat }

      rule(:comment) { 
        str('#') >> line_comment_text.as(:text) |
        str('//') >> line_comment_text.as(:text) |
        str('/*') >> multiline_comment_text.as(:text) >> str('*/')
      }

      # Empty whitespace is any whitespace that has no content of any sort (
      # so unlike iws, it doesn't include comments).
      # Basically, it's whitespace before something really starts.
      rule(:ews) { (ws | br).repeat(1) }
      rule(:ews?) { ews.maybe }

      # Line whitespace is any kind of whitespace on the same line (including comments).
      # Use for stuff that shouldn't go to the next line, but still expects something
      # more to complete the expression (eg. a >> lws >> '+' >> iws >> b)
      rule(:lws) { (ws | comment.as(:inline_doc)).repeat(1) }
      rule(:lws?) { lws.maybe }

      # Inner whitespace is space between elements of the same line.
      # It can include spaces, line breaks, and comments.
      rule(:iws) { (ws | br | comment.as(:inline_doc)).repeat(1) }
      rule(:iws?) { iws.maybe }

      # End of line can be any amount of whitespace, including comments,
      # and ending in either a \n or semicolon
      rule(:eol) { (ws | comment.as(:inline_doc)).repeat >> (br | str(';')) }

      rule(:symbol) {
        match('[a-zA-Z_]') >> match('[a-zA-Z0-9_]').repeat
      }

      rule(:local_var) {
        symbol.as(:local_var)
      }
      rule(:declare_var) {
        (str("var") >> lws >> symbol.as(:declare_var) >> lws? >> str("=") >> lws? >> expression.as(:assign)) |
        (str("var") >> lws >> symbol.as(:declare_var))
      }
      rule(:special_var) {
        str('$') >> symbol.as(:special_var)
      }

      rule(:variable) { # Note: order IS important here.
        (declare_var | local_var | special_var)
      }

      rule(:nil_const) {
        str("nil").as(:nil)
      }
      rule(:undef_const) {
        str("undef").as(:undef)
      }
      rule(:true_const) {
        str("true").as(:true)
      }
      rule(:false_const) {
        str("false").as(:false)
      }

      rule(:integer_const) {
        match('[0-9]').repeat(1).as(:integer)
      }

      rule(:string_const) {
        ((str('"') >> str('"').absent?.maybe.as(:string) >> str('"')) |
         (str("'") >> str("'").absent?.maybe.as(:string) >> str("'")) |
         (str('"') >> (str('"').absnt? >> any).repeat.as(:string) >> str('"')) |
         (str("'") >> (str("'").absnt? >> any).repeat.as(:string) >> str("'")) |
         (str(":") >> (ws.absnt? >> any).repeat.as(:string))
        )
      }

      rule(:list_const) {
        (str("[") >> iws? >> (expression >> iws? >> str(",").maybe).repeat >> iws? >> str("]")).as(:list)
      }

      rule(:argdef) {
        local_var >> (iws? >> str(',') >> iws? >> local_var).repeat
      }

      rule(:argdef_list) {
        (str('(') >> iws? >> argdef.as(:args) >> iws? >> str(",") >> iws? >> str('&') >> symbol.as(:msg_var) >> iws? >> str(')')) |
        (str('(') >> iws? >> str('&') >> symbol.as(:msg_var) >> iws? >> str(')')) |
        (str('(') >> iws? >> argdef.maybe.as(:args) >> iws? >> str(')'))
      }

      rule(:const) {
        nil_const | undef_const | true_const | false_const | integer_const | string_const | list_const
{'fraction_non_alphanumeric': 0.21450880181714935, 'fraction_numerical': 0.0028392958546280523, 'mean_word_length': 2.4602161100196462, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 239, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '750210', 'n_tokens_mistral': 2441, 'n_tokens_neox': 2380, 'n_words': 497}