<commit_before>"use strict"
let NodeStream = require('stream')
let Path = require('path')
let FS = require('fs')
class Plugin{
  static Initialize(){
    this.Tags = new Map()
    this.Tags.set('Compiler-Output', function(Name, Value, Buffer, Options){
      Options.Output = Path.resolve(Path.dirname(Options.File.Path), Value)
    })
    this.Tags.set('Compiler-Include', function(Name, Value, Buffer, Options){
      return new Promise(function(Resolve){
        FS.readFile(Path.resolve(Path.dirname(Options.File.Path), Value), function(_, Contents){
          Resolve(Contents || ' ')
        })
      })
    })
    this.Info = {
      Name: "Plugin",
      Priority: 1,
      Extensions: []
    }
  }
  static Stream(Options){
    let Me = this
    var Stream = new NodeStream.Transform({
      objectMode: true,
      transform: function (Buffer, Encoding, Callback) {
        new Promise(function (Resolve) {
          Resolve(Me.Process(Buffer.toString("utf8"), Options))
        }).then(function(Contents){
            Stream.push(Contents)
            Callback()
          }, function(e){
            Stream.emit('error', e)
          })
      }
    })
    return Stream
  }
  static ProcessTags(CommentToken, Buffer, Options){
    let Regex = new RegExp(' *' + CommentToken.replace('//', '\\/\\/') + ' @([\\w-]+) "(.+)"', 'g')
    let Results = []
    let Sequence = Promise.resolve()
    for(let Result; (Result = Regex.exec(Buffer)) !== null;){
      let Value
      if(this.Tags.has(Result[1])){
        Value = this.Tags.get(Result[1])(Result[1], Result[2], Buffer, Options) || ''
      } else Value = Result[0]
      Sequence = Sequence.then(function(){
        if(Value && Value.constructor.name === 'Promise'){
          return Value.then(function(Value){
            Results.push(Value)
          })
        } else Results.push(Value)
      })
    }
    return Sequence.then(function(){
      let Regex = new RegExp(' *' + CommentToken.replace('//', '\\/\\/') + ' @([\\w-]+) "(.+)"', 'g')
      return Buffer.replace(Regex, function(){
        return Results.shift()
      })
    })
  }
  static Validate(Options){<commit_msg>:art: Run all tags in parallel<commit_after>"use strict"
let NodeStream = require('stream')
let Path = require('path')
let FS = require('fs')
class Plugin{
  static Initialize(){
    this.Tags = new Map()
    this.Tags.set('Compiler-Output', function(Name, Value, Buffer, Options){
      Options.Output = Path.resolve(Path.dirname(Options.File.Path), Value)
    })
    this.Tags.set('Compiler-Include', function(Name, Value, Buffer, Options){
      return new Promise(function(Resolve){
        FS.readFile(Path.resolve(Path.dirname(Options.File.Path), Value), function(_, Contents){
          Resolve(Contents || ' ')
        })
      })
    })
    this.Info = {
      Name: "Plugin",
      Priority: 1,
      Extensions: []
    }
  }
  static Stream(Options){
    let Me = this
    var Stream = new NodeStream.Transform({
      objectMode: true,
      transform: function (Buffer, Encoding, Callback) {
        new Promise(function (Resolve) {
          Resolve(Me.Process(Buffer.toString("utf8"), Options))
        }).then(function(Contents){
            Stream.push(Contents)
            Callback()
          }, function(e){
            Stream.emit('error', e)
          })
      }
    })
    return Stream
  }
  static ProcessTags(CommentToken, Buffer, Options){
    let Regex = new RegExp(' *' + CommentToken.replace('//', '\\/\\/') + ' @([\\w-]+) "(.+)"', 'g')
    let Promises = []
    for(let Result; (Result = Regex.exec(Buffer)) !== null;){
      let Value
      if(this.Tags.has(Result[1])){
        Value = this.Tags.get(Result[1])(Result[1], Result[2], Buffer, Options) || ''
      } else Value = Result[0]
      Promises.push(Value)
    }
    return Promise.all(Promises).then(function(Results){
      let Regex = new RegExp(' *' + CommentToken.replace('//', '\\/\\/') + ' @([\\w-]+) "(.+)"', 'g')
      return Buffer.replace(Regex, function(){
        return Results.shift()
      })
    })
  }
  static Validate(Options){
{'fraction_non_alphanumeric': 0.1772771792360431, 'fraction_numerical': 0.0034280117531831538, 'mean_word_length': 2.533737024221453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '605631', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1231, 'n_words': 256}