<commit_before>/* @flow */

import _ from 'lodash'
import path from 'path'
import readdir from 'readdir-enhanced'

import State from './State'
import StateConsumer from './StateConsumer'
import File from './File'
import Rule from './Rule'

import type { Action, Command, Option, Phase, RuleInfo } from './types'

export default class DiCy extends StateConsumer {
  static async create (filePath: string, options: Object = {}) {
    const schema = await DiCy.getOptionDefinitions()
    const state = await State.create(filePath, schema)
    const builder = new DiCy(state)

    await builder.initialize()
    await builder.setInstanceOptions(options)

    state.assignOptions(options)

    return builder
  }

  async initialize () {
    const ruleClassPath: string = path.join(__dirname, 'Rules')
    const entries: Array<string> = await readdir.async(ruleClassPath)
    this.state.ruleClasses = entries
      .map(entry => require(path.join(ruleClassPath, entry)).default)
  }

  async setInstanceOptions (options: Object = {}) {
    const instance = await this.getFile('dicy-instance.yaml-ParsedYAML')
    if (instance) instance.value = options
  }

  async analyzePhase (command: Command, phase: Phase) {
    this.checkForKill()

    for (const ruleClass: Class<Rule> of this.ruleClasses) {
      const jobNames = ruleClass.ignoreJobName ? [undefined] : this.options.jobNames
      for (const jobName of jobNames) {
        const rule = await ruleClass.analyzePhase(this.state, command, phase, jobName)
        if (rule) {
          await this.addRule(rule)
        }
      }
    }
  }

  async analyzeFiles (command: Command, phase: Phase) {
    this.checkForKill()

    while (true) {
      const file: ?File = Array.from(this.files).find(file => !file.analyzed)

      if (!file) break

      for (const ruleClass: Class<Rule> of this.ruleClasses) {
        const jobNames = file.jobNames.size === 0 ? [undefined] : Array.from(file.jobNames.values())
        for (const jobName of jobNames) {
          const rules: Array<Rule> = await ruleClass.analyzeFile(this.state, command, phase, jobName, file)
          for (const rule of rules) {
            await this.addRule(rule)
          }
        }
      }

      file.analyzed = true
    }
  }

  getAvailableRules (command: ?Command): Array<RuleInfo> {
    return this.ruleClasses
      .filter(rule => !command || rule.commands.has(command))
      .map(rule => ({ name: rule.name, description: rule.description }))
  }

  async evaluateRule (rule: Rule, action: Action): Promise<boolean> {
    this.checkForKill()

    if (rule.failures.has(action)) {
      this.info(`Skipping rule ${rule.id} because of previous failure.`)
      return false
    }

    await rule.evaluate(action)
    return true
  }

  async evaluate (command: Command, phase: Phase, action: Action): Promise<boolean> {
    this.checkForKill()

    const primaryCount = (ruleGroup: Array<Rule>) => ruleGroup.reduce(
      (total, rule) => total + rule.parameters.reduce((count, parameter) => parameter.filePath === this.filePath ? count + 1 : count, 0),
      0)
    const evaluationNeeded = rule => rule.actions.has(action) && rule.command === command && rule.phase === phase

    const ruleGroups: Array<Array<Rule>> = _.sortBy(this.components
      .map(component => {
        const ruleGroup = _.sortBy(component.filter(evaluationNeeded), [rule => rule.inputs.length])

        return ruleGroup.reduce((current, x) => {
          const weight = ruleGroup.reduce((count, y) => this.isChild(y, x) ? count + 1 : count, 0)

          if (weight < current.weight) return { weight, rules: [x] }
          if (weight === current.weight) current.rules.push(x)

          return current
        }, { weight: Number.MAX_SAFE_INTEGER, rules: [] }).rules
      })
      .filter(component => component.length > 0), [primaryCount])

    if (ruleGroups.length === 0) return false

    let didEvaluation: boolean = false

    for (const ruleGroup of ruleGroups) {
      for (const rule of ruleGroup) {
        await this.checkUpdates(command, phase)
        didEvaluation = await this.evaluateRule(rule, action) || didEvaluation
      }
    }

    await this.checkUpdates(command, phase)

    return didEvaluation
  }

  async checkUpdates (command: Command, phase: Phase) {
    this.checkForKill()

    for (const file of this.files) {
      for (const rule of this.state.getInputRules(file)) {
        await rule.addFileActions(file, command, phase)
      }
      file.hasBeenUpdated = false<commit_msg>Add some comments<commit_after>/* @flow */

import _ from 'lodash'
import path from 'path'
import readdir from 'readdir-enhanced'

import State from './State'
import StateConsumer from './StateConsumer'
import File from './File'
import Rule from './Rule'

import type { Action, Command, Option, Phase, RuleInfo } from './types'

export default class DiCy extends StateConsumer {
  static async create (filePath: string, options: Object = {}) {
    const schema = await DiCy.getOptionDefinitions()
    const state = await State.create(filePath, schema)
    const builder = new DiCy(state)

    await builder.initialize()
    await builder.setInstanceOptions(options)

    state.assignOptions(options)

    return builder
  }

  async initialize () {
    const ruleClassPath: string = path.join(__dirname, 'Rules')
    const entries: Array<string> = await readdir.async(ruleClassPath)
    this.state.ruleClasses = entries
      .map(entry => require(path.join(ruleClassPath, entry)).default)
  }

  async setInstanceOptions (options: Object = {}) {
    const instance = await this.getFile('dicy-instance.yaml-ParsedYAML')
    if (instance) instance.value = options
  }

  async analyzePhase (command: Command, phase: Phase) {
    this.checkForKill()

    for (const ruleClass: Class<Rule> of this.ruleClasses) {
      const jobNames = ruleClass.ignoreJobName ? [undefined] : this.options.jobNames
      for (const jobName of jobNames) {
        const rule = await ruleClass.analyzePhase(this.state, command, phase, jobName)
        if (rule) {
          await this.addRule(rule)
        }
      }
    }
  }

  async analyzeFiles (command: Command, phase: Phase) {
    this.checkForKill()

    while (true) {
      const file: ?File = Array.from(this.files).find(file => !file.analyzed)

      if (!file) break

      for (const ruleClass: Class<Rule> of this.ruleClasses) {
        const jobNames = file.jobNames.size === 0 ? [undefined] : Array.from(file.jobNames.values())
        for (const jobName of jobNames) {
          const rules: Array<Rule> = await ruleClass.analyzeFile(this.state, command, phase, jobName, file)
          for (const rule of rules) {
            await this.addRule(rule)
          }
        }
      }

      file.analyzed = true
    }
  }

  getAvailableRules (command: ?Command): Array<RuleInfo> {
    return this.ruleClasses
      .filter(rule => !command || rule.commands.has(command))
      .map(rule => ({ name: rule.name, description: rule.description }))
  }

  async evaluateRule (rule: Rule, action: Action): Promise<boolean> {
    this.checkForKill()

    if (rule.failures.has(action)) {
      this.info(`Skipping rule ${rule.id} because of previous failure.`)
      return false
    }

    await rule.evaluate(action)
    return true
  }

  async evaluate (command: Command, phase: Phase, action: Action): Promise<boolean> {
    this.checkForKill()

    const primaryCount = (ruleGroup: Array<Rule>) => ruleGroup.reduce(
      (total, rule) => total + rule.parameters.reduce((count, parameter) => parameter.filePath === this.filePath ? count + 1 : count, 0),
      0)
    const evaluationNeeded = rule => rule.actions.has(action) && rule.command === command && rule.phase === phase

    // First separate the rule graph into connected components. For each
    // component only retain rules that are pertinent to the current command,
    // phase and action. Rank the rules in the component by the number of other
    // rules that it is directly dependent on within the same component. Only
    // retain those that have the lowest dependency rank. Sort the remaining
    // rules by number of inputs in an ascending order. Finally sort the
    // components by number of primaries in an ascending order. A rule is
    // considered a primary is it has as an input the main source file for that
    // job. Note: we are using lodash's sortBy because the standard sort is
    // not guaranteed to be a stable sort.
    const rules: Array<Rule> = _.flatten(_.sortBy(this.components
      .map(component => {
        const ruleGroup = _.sortBy(component.filter(evaluationNeeded), [rule => rule.inputs.length])

        return ruleGroup.reduce((current, rule) => {
          // Rank the rule by how many other rules it is directly dependent on.
          const rank = ruleGroup.reduce(
            (count, otherRule) => this.isChild(otherRule, rule) ? count + 1 : count,
            0)

          // The rank is lower than the current rank so start a new list.
          if (rank < current.rank) return { rank, rules: [rule] }
          // The ranks is the same as the current rank so just add us to the
          if (rank === current.rank) current.rules.push(rule)
          // list.

          return current
        }, { rank: Number.MAX_SAFE_INTEGER, rules: [] }).rules
      }), [primaryCount]))

    if (rules.length === 0) return false

    let didEvaluation: boolean = false

    for (const rule of rules) {
      await this.checkUpdates(command, phase)
      didEvaluation = await this.evaluateRule(rule, action) || didEvaluation
    }

    await this.checkUpdates(command, phase)

    return didEvaluation
  }

  async checkUpdates (command: Command, phase: Phase) {
    this.checkForKill()

    for (const file of this.files) {
      for (const rule of this.state.getInputRules(file)) {
        await rule.addFileActions(file, command, phase)
      }
      file.hasBeenUpdated = false
{'fraction_non_alphanumeric': 0.11394694533762058, 'fraction_numerical': 0.0015072347266881028, 'mean_word_length': 2.9558823529411766, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4791316', 'n_tokens_mistral': 2890, 'n_tokens_neox': 2797, 'n_words': 866}