<commit_before>#!/usr/bin/env node

var dists = require('../dockerfiles').dists
var request = require('request')
var url = "https://hub.docker.com/v2/repositories/nodesource/"
var url_tail = "/tags"
var async = require('async')
var columnify = require('columnify')
var colors = require('colors/safe')
var full_version = /^.*[0-9]*\.[0-9]\.[0-9]$/

var output = []

var count = 0
console.log(colors.green("ADD (missing on registry)")+"\n"+colors.red("REMOVE (missing locally)")+"\n")
for(dist in dists) {
  for(release in dists[dist]) {
    count++
    if( dist == 'centos' ||
      dist == 'fedora') patchedRelease = dist+release
    else patchedRelease = release
    ;(function scopeVars(dist,release) {
      getTags(patchedRelease, function doneGettingTags(e,tags) {
        if(e) return console.log(e)
        for(project in dists[dist][release]) {
          for(version in dists[dist][release][project]) {
            var tag = version
            if(project !== 'node') tag = project+'-'+tag
            var location = tags.indexOf(tag)
            if(location === -1) {
              output.push({
                Name: colors.green('master'),
                Dockerfile: colors.green('/'+dist+'/'+release+'/'+project+'/'+version),
                Tag: colors.green(tag)
              })
            } else {
              tags.splice(location,1)
            }
          } // end version
        } // end project
        for( tag in Object.keys(tags) ) {
          if(!tags[tag].match(full_version)) continue;
          output.push({
            Name: colors.red('master'),
            Dockerfile: colors.red('/'+dist+'/'+release+'/???'),
            Tag: colors.red(tags[tag])
          })
        }
        count--
        if(count===0) console.log(columnify(output))
      }) // end getTags
    })(dist,release)
  } // end release
} // end dist

function getTags(release,cb) {
  var tags = []
  var full_url = url + release + url_tail
  async.during(function getPartialTagList(cb) {
    return cb(null,full_url)
  },function doNothing(cb) {
    request(full_url,function(e,req,body) {
      if(e) return cb(e)
      try {
        body =JSON.parse(body)
        var new_tags = body.results.map(function getTagsFromResult(c) {
          return c.name
        })
        tags = tags.concat(new_tags)
        full_url = body.next
        return cb()
      } catch(e) {
        return cb(e)
      }
    })
  },function doneGettingTags(e) {
    return cb(e,tags)
  })
}<commit_msg>Support tags on images<commit_after>#!/usr/bin/env node

var dists = require('../dockerfiles').dists
var request = require('request')
var url = "https://hub.docker.com/v2/repositories/nodesource/"
var url_tail = "/build"
var async = require('async')
var columnify = require('columnify')
var colors = require('colors/safe')
var full_version = /^.*[0-9]*\.[0-9]\.[0-9]$/


console.log(colors.green("ADD (missing on registry)"))
console.log(colors.red("REMOVE (missing locally)"))
console.log(colors.yellow("MODIFY (incorrect entry)"))
console.log()

// count keeps track of how many images we are going to process so that we
// know when to print our report
var count = 0

// output holds all of the table data for our final report, so we can pass
// to columnify for pretty output
var output = []

// Alias jessie => node
dists["alias"] = {"node":dists["jessie"]}

for(dist in dists) {

  for(release in dists[dist]) {

    count++

    // Patched release fixes the release name to match the Docker hub tag
    var patchedRelease = ""
    if( dist == 'centos' || dist == 'fedora') {
      patchedRelease = dist+release
    } else if( release === "node") {
      patchedRelease = "jessie"
    } else {
      patchedRelease = release
    }

    // Scope the variables dist and release so they survive yielding
    // to the event loop.
    ;(function scopeVars(dist,release) {

      getTags(patchedRelease, function doneGettingTags(e,tags) {
        if(e) return console.log(e.stack)

        var patchedRelease = ""
        if( dist == 'centos' || dist == 'fedora') {
          patchedRelease = dist+release
        } else {
          patchedRelease = release
        }

        for(project in dists[dist][release]) {

          var versions = Object.keys(dists[dist][release][project])

          // No need to process projects with no tags, simplifies logic later
          if(versions.length === 0) continue;

          // Get a hash of all tags that should exist
          var localTags = generateTags(dist,release,project,versions)

          // Ensure local tags exist externally
          verifyTags(localTags,tags,output)
        }

        // Any tags which haven't been found locally are broken
        for( tag in Object.keys(tags) ) {
          output.push({
            Repo: colors.red(patchedRelease),
            Name: colors.red(tags[tag].branch),
            Dockerfile: colors.red(tags[tag].path),
            Tag: colors.red(tags[tag].tag)
          })
        }

        // Finished processing a release
        count--

        // If we are finished processing all releases, print report.
        if(count===0) {
          console.log(columnify(output))
        }

      }) // end getTags

    })(dist,release)

  } // end release

} // end dist

/* Compute all tags, including aliases */
function generateTags(dist,release,project,versions) {
  // Versions holds an array of arrays. Each index of the subarray
  // correpsonds to the major, minor, and patch respectively. This
  // simplifies the logic later.
  var versions = Object.keys(dists[dist][release][project])
    .sort()
    .map(function(v) {
      return v.split(".")
    })

  // Get the prefix for a tag if necessary
  var tagPrefix = (project === "node") ? "" : project + "-"

  var majors = versions.reduce(function(p,c) {
    if(c[0]==="0") return p
    p[tagPrefix+c[0]] = {
      dist: dist,
      release: release,
      project: project,
      version: c.join('.'),
      path: '/'+dist+'/'+release+'/'+project+'/'+c.join('.')
    }
    return p
  },[])

  var minors = versions.reduce(function(p,c) {
    p[tagPrefix+c[0]+"."+c[1]] = {
      dist: dist,
      release: release,
      project: project,
      version: c.join('.'),
      path: '/'+dist+'/'+release+'/'+project+'/'+c.join('.')
    }
    return p
  },[])

  var patches = versions.reduce(function(p,c) {
    p[tagPrefix+c.join('.')] = {
      dist: dist,
      release: release,
      project: project,
      version: c.join('.'),
      path: '/'+dist+'/'+release+'/'+project+'/'+c.join('.')
    }
    return p
  },[])

  var latest = []
  var latestVersion = versions.pop().join(".")

  if(project === "node") {
    latest["latest"] = {
      dist: dist,
      release: release,
      project: project,
      version: latestVersion,
      path: '/'+dist+'/'+release+'/'+project+'/'+latestVersion
    }
  } else {
    latest[project] = {
      dist: dist,
      release: release,
      project: project,
      version: latestVersion,
      path: '/'+dist+'/'+release+'/'+project+'/'+latestVersion
    }
  }

  versions = objectConcat(majors, minors, patches, latest)
  return versions
}

function verifyTags(versions, tags, output) {
  for( version in versions ) {
    var location = tags.map(function(v) {
      return v.tag
    }).indexOf(version)
    if(location === -1) {
      var release = (versions[version].dist === "centos" || versions[version].dist === "fedora") ?
        versions[version].dist + versions[version].release : versions[version].release
      output.push({
        Repo: colors.green(release),
        Name: colors.green('master'),
        Dockerfile: colors.green(versions[version].path),
        Tag: colors.green(version)
      })
    }
    else {
      verifyEntry(versions[version],tags[location],output)
      tags.splice(location,1)
    }
  }
}

function verifyEntry(local,remote,output) {
  var release = (local.dist === "centos" || local.dist === "fedora") ?
    local.dist + local.release : local.release
  var obj = {
    Repo: colors.yellow(release),
    Name: colors.yellow('master'),
    Dockerfile: colors.yellow(remote.path),
    Tag: colors.yellow(remote.tag)
  }

  var log = false

  if(remote.branch !== 'master') {
    obj.Name = color.yellow(remote.branch+"=>master")
    log = true
  }
  if(remote.path !== local.path) {
    obj.Dockerfile = colors.yellow(remote.path+" => "+local.path)
    log = true
  }

  if(log) {
    output.push(obj)
  }
}

function getTags(release,cb) {
  var tags = []
  var full_url = url + release + url_tail

  request(full_url,function(e,req,body) {
    if(e) return cb(e)
    try {
      body =JSON.parse(body)
      var new_tags = body.build_tags.map(function getTagsFromResult(c) {
        return {
          "tag": c.name,
          "branch" : c.source_name,
          "path": c.dockerfile_location
        }
      })
      tags = tags.concat(new_tags)
      full_url = body.next
      return cb(null,tags)
    } catch(e) {
      return cb(e)
    }
  })
}

function objectConcat() {
  var ret = {};
  var len = arguments.length;
  for (var i=0; i<len; i++) {
    for (p in arguments[i]) {
      if (arguments[i].hasOwnProperty(p)) {
        ret[p] = arguments[i][p];
      }
    }
  }
  return ret;
}
{'fraction_non_alphanumeric': 0.15654813529921943, 'fraction_numerical': 0.0031439722463139637, 'mean_word_length': 2.4615384615384617, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742817', 'n_tokens_mistral': 2943, 'n_tokens_neox': 2881, 'n_words': 675}