<commit_before>
async = require 'async'
fs = require 'fs'
path = require 'path'
url = require 'url'
colors = require 'colors'
minimatch = require 'minimatch'
{logger} = require './common'

contentPlugins = []
registerContentPlugin = (treeName, handles, plugin) ->
  ### register a plugin. arguments:
      *treeName* - name that will be shown in the content tree (eg. 'textFiles')
                   generally a plural name is recommended since it will appear in the content-tree
                   as an array of *plugin* instances (eg. contents.somedir.textFiles)
      *handles*: glob-pattern to match (eg. '** / *.*(txt|text)' )
      *plugin*: the <ContentPlugin> subclass ###
  contentPlugins.push
    treeName: treeName
    pattern: handles
    class: plugin

class Model
  @property = (name, method) ->
    ### define read-only property with *name* ###
    Object.defineProperty @prototype, name,
      get: -> method.call @
      enumerable: true

class ContentPlugin extends Model

  render: (locals, contents, templates, callback) ->
    ### *callback* with a ReadStream/Buffer or null if the contents should not be rendered
        *locals* rendering context variables
        *contents* is the full content tree
        *templates* is a map of all templates as: {filename: templateInstance} ###
    throw new Error 'not implemented'

  getFilename: ->
    ### return filename for this content ###
    throw new Error 'not implemented'

  getUrl: (base='/') ->
    ### return url for this content relative to *base* ###
    filename = @getFilename()
    if not base.match /\/$/
      base += '/'
    if process.platform is 'win32'
      filename = filename.replace /\\/g, '/' #'
    url.resolve base, filename

  # some shorthands
  @property 'url', -> @getUrl()
  @property 'filename', -> @getFilename()

ContentPlugin.fromFile = (filename, base, callback) ->
  ### *callback* with an instance of class. *filename* is the relative filename
      from *base* wich is the working directory (content directory) ###

class StaticFile extends ContentPlugin
  ### static file handler, simply serves content as-is. last in chain ###

  constructor: (@_filename, @_base) ->

  getFilename: ->
    @_filename

  render: (args..., callback) ->
    # locals, contents etc not used in this plugin
    try
      rs = fs.createReadStream path.join(@_base, @_filename)
    catch error
      return callback error
    callback null, rs

StaticFile.fromFile = (filename, base, callback) ->
  callback null, new StaticFile(filename, base)

registerContentPlugin 'files', '**/*', StaticFile

slugify = (s) ->
  s = s.replace(/[^\w\s-]/g, '').trim().toLowerCase()
  s = s.replace /[-\s]+/g, '-'
  return s

# Class ContentTree
# not using Class since we need a clean prototype
ContentTree = (filename) ->
  groups = {directories: []}
  for plugin in contentPlugins
    groups[plugin.treeName] = []
  Object.defineProperty @, '_',
    get: -> groups
  Object.defineProperty @, 'filename',
    get: -> filename
  Object.defineProperty @, 'index',
    get: -> @['index.md'] or @['index.markdown']

ContentTree.fromDirectory = (directory, base, callback) ->
  if !callback?
    callback = base
    base = directory

  # create the base tree from *directory*
  tree = new ContentTree path.relative(base, directory)

  async.waterfall [
    # read directory
    async.apply fs.readdir, directory
    (filenames, callback) ->
      async.forEach filenames, (filename, callback) ->
        filename = path.join directory, filename
        async.waterfall [
          async.apply fs.lstat, filename
          (stats, callback) ->
            if stats.isDirectory()
              # recursively map directories to content tree instances
              ContentTree.fromDirectory filename, base, (error, result) -><commit_msg>Don't use hardcoded index files<commit_after>
async = require 'async'
fs = require 'fs'
path = require 'path'
url = require 'url'
colors = require 'colors'
minimatch = require 'minimatch'
{logger} = require './common'

contentPlugins = []
registerContentPlugin = (treeName, handles, plugin) ->
  ### register a plugin. arguments:
      *treeName* - name that will be shown in the content tree (eg. 'textFiles')
                   generally a plural name is recommended since it will appear in the content-tree
                   as an array of *plugin* instances (eg. contents.somedir.textFiles)
      *handles*: glob-pattern to match (eg. '** / *.*(txt|text)' )
      *plugin*: the <ContentPlugin> subclass ###
  contentPlugins.push
    treeName: treeName
    pattern: handles
    class: plugin

class Model
  @property = (name, method) ->
    ### define read-only property with *name* ###
    Object.defineProperty @prototype, name,
      get: -> method.call @
      enumerable: true

class ContentPlugin extends Model

  render: (locals, contents, templates, callback) ->
    ### *callback* with a ReadStream/Buffer or null if the contents should not be rendered
        *locals* rendering context variables
        *contents* is the full content tree
        *templates* is a map of all templates as: {filename: templateInstance} ###
    throw new Error 'not implemented'

  getFilename: ->
    ### return filename for this content ###
    throw new Error 'not implemented'

  getUrl: (base='/') ->
    ### return url for this content relative to *base* ###
    filename = @getFilename()
    if not base.match /\/$/
      base += '/'
    if process.platform is 'win32'
      filename = filename.replace /\\/g, '/' #'
    url.resolve base, filename

  # some shorthands
  @property 'url', -> @getUrl()
  @property 'filename', -> @getFilename()

ContentPlugin.fromFile = (filename, base, callback) ->
  ### *callback* with an instance of class. *filename* is the relative filename
      from *base* wich is the working directory (content directory) ###

class StaticFile extends ContentPlugin
  ### static file handler, simply serves content as-is. last in chain ###

  constructor: (@_filename, @_base) ->

  getFilename: ->
    @_filename

  render: (args..., callback) ->
    # locals, contents etc not used in this plugin
    try
      rs = fs.createReadStream path.join(@_base, @_filename)
    catch error
      return callback error
    callback null, rs

StaticFile.fromFile = (filename, base, callback) ->
  callback null, new StaticFile(filename, base)

registerContentPlugin 'files', '**/*', StaticFile

slugify = (s) ->
  s = s.replace(/[^\w\s-]/g, '').trim().toLowerCase()
  s = s.replace /[-\s]+/g, '-'
  return s

# Class ContentTree
# not using Class since we need a clean prototype
ContentTree = (filename) ->
  groups = {directories: []}
  for plugin in contentPlugins
    groups[plugin.treeName] = []
  Object.defineProperty @, '_',
    get: -> groups
  Object.defineProperty @, 'filename',
    get: -> filename
  Object.defineProperty @, 'index',
    get: ->
      for key, item of this
        if key[0...5] is 'index' then return item

ContentTree.fromDirectory = (directory, base, callback) ->
  if !callback?
    callback = base
    base = directory

  # create the base tree from *directory*
  tree = new ContentTree path.relative(base, directory)

  async.waterfall [
    # read directory
    async.apply fs.readdir, directory
    (filenames, callback) ->
      async.forEach filenames, (filename, callback) ->
        filename = path.join directory, filename
        async.waterfall [
          async.apply fs.lstat, filename
          (stats, callback) ->
            if stats.isDirectory()
              # recursively map directories to content tree instances
              ContentTree.fromDirectory filename, base, (error, result) ->
{'fraction_non_alphanumeric': 0.1219194158299648, 'fraction_numerical': 0.0007823705828660843, 'mean_word_length': 2.988559542381695, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3261599', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2137, 'n_words': 751}