<commit_before>(function () {

  var _ = require("underscore"),
    async = require('async'),
    fs = require('fs'),
    marked = require("marked"),
    path = require('path'),
    contentFiles = fs.readdirSync(path.join(__dirname, "../content")),
    imageFiles = fs.readdirSync(path.join(__dirname, "../public/images")),
    stylesheets = fs.readdirSync(path.join(__dirname, "../public/stylesheets"));

  var loadContentFile = function (filename, done) {
    if (!_.contains([".markdown", ".md"], path.extname(filename))) {
      done(); // don't serve non-markdown content files
      return;
    }
    fs.readFile(path.join(__dirname, "../content", filename), "utf8", function (err, data) {
      if (err) {
        return done(err);
      }
      var dataLines = data.split("\n");
      var basename = path.basename(filename, path.extname(filename));
      var imageName = _.find(imageFiles, function (imageFile) {
        return path.basename(imageFile, path.extname(imageFile)) === basename;
      });
      var stylesheetName = _.find(stylesheets, function (stylesheet) {
        return path.basename(stylesheet, path.extname(stylesheet)) === basename;
      });
      var stylesheetPath = stylesheetName &&
        "/stylesheets/" + path.basename(stylesheetName, path.extname(stylesheetName)) + ".css";

      done(null, {
        markdown: marked(dataLines.splice(2).join("\n")),
        file: basename,
        path: basename === "index" ? "" : basename,
        image: imageName ? "../images/" + imageName : "http://placekitten.com/g/1024/768",
        stylesheet: stylesheetPath,
        title: dataLines[0].replace(/^[^a-zA-Z]+/, ""), // delete leading non-alpha characters
        subtitle: dataLines[1].replace(/^[^a-zA-Z]+/, "")
      });
    });
  };

  exports.processContent = function (callback) {
    async.map(contentFiles, loadContentFile, callback);
  };
}());
<commit_msg>make this more tolerant for weird titled paths and files by URL encoding<commit_after>(function () {

  var _ = require("underscore"),
    async = require('async'),
    fs = require('fs'),
    marked = require("marked"),
    path = require('path'),
    contentFiles = fs.readdirSync(path.join(__dirname, "../content")),
    imageFiles = fs.readdirSync(path.join(__dirname, "../public/images")),
    stylesheets = fs.readdirSync(path.join(__dirname, "../public/stylesheets"));

  var loadContentFile = function (filename, done) {
    if (!_.contains([".markdown", ".md"], path.extname(filename))) {
      done(); // don't serve non-markdown content files
      return;
    }
    fs.readFile(path.join(__dirname, "../content", filename), "utf8", function (err, data) {
      if (err) {
        return done(err);
      }
      filename = encodeURIComponent(filename);
      var dataLines = data.split("\n");
      var basename = path.basename(filename, path.extname(filename));
      var imageName = _.find(imageFiles, function (imageFile) {
        return path.basename(imageFile, path.extname(imageFile)) === basename;
      });
      var stylesheetName = _.find(stylesheets, function (stylesheet) {
        return path.basename(stylesheet, path.extname(stylesheet)) === basename;
      });
      var stylesheetPath = stylesheetName &&
        "/stylesheets/" + path.basename(stylesheetName, path.extname(stylesheetName)) + ".css";

      done(null, {
        markdown: marked(dataLines.splice(2).join("\n")),
        file: basename,
        path: basename === "index" ? "" : basename,
        image: imageName ? "../images/" + imageName : "http://placekitten.com/g/1024/768",
        stylesheet: stylesheetPath,
        title: dataLines[0].replace(/^[^a-zA-Z]+/, ""), // delete leading non-alpha characters
        subtitle: dataLines[1].replace(/^[^a-zA-Z]+/, "")
      });
    });
  };

  exports.processContent = function (callback) {
    async.map(contentFiles, loadContentFile, callback);
  };
}());

{'fraction_non_alphanumeric': 0.17418032786885246, 'fraction_numerical': 0.0056352459016393444, 'mean_word_length': 3.1454352441613587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741787', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1138, 'n_words': 232}