<commit_before>#!/usr/bin/env node

var cli = require('cli'),
fs = require('fs'),
Generator = require('./lib/generator.js'), // the file regeneration script

root = process.cwd(), // where serenity has been executed from.
convert = require('./lib/convert.js'),
config = require('./defaults.js'),
watchr = require('watchr'),
walkdir = require('walkdir');


cli.parse({
  server: ['s', 'Start up a server for the static files', 'boolean', true],
  port:  ['p', 'Listen on this port - overrides any config values', 'number', 4000],
  convert: ['c', 'Convert Jekyll (YAML with Liquid) site to Serenity (JSON with EJS)'],
  version: ['v','Shows the current Serenity version']
});


var walk = function(dir,include,ignore,done) {
  var files = [];
  var finder = walkdir(root);

  finder.on('file',function(file,stat) {
    if ((!include || include.test(file)) && (!ignore || !ignore.test(file))) {
      files.push(file);
    }
  });
  finder.on('end',function() {
    cli.info('Finished reading directory '+dir);
    done(null,files);
  });
};

var version = function() {
  cli.info('serenity '+require('./package.json').version);
};

cli.main(function (args,options) {

  if (options.version) {
    return version();
  }

  if (options.convert) {
    walk(root, new RegExp('.*\\.html$|.*_config\\.yml$'), new RegExp('.*/_site/.*'),function(err,list) {
      // list of all html files, not in _site
      if (err) cli.fatal('Error walking through files, please try again');
      if (list.length < 1) cli.fatal('Could not find any files to convert');
      convert.run(list);
    });
    return; // we dont want to boot up
  }

  if (options.server) {
    var http = require('http');
    var send = require('send');
    var url = require('url');

    var app = http.createServer(function(req, res){
      cli.debug(req.method + ' ' + req.url);
      function error(err) {
        cli.debug('error serving '+req.url + ' ' +err.status);
        res.statusCode = err.status || 500;
        res.end(err.message);
      }

      function redirect() {
        res.statusCode = 301;
        res.setHeader('Location', req.url + '/');
        res.end('Redirecting to ' + req.url + '/');
      }

      send(req, url.parse(req.url).pathname)
      .root(root+'/_site')
      .on('error', error)
      .on('directory', redirect)
      .pipe(res);
    }).listen(options.port);
    cli.ok('Server started on port '+options.port);
  }

  // sort out config
  var usrconfig = {};<commit_msg>fix annoying redirect loops on query strings<commit_after>#!/usr/bin/env node

var cli = require('cli'),
fs = require('fs'),
Generator = require('./lib/generator.js'), // the file regeneration script

root = process.cwd(), // where serenity has been executed from.
convert = require('./lib/convert.js'),
config = require('./defaults.js'),
watchr = require('watchr'),
walkdir = require('walkdir'),
url = require('url');


cli.parse({
  server: ['s', 'Start up a server for the static files', 'boolean', true],
  port:  ['p', 'Listen on this port - overrides any config values', 'number', 4000],
  convert: ['c', 'Convert Jekyll (YAML with Liquid) site to Serenity (JSON with EJS)'],
  version: ['v','Shows the current Serenity version']
});


var walk = function(dir,include,ignore,done) {
  var files = [];
  var finder = walkdir(root);

  finder.on('file',function(file,stat) {
    if ((!include || include.test(file)) && (!ignore || !ignore.test(file))) {
      files.push(file);
    }
  });
  finder.on('end',function() {
    cli.info('Finished reading directory '+dir);
    done(null,files);
  });
};

var version = function() {
  cli.info('serenity '+require('./package.json').version);
};

cli.main(function (args,options) {

  if (options.version) {
    return version();
  }

  if (options.convert) {
    walk(root, new RegExp('.*\\.html$|.*_config\\.yml$'), new RegExp('.*/_site/.*'),function(err,list) {
      // list of all html files, not in _site
      if (err) cli.fatal('Error walking through files, please try again');
      if (list.length < 1) cli.fatal('Could not find any files to convert');
      convert.run(list);
    });
    return; // we dont want to boot up
  }

  if (options.server) {
    var http = require('http');
    var send = require('send');
    var url = require('url');

    var app = http.createServer(function(req, res){
      cli.debug(req.method + ' ' + req.url);
      function error(err) {
        cli.debug('error serving '+req.url + ' ' +err.status);
        res.statusCode = err.status || 500;
        res.end(err.message);
      }

      function redirect() {
        var u = url.parse(req.url);
        var loc = u.pathname + '/' + (u.search || '');
        res.statusCode = 301;
        res.setHeader('Location', loc);
        res.end('Redirecting to ' + loc);
      }

      send(req, url.parse(req.url).pathname)
      .root(root+'/_site')
      .on('error', error)
      .on('directory', redirect)
      .pipe(res);
    }).listen(options.port);
    cli.ok('Server started on port '+options.port);
  }

  // sort out config
  var usrconfig = {};
{'fraction_non_alphanumeric': 0.18528018942383584, 'fraction_numerical': 0.004340962904498816, 'mean_word_length': 3.045490822027135, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5161999', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1610, 'n_words': 419}