<commit_before>#!/usr/bin/env node
var fs = require('fs')
var child_process = require('child_process')
var exec = require('sync-exec')
var path = require('path');

var divePath = process.cwd();
var attachmentsExportPath = "/public/assets/images/"
var markdownImageReference = "assets/images/"
// print process.argv
process.argv.forEach(function (val, index, array) {
  if (index === 2){
    divePath = process.cwd() + "/" + val;
  }else if (index === 3){
    attachmentsExportPath = val
  }else if(index === 4){
    markdownImageReference = val
  }
});
console.log("TEST")
dive(divePath)

function dive(dir) {
  var list = []
  var stat = ""
    // Read the directory
  list = fs.readdirSync(dir);
  list.forEach(function (file) {
    // Full path of that file
    var path = dir + "/" + file

    // Get the file's stats
    stat = fs.statSync(path)

    // If the file is a directory
    if (stat && stat.isDirectory()) {
      dive(path);
    } else {
      console.log(file)
      if (file.endsWith('.html')) {
        var titleRegex = /<title>(?:Blue Planet : |)([a-z|\s|\-|\(|\)]+)+<\/title>/i
        var content = fs.readFileSync(path, 'utf8')
        var match = content.match(titleRegex)
        if (match != null && match.length > 1) {
          //          console.log(match[1])
          mkdirpSync("/Markdown")
//          console.log("Making Markdown")
          var outputFile = "Markdown/" + match[1].replace(/ /g, "-").replace(/[(|)]/g, "") + ".md"
          var out = exec("pandoc -f html -t markdown_github -o " + outputFile + " " + JSON.stringify(path.replace(/ /g, "\ ")), {cwd: process.cwd()})
          console.log(out)<commit_msg>Fix title check to be more generic. Soves #1<commit_after>#!/usr/bin/env node
var fs = require('fs')
var child_process = require('child_process')
var exec = require('sync-exec')
var path = require('path');

var divePath = process.cwd();
var attachmentsExportPath = "/public/assets/images/"
var markdownImageReference = "assets/images/"
// print process.argv
process.argv.forEach(function (val, index, array) {
  if (index === 2){
    divePath = process.cwd() + "/" + val;
  }else if (index === 3){
    attachmentsExportPath = val
  }else if(index === 4){
    markdownImageReference = val
  }
});
dive(divePath)

function dive(dir) {
  var list = []
  var stat = ""
    // Read the directory
  list = fs.readdirSync(dir);
  list.forEach(function (file) {
    // Full path of that file
    var path = dir + "/" + file

    // Get the file's stats
    stat = fs.statSync(path)

    // If the file is a directory
    if (stat && stat.isDirectory()) {
      dive(path);
    } else {
      console.log(file)
      if (file.endsWith('.html')) {
        var titleRegex = /<title>([^<]*)<\/title>/i
        var content = fs.readFileSync(path, 'utf8')
        var match = content.match(titleRegex)
        if (match != null && match.length > 1) {
          //          console.log(match[1])
          mkdirpSync("/Markdown")
//          console.log("Making Markdown")
          var outputFile = "Markdown/" + match[1].replace(/ /g, "-").replace(/[(|)]/g, "") + ".md"
          var out = exec("pandoc -f html -t markdown_github -o " + outputFile + " " + JSON.stringify(path.replace(/ /g, "\ ")), {cwd: process.cwd()})
          console.log(out)
{'fraction_non_alphanumeric': 0.17883211678832117, 'fraction_numerical': 0.004562043795620438, 'mean_word_length': 2.6503884572697003, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3730509', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1065, 'n_words': 268}