<commit_before>var request = require('request');

var MEDIUM_IMG_CDN = "https://medium2.global.ssl.fastly.net/max/";

var utils = {
  loadMediumPost: function(mediumURL, cb) {
    if(mediumURL.match(/^http/i)) {
      request(mediumURL+"?format=json", function(err, res, body) {
        if(err) return cb(err);
        var json_string = body.substr(body.indexOf('{'));
        var json = JSON.parse(json_string);
        return cb(null, json);
      });
    }
    else {
      json = require(process.cwd() + "/" + mediumURL);
      return cb(null, json);
    }
  },
  processSection: function(s) {
    var section = "";
    if(s.backgroundImage) {
      var imgwidth = parseInt(s.backgroundImage.originalWidth,10);
      var imgsrc = MEDIUM_IMG_CDN+Math.max(imgwidth*2,2000)+"/"+s.backgroundImage.id;
      section = "\n![]("+imgsrc+")";
    }
    return section;
  },
  processParagraph: function(p) {

    var markups_array = utils.createMarkupsArray(p.markups);

    if(markups_array.length > 0) {
      var previousIndex=0, text=p.text, tokens=[];
      for(var j=0;j<markups_array.length;j++) {
        if(markups_array[j]) {
          token = text.substring(previousIndex, j);
          previousIndex = j;
          tokens.push(token);
          tokens.push(markups_array[j]);
        }
      }
      tokens.push(text.substring(j-1));
      p.text = tokens.join('');
    }

    var markup = "";
    switch(p.type) {
      case 1:
        markup = "\n";
        break;
      case 2:
        p.text = "\n#"+p.text.replace(/\n/g,'\n#');
        break;
      case 3:
        p.text = "\n##"+p.text.replace(/\n/g,'\n##');
        break;
      case 4: // image & caption
        var imgwidth = parseInt(p.metadata.originalWidth,10);
        var imgsrc = MEDIUM_IMG_CDN+Math.max(imgwidth*2,2000)+"/"+p.metadata.id;
        p.text = "\n!["+p.text+"]("+imgsrc+")*"+p.text+"*";
        break;
      case 6:
        markup = "> ";
        break;
      case 7: // quote
        p.text = "> #"+p.text.replace(/\n/g,'\n> #');
        break;
      case 8:
        p.text = "\n    "+p.text.replace(/\n/g,'\n    ');
        break;
      case 9:
        markup = "\n* ";
        break;
      case 10:
        markup = "\n1. ";
        break;
      case 11:
        p.text = '\n<iframe src="https://medium.com/media/'+p.iframe.mediaResourceId+'" frameborder=0></iframe>';
        break;
      case 13:
        markup = "\n###";
        break;
      case 15: // caption for section image
        p.text = "*"+p.text+"*";<commit_msg>adding a space after #*<commit_after>var request = require('request');

var MEDIUM_IMG_CDN = "https://medium2.global.ssl.fastly.net/max/";

var utils = {
  loadMediumPost: function(mediumURL, cb) {
    if(mediumURL.match(/^http/i)) {
      request(mediumURL+"?format=json", function(err, res, body) {
        if(err) return cb(err);
        var json_string = body.substr(body.indexOf('{'));
        var json = JSON.parse(json_string);
        return cb(null, json);
      });
    }
    else {
      json = require(process.cwd() + "/" + mediumURL);
      return cb(null, json);
    }
  },
  processSection: function(s) {
    var section = "";
    if(s.backgroundImage) {
      var imgwidth = parseInt(s.backgroundImage.originalWidth,10);
      var imgsrc = MEDIUM_IMG_CDN+Math.max(imgwidth*2,2000)+"/"+s.backgroundImage.id;
      section = "\n![]("+imgsrc+")";
    }
    return section;
  },
  processParagraph: function(p) {

    var markups_array = utils.createMarkupsArray(p.markups);

    if(markups_array.length > 0) {
      var previousIndex=0, text=p.text, tokens=[];
      for(var j=0;j<markups_array.length;j++) {
        if(markups_array[j]) {
          token = text.substring(previousIndex, j);
          previousIndex = j;
          tokens.push(token);
          tokens.push(markups_array[j]);
        }
      }
      tokens.push(text.substring(j-1));
      p.text = tokens.join('');
    }

    var markup = "";
    switch(p.type) {
      case 1:
        markup = "\n";
        break;
      case 2:
        p.text = "\n# "+p.text.replace(/\n/g,'\n# ');
        break;
      case 3:
        p.text = "\n## "+p.text.replace(/\n/g,'\n## ');
        break;
      case 4: // image & caption
        var imgwidth = parseInt(p.metadata.originalWidth,10);
        var imgsrc = MEDIUM_IMG_CDN+Math.max(imgwidth*2,2000)+"/"+p.metadata.id;
        p.text = "\n!["+p.text+"]("+imgsrc+")*"+p.text+"*";
        break;
      case 6:
        markup = "> ";
        break;
      case 7: // quote
        p.text = "> # "+p.text.replace(/\n/g,'\n> # ');
        break;
      case 8:
        p.text = "\n    "+p.text.replace(/\n/g,'\n    ');
        break;
      case 9:
        markup = "\n* ";
        break;
      case 10:
        markup = "\n1. ";
        break;
      case 11:
        p.text = '\n<iframe src="https://medium.com/media/'+p.iframe.mediaResourceId+'" frameborder=0></iframe>';
        break;
      case 13:
        markup = "\n### ";
        break;
      case 15: // caption for section image
        p.text = "*"+p.text+"*";
{'fraction_non_alphanumeric': 0.18404907975460122, 'fraction_numerical': 0.01464476548584999, 'mean_word_length': 2.044578313253012, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6729039', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1774, 'n_words': 307}