<commit_before>var bampoPageRegex = /(<pb id[^<>]+?>(?=([\s\S](?!<pb))*?(?=<bampo n="\d+[a-z]?\.\d+(\.1)?"\/>)))/g;
var bampoRegex = /<bampo n="(\d+[a-z]?)\.(\d+)(\.1)?"\/>/;
var allBampoRegex = /<bampo n="\d+[a-z]?\.\d+(\.1)?"\/>/g;
var sutraPageRegex = /(<pb id[^<>]+?>(?=([\s\S](?!<pb))*?(?=<sutra id="[^<>]+?"\/>)))/g;
var sutraRegex = /<sutra id="[^<>]*[^\d](\d+)[^<>\d]*"\/>/;
var allSutraRegex = /<sutra id="[^<>]*[^\d]\d+[^<>\d]*"\/>/g;
var pbRegex = /<pb id="[^<>]+?"\/>/g;
var delim = '~!@#$%';

var tools = require('./tools.js');
var has = tools.has;

function textsBy(pageRegex, originalText) {
  var texts = originalText.replace(pageRegex, delim + '$1')
    .split(delim);
  texts.splice(0, 2, texts[0].concat(texts[1]));

  return texts;
}

// split with bampo tag

function objsByBampo(bampoText) {
  var bampoObj = {};
  var lastBampoTag = bampoText.match(allBampoRegex)
    .pop()
    .match(bampoRegex);

  bampoObj['bampoN'] = lastBampoTag[1] + '_' + lastBampoTag[2];
  bampoObj['bampoText'] = bampoText;

  return bampoObj;
}

function splitByBampoTag(text) {
  var bampoTexts = textsBy(bampoPageRegex, text);
  var bampoObjs = bampoTexts.map(objsByBampo);
  var lastBampoN = bampoObjs[bampoObjs.length - 1].bampoN;

  return {'bampoObjs': bampoObjs, 'lastBampoN': lastBampoN};
}

// split with sutra tag

function objsBySutra(sutraText) {
  var bampoObjs = [];
  var lastSutraTag = sutraText.match(allSutraRegex)
    .pop()
    .match(sutraRegex);
  var sutraId = lastSutraTag[1];
  var pages = textsBy(pbRegex, sutraText);
  var bampoCount = 1;

  do {
    var bampoObj = {};
    bampoObj['bampoText'] = pages.splice(0, 40)
      .join('');
    bampoObj['bampoN'] = sutraId + '_' + bampoCount;
    bampoObjs.push(bampoObj);
    bampoCount ++;
  }
  while (pages.length > 0);

  return bampoObjs;
}

function splitBySutraTag(text) {
  var bampoTexts = textsBy(sutraPageRegex, text);
  var bampoObjsInSutras = bampoTexts.map(objsBySutra);
}

//

function splitWithBampo(volObjs) {
  var lastBampoN;

  var bamposInVols = volObjs.map(function(volObj) {
    var volText = volObj.volText;

    if (has(bampoRegex, volText)) {
      var objsAndBampoN = splitByBampoTag(volText);
      lastBampoN = objsAndBampoN.lastBampoN;

      return objsAndBampoN.bampoObjs;
    }
    else if (has(sutraRegex, volText)) {
      var objsAndBampoN = splitBySutraTag(volText);
      //lastBampoN = objsAndBampoN.lastBampoN;

      //return objsAndBampoN.bampoObjs;
    }
    else {

    }
  });

  return bamposInVols;
}

module.exports = splitWithBampo;<commit_msg>complete split bampos by sutra tag in splitWithBampo.js<commit_after>var bampoPageRegex = /(<pb id[^<>]+?>(?=([\s\S](?!<pb))*?(?=<bampo n="\d+[a-z]?\.\d+(\.1)?"\/>)))/g;
var bampoRegex = /<bampo n="(\d+[a-z]?)\.(\d+)(\.1)?"\/>/;
var allBampoRegex = /<bampo n="\d+[a-z]?\.\d+(\.1)?"\/>/g;
var sutraPageRegex = /(<pb id[^<>]+?>(?=([\s\S](?!<pb))*?(?=<sutra id="[^<>]+?"\/>)))/g;
var sutraRegex = /<sutra id="[^<>]*[^\d](\d+)[^<>\d]*"\/>/;
var allSutraRegex = /<sutra id="[^<>]*[^\d]\d+[^<>\d]*"\/>/g;
var pbRegex = /<pb id="[^<>]+?"\/>/g;
var delim = '~!@#$%';

var tools = require('./tools.js');
var has = tools.has;

function textsBy(pageRegex, originalText) {
  var texts = originalText.replace(pageRegex, delim + '$1')
    .split(delim);
  texts.splice(0, 2, texts[0].concat(texts[1]));

  return texts;
}

// split with bampo tag

function objsByBampo(bampoText) {
  var bampoObj = {};
  var lastBampoTag = bampoText.match(allBampoRegex)
    .pop()
    .match(bampoRegex);

  bampoObj['bampoN'] = lastBampoTag[1] + '_' + lastBampoTag[2];
  bampoObj['bampoText'] = bampoText;

  return bampoObj;
}

function splitByBampoTag(text) {
  var bampoTexts = textsBy(bampoPageRegex, text);
  var bampoObjs = bampoTexts.map(objsByBampo);
  var lastBampoN = bampoObjs[bampoObjs.length - 1].bampoN;

  return {'bampoObjs': bampoObjs, 'lastBampoN': lastBampoN};
}

// split with sutra tag

function objsBySutra(sutraText) {
  var bampoObjs = [];
  var lastSutraTag = sutraText.match(allSutraRegex)
    .pop()
    .match(sutraRegex);
  var sutraId = lastSutraTag[1];
  var pages = textsBy(pbRegex, sutraText);
  var bampoCount = 1;

  do {
    var bampoObj = {};
    bampoObj['bampoText'] = pages.splice(0, 40)
      .join('');
    bampoObj['bampoN'] = sutraId + '_' + bampoCount;
    bampoObjs.push(bampoObj);
    bampoCount ++;
  }
  while (pages.length > 0);

  return bampoObjs;
}

function splitBySutraTag(text) {
  var bampoTexts = textsBy(sutraPageRegex, text);
  var bampoObjsInSutras = bampoTexts.map(objsBySutra);
  var bampoObjs = bampoObjsInSutras.reduce(function(arr1, arr2) {
    return arr1.concat(arr2);
  });
  var lastBampoN = bampoObjs[bampoObjs.length - 1].bampoN;

  return {'bampoObjs': bampoObjs, 'lastBampoN': lastBampoN};
}

//

function splitWithBampo(volObjs) {
  var lastBampoN;

  var bamposInVols = volObjs.map(function(volObj) {
    var volText = volObj.volText;

    if (has(bampoRegex, volText)) {
      var objsAndBampoN = splitByBampoTag(volText);
      lastBampoN = objsAndBampoN.lastBampoN;

      return objsAndBampoN.bampoObjs;
    }
    else if (has(sutraRegex, volText)) {
      var objsAndBampoN = splitBySutraTag(volText);
      lastBampoN = objsAndBampoN.lastBampoN;

      return objsAndBampoN.bampoObjs;
    }
    else {

    }
  });

  return bamposInVols;
}

module.exports = splitWithBampo;
{'fraction_non_alphanumeric': 0.19981464318813716, 'fraction_numerical': 0.007228915662650603, 'mean_word_length': 3.8568856885688567, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6680234', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2436, 'n_words': 371}