<commit_before>/**
 * @fileoverview Bed data processing script.
 */

var fs = require('fs');

/** @type {bed} */
module.exports = bed;

/**
 * @constructor
 */
function bed() {}

/**
 * @typedef {{
 *   chrStart: number,
 *   chrEnd: number,
 *   label: string
 * }}
 */
bed.Motif;

/** @const */
bed.query = {};

/**
 * @typedef {{
 *   bedName: string,
 *   chr: string,
 *   xl: (number|undefined),
 *   xr: (number|undefined)
 * }}
 */
bed.query.Motifs;

// Start public APIs
/**
 * @param {!bed.query.Motifs} query
 * @param {string} bedPath
 * @return {!Array<!bed.Motif>}
 */
bed.query.motifs = function(query, bedPath) {
  var bedName = query.bedName;
  var chr = query.chr;
  var dir = bedPath + bedName + '_chr/' + bedName + '_chr' + chr;
  return bed.readBed_(dir, query.xl, query.xr);
};

/**
 * @param {string} bedPath
 * @return {!Array<{
 *   bedName: string,
 *   description: string
 * }>}
 */
bed.query.list = function(bedPath) {
  return bed.listBed_(bedPath);
};
// End public APIs

/**
 * Reads a separated bed file for one chromosome.
 * @param {string} bedFile Path to the bed file.
 * @param {number|undefined} xl Left coordinate of the query range.
 * @param {number|undefined} xr Right coordinate of the query range.
 * @return {!Array<!bed.Motif>} Contains the intervals of bed data.
 * @private
 */
bed.readBed_ = function(bedFile, xl, xr) {
  var data = [];
  var lines = fs.readFileSync(bedFile).toString().split('\n');
  lines.forEach(function(line) {
    var parts = line.split('\t');
    var xLeft = parseInt(parts[0]);
    var xRight = parseInt(parts[1]);
    if (xl > xRight || xr < xLeft) {
      return;
    }
    data.push({
      chrStart: xLeft,
      chrEnd: xRight,
      label: parts[2]
    });
  });
  return data;
};

/**
 * @param {string} bedPath
 * @return {!Array<{
 *   bedName: string,<commit_msg>return # aggregated motifs under constant threshold<commit_after>/**
 * @fileoverview Bed data processing script.
 */

var fs = require('fs');

/** @type {bed} */
module.exports = bed;

/**
 * @constructor
 */
function bed() {}

/**
 * @typedef {{
 *   chrStart: number,
 *   chrEnd: number,
 *   label: string
 * }}
 */
bed.Motif;

/** @const */
bed.query = {};

/**
 * @typedef {{
 *   bedName: string,
 *   chr: string,
 *   xl: (number|undefined),
 *   xr: (number|undefined)
 * }}
 */
bed.query.Motifs;

// Start public APIs
/**
 * @param {!bed.query.Motifs} query
 * @param {string} bedPath
 * @return {!Array<!bed.Motif>}
 */
bed.query.motifs = function(query, bedPath) {
  var bedName = query.bedName;
  var chr = query.chr;
  var dir = bedPath + bedName + '_chr/' + bedName + '_chr' + chr;
  return bed.readBed_(dir, query.xl, query.xr);
};

/**
 * @param {string} bedPath
 * @return {!Array<{
 *   bedName: string,
 *   description: string
 * }>}
 */
bed.query.list = function(bedPath) {
  return bed.listBed_(bedPath);
};
// End public APIs


/**
 * Maximum number of motifs to return. If the number of motifs in the query
 * range exceeds this limit, return aggregated ranges instead.
 * @private @const {number}
 */
bed.MOTIF_THRESHOLD_ = 200;

/**
 * Reads a separated bed file for one chromosome.
 * @param {string} bedFile Path to the bed file.
 * @param {number|undefined} xl Left coordinate of the query range.
 * @param {number|undefined} xr Right coordinate of the query range.
 * @return {!Array<!bed.Motif>} Contains the intervals of bed data.
 * @private
 */
bed.readBed_ = function(bedFile, xl, xr) {
  var data = [];
  var lines = fs.readFileSync(bedFile).toString().split('\n');
  var numLines = lines.length;
  var xMin = Infinity, xMax = -Infinity;
  for (var i = 0; i < numLines; i++) {
    var line = lines[i];
    var parts = line.split('\t');
    var xLeft = parseInt(parts[0]);
    var xRight = parseInt(parts[1]);
    if (xLeft > xr) {
      break;
    }
    if (xl > xRight || xr < xLeft) {
      continue;
    }
    data.push({
      chrStart: xLeft,
      chrEnd: xRight,
      label: parts[2]
    });
    xMin = Math.min(xMin, xLeft);
    xMax = Math.max(xMax, xRight);
  }

  /**
   * Computes the aggregated motifs, assuming each interval [l, r] is extended
   * to [l, r + extend].
   * @param {number} extend
   * @param {boolean} returnArray Whether to return a result array. If false,
   *     return only the count of the array elements.
   * @return {number|!Array<bed.Motif>}
   */
  var aggregatedMotifs = function(extend, returnArray) {
    var result = returnArray ? [] : 0;
    var start, end = -Infinity;
    data.forEach(function(motif) {
      if (motif.chrStart > end) {
        if (end != -Infinity) {
          if (returnArray) {
            result.push({
              chrStart: start,
              chrEnd: end
            });
          } else {
            result++;
          }
        }
        start = motif.chrStart;
        end = motif.chrEnd + extend;
      } else {
        end = Math.max(end, motif.chrEnd + extend);
      }
    });
    return result;
  };

  if (data.length > bed.MOTIF_THRESHOLD_) {
    data.push({
      chrStart: Infinity
    });
    var minExtend = 0, maxExtend = xMax - xMin;
    while (minExtend <= maxExtend) {
      var extend = (minExtend + maxExtend) >> 1;
      var result = aggregatedMotifs(extend, false);
      if (result > bed.MOTIF_THRESHOLD_) {
        minExtend = extend + 1;
      } else {
        maxExtend = extend - 1;
      }
    }
    var aggregatedData = aggregatedMotifs(minExtend, true);
    console.log(aggregatedData.length, 'aggregated motifs with extend',
      maxExtend);
    return {
      aggregated: true,
      motifs: aggregatedData
    };
  }
  console.log(data.length, 'motifs');
  return {
    aggregated: false,
    motifs: data
  };
};

/**
 * @param {string} bedPath
 * @return {!Array<{
 *   bedName: string,
{'fraction_non_alphanumeric': 0.15794021081734924, 'fraction_numerical': 0.002592016588906169, 'mean_word_length': 2.65865992414665, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3570190', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1974, 'n_words': 527}