<commit_before>import { discover } from 'opensearch-browser';
import { DOMParser } from 'xmldom';
// const BluebirdPromise = require('bluebird');

self.DOMParser = DOMParser;

self.services = {};
self.promises = {};

function prepareBox(bbox) {
  bbox[1] = Math.max(bbox[1], -90);
  bbox[3] = Math.min(bbox[3], 90);

  for (let i = 0; i <= 2; i += 2) {
    while (bbox[i] > 180) {
      bbox[i] -= 360;
    }
    while (bbox[i] < -180) {
      bbox[i] += 360;
    }
  }
  return bbox;
}

function prepareRecords(records) {
  return records.map(record => {
    if (record.geometry && record.geometry.type === 'Polygon') {
      for (let ringIndex = 0; ringIndex < record.geometry.coordinates.length; ++ringIndex) {
        const ring = record.geometry.coordinates[ringIndex];
        let last = null;
        for (let i = 0; i < ring.length; ++i) {
          const current = ring[i];
          if (last) {
            if (current[0] - last[0] < -180) {
              current[0] += 360;
            }
            if (current[0] - last[0] > 180) {
              current[0] -= 360;
            }
          }
          last = current;
        }
      }
    }
    return record;
  });
}

/**
 * Convert a filters model, map model, and options to an OpenSearch parameters
 * object
 */
function convertFilters(filterParams, mapParams, options, format, service) {
  const description = service.getDescription();
  const url = description.getUrl(null, format || null);

  const parameters = {};

  const time = filterParams.time || mapParams.time;
  if (time) {
    if (Array.isArray(time)) {
      parameters['time:start'] = time[0];
      parameters['time:end'] = time[1];
    } else {
      parameters['time:start'] = time;
      parameters['time:end'] = time;
    }
  }

  const area = filterParams.area;
  if (area) {
    if (Array.isArray(area)) {
      parameters['geo:box'] = prepareBox(area);
    } else if (area.geometry) {
      const geometry = area.geometry;
      if (geometry.type === 'Point' && url.hasParameter('geo:lon') && url.hasParameter('geo:lat')) {
        parameters['geo:lon'] = geometry.coordinates[0];
        parameters['geo:lat'] = geometry.coordinates[1];
      } else {
        parameters['geo:geometry'] = geometry;
      }
    }
  } else if (mapParams) {
    // use the maps BBox by default
    parameters['geo:box'] = prepareBox(mapParams.bbox);
  }

  if (options.hasOwnProperty('itemsPerPage') && url.hasParameter('count')) {
    parameters.count = options.itemsPerPage;
  }

  if (options.hasOwnProperty('page')) {
    if (url.hasParameter('startIndex') && options.hasOwnProperty('itemsPerPage')) {
      parameters.startIndex = options.page * options.itemsPerPage + url.indexOffset;
    } else if (url.hasParameter('startPage')) {
      parameters.startPage = options.page + url.pageOffset;
    }
  }

  Object.keys(filterParams).forEach((key) => {
    if (url.hasParameter(key)) {
      parameters[key] = filterParams[key];
    }
  });

  return parameters;
}


function getService(url) {
  if (!self.services[url]) {
    // add a new promise
    // self.services[url] = discover(url, { useXHR: true, PromiseClass: BluebirdPromise });
    self.services[url] = discover(url, { useXHR: true, PromiseClass: BluebirdPromise });
  }
  return self.services[url];
}


function searchAll(url, method, filterParams, mapParams, options, format) {
  const maxCount = options.maxCount;

  return getService(url)
    .then(service => {
      const parameters = convertFilters(filterParams, mapParams, options, format, service);
      const paginator = service.getPaginator(parameters, format, method, true);
      if (maxCount) {
        return paginator.fetchFirstRecords(maxCount)
          .then(result => ({
            totalResults: result.totalResults,
            itemsPerPage: result.itemsPerPage,
            startIndex: result.startIndex,
            records: prepareRecords(result.records),
          }));
      }
      return paginator.fetchAllRecords()<commit_msg>Fixing issue with currently not used Bluebird Promise<commit_after>import { discover } from 'opensearch-browser';
import { DOMParser } from 'xmldom';
// const BluebirdPromise = require('bluebird');

self.DOMParser = DOMParser;

self.services = {};
self.promises = {};

function prepareBox(bbox) {
  bbox[1] = Math.max(bbox[1], -90);
  bbox[3] = Math.min(bbox[3], 90);

  for (let i = 0; i <= 2; i += 2) {
    while (bbox[i] > 180) {
      bbox[i] -= 360;
    }
    while (bbox[i] < -180) {
      bbox[i] += 360;
    }
  }
  return bbox;
}

function prepareRecords(records) {
  return records.map(record => {
    if (record.geometry && record.geometry.type === 'Polygon') {
      for (let ringIndex = 0; ringIndex < record.geometry.coordinates.length; ++ringIndex) {
        const ring = record.geometry.coordinates[ringIndex];
        let last = null;
        for (let i = 0; i < ring.length; ++i) {
          const current = ring[i];
          if (last) {
            if (current[0] - last[0] < -180) {
              current[0] += 360;
            }
            if (current[0] - last[0] > 180) {
              current[0] -= 360;
            }
          }
          last = current;
        }
      }
    }
    return record;
  });
}

/**
 * Convert a filters model, map model, and options to an OpenSearch parameters
 * object
 */
function convertFilters(filterParams, mapParams, options, format, service) {
  const description = service.getDescription();
  const url = description.getUrl(null, format || null);

  const parameters = {};

  const time = filterParams.time || mapParams.time;
  if (time) {
    if (Array.isArray(time)) {
      parameters['time:start'] = time[0];
      parameters['time:end'] = time[1];
    } else {
      parameters['time:start'] = time;
      parameters['time:end'] = time;
    }
  }

  const area = filterParams.area;
  if (area) {
    if (Array.isArray(area)) {
      parameters['geo:box'] = prepareBox(area);
    } else if (area.geometry) {
      const geometry = area.geometry;
      if (geometry.type === 'Point' && url.hasParameter('geo:lon') && url.hasParameter('geo:lat')) {
        parameters['geo:lon'] = geometry.coordinates[0];
        parameters['geo:lat'] = geometry.coordinates[1];
      } else {
        parameters['geo:geometry'] = geometry;
      }
    }
  } else if (mapParams) {
    // use the maps BBox by default
    parameters['geo:box'] = prepareBox(mapParams.bbox);
  }

  if (options.hasOwnProperty('itemsPerPage') && url.hasParameter('count')) {
    parameters.count = options.itemsPerPage;
  }

  if (options.hasOwnProperty('page')) {
    if (url.hasParameter('startIndex') && options.hasOwnProperty('itemsPerPage')) {
      parameters.startIndex = options.page * options.itemsPerPage + url.indexOffset;
    } else if (url.hasParameter('startPage')) {
      parameters.startPage = options.page + url.pageOffset;
    }
  }

  Object.keys(filterParams).forEach((key) => {
    if (url.hasParameter(key)) {
      parameters[key] = filterParams[key];
    }
  });

  return parameters;
}


function getService(url) {
  if (!self.services[url]) {
    // add a new promise
    // self.services[url] = discover(url, { useXHR: true, PromiseClass: BluebirdPromise });
    self.services[url] = discover(url, { useXHR: true });
  }
  return self.services[url];
}


function searchAll(url, method, filterParams, mapParams, options, format) {
  const maxCount = options.maxCount;

  return getService(url)
    .then(service => {
      const parameters = convertFilters(filterParams, mapParams, options, format, service);
      const paginator = service.getPaginator(parameters, format, method, true);
      if (maxCount) {
        return paginator.fetchFirstRecords(maxCount)
          .then(result => ({
            totalResults: result.totalResults,
            itemsPerPage: result.itemsPerPage,
            startIndex: result.startIndex,
            records: prepareRecords(result.records),
          }));
      }
      return paginator.fetchAllRecords()
{'fraction_non_alphanumeric': 0.1480185023127891, 'fraction_numerical': 0.011751468933616702, 'mean_word_length': 2.798670465337132, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1185563', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2460, 'n_words': 536}