<commit_before>import JSZip  from 'jszip';
import pako   from 'pako';

import macro                from 'vtk.js/Sources/macro';
import Endian               from 'vtk.js/Sources/Common/Core/Endian';
import { DataTypeByteSize } from 'vtk.js/Sources/Common/Core/DataArray/Constants';

const { vtkErrorMacro, vtkDebugMacro } = macro;

function removeLeadingSlash(str) {
  return (str[0] === '/') ? str.substr(1) : str;
}

function create(options) {
  let ready = false;
  let requestCount = 0;
  const zip = new JSZip();
  let zipRoot = zip;
  zip.loadAsync(options.zipContent)
    .then(() => {
      ready = true;

      // Find root index.json
      const metaFiles = [];
      zip.forEach((relativePath, zipEntry) => {
        if (relativePath.indexOf('index.json') !== -1) {
          metaFiles.push(relativePath);
        }
      });
      metaFiles.sort((a, b) => a.length - b.length);
      const fullRootPath = metaFiles[0].split('/');
      while (fullRootPath.length > 1) {
        const dirName = fullRootPath.shift();
        zipRoot = zipRoot.folder(dirName);
      }

      if (options.callback) {
        options.callback(zip);
      }
    });
  return {
    fetchArray(instance = {}, baseURL, array, fetchGzip = false) {
      return new Promise((resolve, reject) => {
        if (!ready) {
          vtkErrorMacro('ERROR!!! zip not ready...');
        }
        const url = removeLeadingSlash([baseURL, array.ref.basepath, fetchGzip ? `${array.ref.id}.gz` : array.ref.id].join('/'));

        if (++requestCount === 1 && instance.invokeBusy) {
          instance.invokeBusy(true);
        }

        zipRoot.file(url)
          .async('uint8array')
          .then((uint8array) => {
            array.buffer = new ArrayBuffer(uint8array.length);

            // copy uint8array to buffer
            const view = new Uint8Array(array.buffer);
            view.set(uint8array);

            if (fetchGzip) {
              if (array.dataType === 'string' || array.dataType === 'JSON') {
                array.buffer = pako.inflate(new Uint8Array(array.buffer), { to: 'string' });
              } else {
                array.buffer = pako.inflate(new Uint8Array(array.buffer)).buffer;
              }
            }

            if (array.ref.encode === 'JSON') {
              array.values = JSON.parse(array.buffer);
            } else {
              if (Endian.ENDIANNESS !== array.ref.encode && Endian.ENDIANNESS) {
                // Need to swap bytes
                vtkDebugMacro(`Swap bytes of ${array.name}`);
                Endian.swapBytes(array.buffer, DataTypeByteSize[array.dataType]);
              }

              array.values = new window[array.dataType](array.buffer);
            }

            if (array.values.length !== array.size) {
              vtkErrorMacro(`Error in FetchArray: ${array.name} does not have the proper array size. Got ${array.values.length}, instead of ${array.size}`);
            }

            // Done with the ref and work
            delete array.ref;
            if (--requestCount === 0 && instance.invokeBusy) {
              instance.invokeBusy(false);
            }
            if (instance.modified) {
              instance.modified();
            }
            resolve(array);
          });
      });
    },

    fetchJSON(instance = {}, url, compression) {
      const path = removeLeadingSlash(url);
      if (!ready) {
        vtkErrorMacro('ERROR!!! zip not ready...');
      }

      if (compression) {
        if (compression === 'gz') {
          return zipRoot.file(path).async('uint8array').then((uint8array) => {
            const str = pako.inflate(uint8array, { to: 'string' });
            return new Promise(ok => ok(JSON.parse(str)));
          });
        }
        return new Promise((a, r) => r('Invalid compression'));
      }

      return zipRoot.file(path).async('string').then(str => new Promise(ok => ok(JSON.parse(str))));
    },

    fetchText(instance = {}, url, compression) {
      const path = removeLeadingSlash(url);
      if (!ready) {
        vtkErrorMacro('ERROR!!! zip not ready...');
      }

      if (compression) {
        if (compression === 'gz') {<commit_msg>fix(JSZipDataAccessHelper): Support fetching string arrays<commit_after>import JSZip  from 'jszip';
import pako   from 'pako';

import macro                from 'vtk.js/Sources/macro';
import Endian               from 'vtk.js/Sources/Common/Core/Endian';
import { DataTypeByteSize } from 'vtk.js/Sources/Common/Core/DataArray/Constants';

const { vtkErrorMacro, vtkDebugMacro } = macro;

function handleUint8Array(array, fetchGzip, done) {
  return (uint8array) => {
    array.buffer = new ArrayBuffer(uint8array.length);

    // copy uint8array to buffer
    const view = new Uint8Array(array.buffer);
    view.set(uint8array);

    if (fetchGzip) {
      if (array.dataType === 'string' || array.dataType === 'JSON') {
        array.buffer = pako.inflate(new Uint8Array(array.buffer), { to: 'string' });
      } else {
        array.buffer = pako.inflate(new Uint8Array(array.buffer)).buffer;
      }
    }

    if (array.ref.encode === 'JSON') {
      array.values = JSON.parse(array.buffer);
    } else {
      if (Endian.ENDIANNESS !== array.ref.encode && Endian.ENDIANNESS) {
        // Need to swap bytes
        vtkDebugMacro(`Swap bytes of ${array.name}`);
        Endian.swapBytes(array.buffer, DataTypeByteSize[array.dataType]);
      }

      array.values = new window[array.dataType](array.buffer);
    }

    if (array.values.length !== array.size) {
      vtkErrorMacro(`Error in FetchArray: ${array.name} does not have the proper array size. Got ${array.values.length}, instead of ${array.size}`);
    }

    done();
  };
}

function handleString(array, fetchGzip, done) {
  return (string) => {
    array.values = JSON.parse(string);
    done();
  };
}

const handlers = {
  uint8array: handleUint8Array,
  string: handleString,
};

function removeLeadingSlash(str) {
  return (str[0] === '/') ? str.substr(1) : str;
}

function create(options) {
  let ready = false;
  let requestCount = 0;
  const zip = new JSZip();
  let zipRoot = zip;
  zip.loadAsync(options.zipContent)
    .then(() => {
      ready = true;

      // Find root index.json
      const metaFiles = [];
      zip.forEach((relativePath, zipEntry) => {
        if (relativePath.indexOf('index.json') !== -1) {
          metaFiles.push(relativePath);
        }
      });
      metaFiles.sort((a, b) => a.length - b.length);
      const fullRootPath = metaFiles[0].split('/');
      while (fullRootPath.length > 1) {
        const dirName = fullRootPath.shift();
        zipRoot = zipRoot.folder(dirName);
      }

      if (options.callback) {
        options.callback(zip);
      }
    });
  return {
    fetchArray(instance = {}, baseURL, array, fetchGzip = false) {
      return new Promise((resolve, reject) => {
        if (!ready) {
          vtkErrorMacro('ERROR!!! zip not ready...');
        }
        const url = removeLeadingSlash([baseURL, array.ref.basepath, fetchGzip ? `${array.ref.id}.gz` : array.ref.id].join('/'));

        if (++requestCount === 1 && instance.invokeBusy) {
          instance.invokeBusy(true);
        }

        function doneCleanUp() {
          // Done with the ref and work
          delete array.ref;
          if (--requestCount === 0 && instance.invokeBusy) {
            instance.invokeBusy(false);
          }
          if (instance.modified) {
            instance.modified();
          }
          resolve(array);
        }

        const asyncType = array.dataType === 'string' ? 'string' : 'uint8array';
        const asyncCallback = handlers[asyncType](array, fetchGzip, doneCleanUp);

        zipRoot.file(url)
          .async(asyncType)
          .then(asyncCallback);
      });
    },

    fetchJSON(instance = {}, url, compression) {
      const path = removeLeadingSlash(url);
      if (!ready) {
        vtkErrorMacro('ERROR!!! zip not ready...');
      }

      if (compression) {
        if (compression === 'gz') {
          return zipRoot.file(path).async('uint8array').then((uint8array) => {
            const str = pako.inflate(uint8array, { to: 'string' });
            return new Promise(ok => ok(JSON.parse(str)));
          });
        }
        return new Promise((a, r) => r('Invalid compression'));
      }

      return zipRoot.file(path).async('string').then(str => new Promise(ok => ok(JSON.parse(str))));
    },

    fetchText(instance = {}, url, compression) {
      const path = removeLeadingSlash(url);
      if (!ready) {
        vtkErrorMacro('ERROR!!! zip not ready...');
      }

      if (compression) {
        if (compression === 'gz') {
{'fraction_non_alphanumeric': 0.1503802719520627, 'fraction_numerical': 0.00472459091956672, 'mean_word_length': 2.1955081001472756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4737262', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2595, 'n_words': 543}