<commit_before>const parse = require('co-busboy');
const sendfile = require('koa-sendfile');
const path = require('path');
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs-extra'));
const config = require('config');
const gm = require('gm');
const logger = require('./logger');
const util = require('./util');

Promise.promisifyAll(Object.getPrototypeOf(gm()));

function getFilePath(param) {
  const result = /(\d+)(.*)/.exec(param);
  if (!result) {
    return null;
  }

  const bookmarkIndex = parseInt(result[1], 10);
  const bookmark = config.bookmarks[bookmarkIndex];
  if (!bookmark) {
    return null;
  }

  let subdir = result[2];
  if (subdir.length > 0) {
    if (subdir[0] !== '/') {
      return null;
    }

    subdir = subdir.substr(1);
  }

  const realpath = path.normalize(path.resolve(bookmark.dir, subdir));
  if (!realpath.startsWith(bookmark.dir)) {
    return null;
  }

  return realpath;
}

function getImageInfo(filepath) {
  const info = {};
  return new Promise((resolve, reject) => {
    const img = gm(filepath);
    img.size((err, value) => {
      if (err) {
        reject(err);
        return;
      }

      info.size = value;
      img.orientation((err2, value2) => {
        if (err2) {
          reject(err2);
          return;
        }

        info.orientation = value2;
        resolve(info);
      });
    });
  });
}

const funcs = {
  *bookmarks() {
    const bookmarks = config.get('bookmarks');
    this.body = JSON.stringify(bookmarks.map(b => b.name));
  },

  *dir(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    let files = yield fs.readdirAsync(dir);
    files = files.map(file => {
      const stats = fs.lstatSync(path.resolve(dir, file));
      return {
        name: file,
        isDirectory: stats.isDirectory(),
        size: stats.size,
        mtime: stats.mtime.getTime(),
      };
    });
    this.body = files;
  },

  *imageInfo(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    this.body = yield getImageInfo(filepath);
  },

  *download(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    yield sendfile(this, filepath);
    if (!this.status) {
      this.throw(404);
    }
  },

  *image(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    if (!config.cacheDir) {
      yield sendfile(this, filepath);
      return;
    }

    const type = this.query.type;
    if (type !== 'sq100' && type !== 'max800') {
      this.body = 'invalid type';
      return;
    }

    const cacheFilepath = util.getImageCacheFilepath(filepath, type);
    yield util.createImageCache(type, filepath, cacheFilepath);

    yield sendfile(this, cacheFilepath);
    if (!this.status) {
      this.throw(404);
    }
  },

  *upload(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const parts = parse(this);
    let part = yield parts;
    while (part) {
      const filename = path.resolve(dir, part.filename);
      const readStream = part;
      const writeStream = fs.createWriteStream(filename);
      yield new Promise((resolve, reject) => {
        readStream.on('error', reject);
        writeStream.on('error', reject);
        writeStream.on('finish', resolve);
        readStream.pipe(writeStream);
      });

      part = yield parts;
    }

    this.status = 200;
  },

  *createFolder(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const folderName = this.request.body.name;

    const folderFilepath = path.resolve(dir, folderName);
    yield fs.mkdirAsync(folderFilepath);

    this.body = '{}';
  },

  *delete(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const files = this.request.body;

    if (!config.trashDir) {
      for (const file of files) {
        yield fs.unlinkAsync(path.resolve(dir, file));
      }
      this.body = '{}';
      return;
    }

    for (const file of files) {
      const sourceFilepath = path.resolve(dir, file);
      const trashFilename = `${Date.now()}_${file}`;
      const trashFilepath = path.resolve(config.trashDir, trashFilename);
      yield fs.moveAsync(sourceFilepath, trashFilepath);
    }
    this.body = '{}';
  },
};

function init(app) {
  app.use(function* apiHandler(next) {
    const reqPath = decodeURIComponent(this.path);
    const result = /^\/api\/(\w+)\/?(.*)/.exec(reqPath);
    if (!result) {
      yield next;
      return;
    }

    const apiName = result[1];
    const param = result[2];
    if (!funcs[apiName]) {
      logger.warn(`Invalid api: ${apiName}`);
      return;
    }

    try {
      yield* funcs[apiName].call(this, param);
    } catch (err) {
      logger.warn(`Failed to handle api: ${apiName}`, err, err.stack);
      this.status = 500;
    }
  });
}

module.exports = init;
<commit_msg>Add azure storage support<commit_after>const parse = require('co-busboy');
const sendfile = require('koa-sendfile');
const path = require('path');
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs-extra'));
const azure = require('azure-storage');
const config = require('config');
const gm = require('gm');
const logger = require('./logger');
const util = require('./util');

Promise.promisifyAll(Object.getPrototypeOf(gm()));
const blobService = azure.createBlobService();
let containers = [];

function getFilePath(param) {
  logger.info("param: " + param);

  const result = /(\d+)(.*)/.exec(param);
  logger.info(result);
  if (!result) {
    return null;
  }

  const containerIndex = parseInt(result[1], 10);
  const container = containers[containerIndex];
  logger.info("container: " + container.name);
  if (!container) {
    return null;
  }

  let subdir = result[2];
  if (subdir.length > 0) {
    if (subdir[0] !== '/') {
      return null;
    }

    subdir = subdir.substr(1);
  }
  return container.name;
}

function getImageInfo(filepath) {
  const info = {};
  return new Promise((resolve, reject) => {
    const img = gm(filepath);
    img.size((err, value) => {
      if (err) {
        reject(err);
        return;
      }

      info.size = value;
      img.orientation((err2, value2) => {
        if (err2) {
          reject(err2);
          return;
        }

        info.orientation = value2;
        resolve(info);
      });
    });
  });
}
let blobs = [];

function aggregateContainers(err, result, cb) {
  if (err) {
      cb(err);
  } else {
      containers = containers.concat(result.entries);
      if (result.continuationToken !== null) {
          blobService
              .listContainersSegmented(result.continuationToken, aggregateContainers);
      } else {
          cb(null, containers);
      }
  }
}

function getContainersAsync() {
  logger.info('getContainers');
  containers = [];
  return new Promise(function(resolve, reject) {
    blobService.listContainersSegmented(null, function(err, result) {
      aggregateContainers(err, result, function(err, containers) {
        if (err) {
          logger.warn(err);
          reject(err);
        } else {
          resolve(containers);
        }
      });
    });
  });
}


function aggregateBlobs(containerName, err, result, cb) {
  if (err) {
      cb(err);
  } else {
      blobs = blobs.concat(result.entries);
      if (result.continuationToken !== null) {
          blobService
              .listBlobsSegmented(containerName, result.continuationToken, aggregateBlobs);
      } else {
        logger.info(containerName + ": " + blobs.length);
        cb(null, blobs);
      }
  }
}

function getBlobsAsync(containerName) {
  logger.info('getBlobs');
  blobs = [];
  return new Promise(function(resolve, reject) {
    blobService.listBlobsSegmented(containerName, null, function(err, result) {
      aggregateBlobs(containerName, err, result, function(err, blobs) {
        if (err) {
          logger.warn(err);
          reject(err);
        } else {
          logger.info(containerName + " resolve: " + blobs.length);
          resolve(blobs);
        }
      });
    });
  });
}

const funcs = {
  *bookmarks() {
    const bookmarks = config.get('bookmarks');
    this.body = JSON.stringify(bookmarks.map(b => b.name));
  },

  *containers() {
    containers = yield getContainersAsync();
    this.body = JSON.stringify(containers.map(c => c.name));;
  },

  *dir(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid directory';
      return;
    }
    let files =[];
    logger.info("dir: " + dir);
    let containerFiles = yield getBlobsAsync(dir);
    files = files.concat(containerFiles);
    
    files = files.map(file => {
      //const stats = fs.lstatSync(path.resolve(dir, file));
      return {
        name: file.name,
        isDirectory: file.blobType === "BlockBlob",
        size: file.contentLength,
        mtime: file.lastModified,
      };
    });
    this.body = files;
  },

  *imageInfo(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    this.body = yield getImageInfo(filepath);
  },

  *download(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    yield sendfile(this, filepath);
    if (!this.status) {
      this.throw(404);
    }
  },

  *image(param) {
    const filepath = getFilePath(param);
    if (!filepath) {
      this.body = 'invalid location';
      return;
    }

    if (!config.cacheDir) {
      yield sendfile(this, filepath);
      return;
    }

    const type = this.query.type;
    if (type !== 'sq100' && type !== 'max800') {
      this.body = 'invalid type';
      return;
    }

    const cacheFilepath = util.getImageCacheFilepath(filepath, type);
    yield util.createImageCache(type, filepath, cacheFilepath);

    yield sendfile(this, cacheFilepath);
    if (!this.status) {
      this.throw(404);
    }
  },

  *upload(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const parts = parse(this);
    let part = yield parts;
    while (part) {
      const filename = path.resolve(dir, part.filename);
      const readStream = part;
      const writeStream = fs.createWriteStream(filename);
      yield new Promise((resolve, reject) => {
        readStream.on('error', reject);
        writeStream.on('error', reject);
        writeStream.on('finish', resolve);
        readStream.pipe(writeStream);
      });

      part = yield parts;
    }

    this.status = 200;
  },

  *createFolder(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const folderName = this.request.body.name;

    const folderFilepath = path.resolve(dir, folderName);
    yield fs.mkdirAsync(folderFilepath);

    this.body = '{}';
  },

  *delete(param) {
    const dir = getFilePath(param);
    if (!dir) {
      this.body = 'invalid location';
      return;
    }

    const files = this.request.body;

    if (!config.trashDir) {
      for (const file of files) {
        yield fs.unlinkAsync(path.resolve(dir, file));
      }
      this.body = '{}';
      return;
    }

    for (const file of files) {
      const sourceFilepath = path.resolve(dir, file);
      const trashFilename = `${Date.now()}_${file}`;
      const trashFilepath = path.resolve(config.trashDir, trashFilename);
      yield fs.moveAsync(sourceFilepath, trashFilepath);
    }
    this.body = '{}';
  },
};

function init(app) {
  app.use(function* apiHandler(next) {
    const reqPath = decodeURIComponent(this.path);
    const result = /^\/api\/(\w+)\/?(.*)/.exec(reqPath);
    if (!result) {
      yield next;
      return;
    }

    const apiName = result[1];
    const param = result[2];
    if (!funcs[apiName]) {
      logger.warn(`Invalid api: ${apiName}`);
      return;
    }

    try {
      yield* funcs[apiName].call(this, param);
    } catch (err) {
      logger.warn(`Failed to handle api: ${apiName}`, err, err.stack);
      this.status = 500;
    }
  });
}

module.exports = init;

{'fraction_non_alphanumeric': 0.15158787297016238, 'fraction_numerical': 0.0051195904327653785, 'mean_word_length': 2.5197072072072073, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '599756', 'n_tokens_mistral': 4048, 'n_tokens_neox': 3960, 'n_words': 839}