<commit_before>import * as paths from "../../os/paths";
import * as sf from "../../os/sf";

import rootLogger from "../../logger";

import getGameCredentials from "./get-game-credentials";

import {
  IDownloadItem,
  IDownloadResult,
  currentRuntime,
  Cancelled,
} from "../../types";
import Context from "../../context";
import { Instance, messages } from "node-buse";
import { ICave } from "../../db/models/cave";

import configure from "../launch/configure";
import { promisedModal } from "../modals";

import * as actions from "../../actions";
import makeUploadButton from "../make-upload-button";

import { map } from "underscore";
import { MODAL_RESPONSE } from "../../constants/action-types";
import { buseGameCredentials, setupClient } from "../../util/buse-utils";
import { computeCaveLocation } from "./compute-cave-location";

export default async function performDownload(
  ctx: Context,
  item: IDownloadItem
): Promise<IDownloadResult> {
  let parentLogger = rootLogger;
  let caveIn: ICave;
  if (item.caveId) {
    caveIn = ctx.db.caves.findOneById(item.caveId);
  }
  const logger = parentLogger.child({ name: `download` });

  const { game } = item;
  const { preferences } = ctx.store.getState();

  const credentials = await getGameCredentials(ctx, item.game);
  if (!credentials) {
    throw new Error(`no game credentials, can't download`);
  }

  const { caveLocation, absoluteInstallFolder } = computeCaveLocation(
    item,
    preferences,
    caveIn
  );

  const stagingFolder = paths.downloadFolderPathForId(
    preferences,
    caveLocation.installLocation,
    item.id
  );
  if (await sf.exists(stagingFolder)) {
    logger.info(`Resuming!`);
  }

  let cave: ICave;
  let butlerExited = false;
  let cancelled = false;

  const instance = new Instance();
  instance.onClient(async client => {
    try {
      setupClient(client, logger, ctx);

      client.onRequest(messages.PickUpload, async ({ params }) => {
        const { uploads } = params;
        const { title } = game;

        const modalRes = await promisedModal(ctx.store, {
          title: ["pick_install_upload.title", { title }],
          message: ["pick_install_upload.message", { title }],
          detail: ["pick_install_upload.detail"],
          bigButtons: map(uploads, (candidate, index) => {
            return {
              ...makeUploadButton(candidate),
              action: actions.modalResponse({
                pickedUploadIndex: index,
              }),
            };
          }),
          buttons: ["cancel"],
        });

        if (modalRes.type === MODAL_RESPONSE) {
          return { index: modalRes.payload.pickedUploadIndex };
        } else {
          throw new Error(`no upload picked`);
        }
      });

      client.onRequest(messages.GetReceipt, async ({ params }) => {
        // TODO: read legacy-format receipt from existing install folder
        // plus DB info from cave
        logger.warn(`buse sent getReceipt: stub!`);

        return { receipt: null };
      });

      const id = item.id;

      ctx.on("graceful-cancel", () => {
        (async () => {
          logger.warn(`Received graceful-cancel`);
          cancelled = true;

          if (butlerExited) {
            // nothing to do
            logger.warn(
              `Nothing to do for graceful-cancel, butler already exited`
            );
            return;
          }

          const deadline = 5 * 1000;
          logger.warn(
            `Asking butler to cancel with a ${(deadline /
              1000).toFixed()}s deadline`
          );
          setTimeout(() => {
            if (butlerExited) {
              logger.info(`butler exited gracefully!`);
              return;
            }
            logger.warn(`butler failed to exit within deadline, killing it!`);

            try {
              instance.cancel();
            } catch (e) {
              logger.warn(`While cancelling instance: ${e.stack}`);
            }
          }, deadline);

          await client.call(messages.Operation.Cancel({ id }));
        })().catch(e => {
          logger.warn(`While discarding: ${e.stack}`);
        });
      });

      const res = await client.call(
        messages.Operation.Start({
          id,
          stagingFolder,
          operation: "install",
          installParams: {
            game,
            installFolder: absoluteInstallFolder,
            credentials: buseGameCredentials(credentials),
            upload: item.upload,
            fresh: item.reason === "install",
          },
        })
      );

      logger.info(`butler says operation ended`);
      logger.info(`final install result:\n${JSON.stringify(res, null, 2)}`);
      const ires = res.installResult;

      cave = {
        ...caveLocation,
        gameId: game.id,

        installedAt: new Date(),
        channelName: ires.upload.channelName,
        build: ires.build,
        upload: ires.upload,
        // TODO: butler should let us know if it was handpicked in the result.
        // We can't keep track of it ourselves, because performDownload() might
        // be called many times for the same download
        handPicked: false,
      };

      logger.info(`Committing game & cave to db`);
      ctx.db.saveOne("games", game.id, game);
      ctx.db.saveOne("caves", cave.id, cave);
    } finally {
      butlerExited = true;
      instance.cancel();
    }
  });

  await ctx.withStopper({
    work: async () => {
      await instance.promise();
    },
    stop: async () => {
      logger.info(`Asked to stop, cancelling butler process`);
      cancelled = true;
      instance.cancel();
    },
  });

  if (cancelled) {
    throw new Cancelled();
  }

  // TODO: move to buse
  logger.info(`Configuring...`);
  await configure(ctx, {
    game,
    cave,
    logger,
    runtime: currentRuntime(),
  });

  return {
    archivePath: null,
  };
}

/**
 * Mark cave as morphing, do `cb`, then clear the morphing marker
 * if it succeeded.
 */
// TODO: use for everything but first installs
export async function doMorphingOperation<T>(
  ctx: Context,
  caveId: string,
  cb: () => Promise<T>
): Promise<T> {
  ctx.db.saveOne("caves", caveId, { morphing: true });
  const res = await cb();
  // if cb throws, we'll never reach here
  ctx.db.saveOne("caves", caveId, { morphing: false });
  return res;
}
<commit_msg>Read legacy receipt + db info when asked to, closes #1594<commit_after>import * as paths from "../../os/paths";
import * as sf from "../../os/sf";

import rootLogger from "../../logger";

import getGameCredentials from "./get-game-credentials";

import {
  IDownloadItem,
  IDownloadResult,
  currentRuntime,
  Cancelled,
} from "../../types";
import Context from "../../context";
import { Instance, messages } from "node-buse";
import { ICave } from "../../db/models/cave";

import configure from "../launch/configure";
import { promisedModal } from "../modals";

import * as actions from "../../actions";
import makeUploadButton from "../make-upload-button";

import { map } from "underscore";
import { MODAL_RESPONSE } from "../../constants/action-types";
import { buseGameCredentials, setupClient } from "../../util/buse-utils";
import { computeCaveLocation } from "./compute-cave-location";
import { readReceipt } from "../../install-managers/common/receipt";

export default async function performDownload(
  ctx: Context,
  item: IDownloadItem
): Promise<IDownloadResult> {
  let parentLogger = rootLogger;
  let caveIn: ICave;
  if (item.caveId) {
    caveIn = ctx.db.caves.findOneById(item.caveId);
  }
  const logger = parentLogger.child({ name: `download` });

  const { game } = item;
  const { preferences } = ctx.store.getState();

  const credentials = await getGameCredentials(ctx, item.game);
  if (!credentials) {
    throw new Error(`no game credentials, can't download`);
  }

  const { caveLocation, absoluteInstallFolder } = computeCaveLocation(
    item,
    preferences,
    caveIn
  );

  const stagingFolder = paths.downloadFolderPathForId(
    preferences,
    caveLocation.installLocation,
    item.id
  );
  if (await sf.exists(stagingFolder)) {
    logger.info(`Resuming!`);
  }

  let cave: ICave;
  let butlerExited = false;
  let cancelled = false;

  const instance = new Instance();
  instance.onClient(async client => {
    try {
      setupClient(client, logger, ctx);

      client.onRequest(messages.PickUpload, async ({ params }) => {
        const { uploads } = params;
        const { title } = game;

        const modalRes = await promisedModal(ctx.store, {
          title: ["pick_install_upload.title", { title }],
          message: ["pick_install_upload.message", { title }],
          detail: ["pick_install_upload.detail"],
          bigButtons: map(uploads, (candidate, index) => {
            return {
              ...makeUploadButton(candidate),
              action: actions.modalResponse({
                pickedUploadIndex: index,
              }),
            };
          }),
          buttons: ["cancel"],
        });

        if (modalRes.type === MODAL_RESPONSE) {
          return { index: modalRes.payload.pickedUploadIndex };
        } else {
          throw new Error(`no upload picked`);
        }
      });

      client.onRequest(messages.GetReceipt, async ({ params }) => {
        logger.info(`butler asked for receipt info`);

        if (!caveIn) {
          logger.info(`no existing cave, returning null receipt`);
          return { receipt: null };
        }

        let files: string[] = [];
        const legacyReceipt = await readReceipt({
          ctx: ctx,
          logger: logger,
          destPath: absoluteInstallFolder,
        });

        if (
          legacyReceipt &&
          legacyReceipt.files &&
          Array.isArray(legacyReceipt.files)
        ) {
          files = legacyReceipt.files;
          logger.info(
            `found legacy receipt! (${legacyReceipt.files.length} files)`
          );
        } else {
          logger.info(`no legacy receipt`);
        }

        return {
          receipt: {
            files,
            upload: caveIn.upload,
            build: caveIn.build,
          },
        };
      });

      const id = item.id;

      ctx.on("graceful-cancel", () => {
        (async () => {
          logger.warn(`Received graceful-cancel`);
          cancelled = true;

          if (butlerExited) {
            // nothing to do
            logger.warn(
              `Nothing to do for graceful-cancel, butler already exited`
            );
            return;
          }

          const deadline = 5 * 1000;
          logger.warn(
            `Asking butler to cancel with a ${(deadline /
              1000).toFixed()}s deadline`
          );
          setTimeout(() => {
            if (butlerExited) {
              logger.info(`butler exited gracefully!`);
              return;
            }
            logger.warn(`butler failed to exit within deadline, killing it!`);

            try {
              instance.cancel();
            } catch (e) {
              logger.warn(`While cancelling instance: ${e.stack}`);
            }
          }, deadline);

          await client.call(messages.Operation.Cancel({ id }));
        })().catch(e => {
          logger.warn(`While discarding: ${e.stack}`);
        });
      });

      const res = await client.call(
        messages.Operation.Start({
          id,
          stagingFolder,
          operation: "install",
          installParams: {
            game,
            installFolder: absoluteInstallFolder,
            credentials: buseGameCredentials(credentials),
            upload: item.upload,
            fresh: item.reason === "install",
          },
        })
      );

      logger.debug(`butler says operation ended`);
      logger.debug(`final install result:\n${JSON.stringify(res, null, 2)}`);
      const ires = res.installResult;

      cave = {
        ...caveLocation,
        gameId: game.id,

        installedAt: new Date(),
        channelName: ires.upload.channelName,
        build: ires.build,
        upload: ires.upload,
        // TODO: butler should let us know if it was handpicked in the result.
        // We can't keep track of it ourselves, because performDownload() might
        // be called many times for the same download
        handPicked: false,
      };

      logger.info(`Committing game & cave to db`);
      ctx.db.saveOne("games", game.id, game);
      ctx.db.saveOne("caves", cave.id, cave);
    } finally {
      butlerExited = true;
      instance.cancel();
    }
  });

  await ctx.withStopper({
    work: async () => {
      await instance.promise();
    },
    stop: async () => {
      logger.info(`Asked to stop, cancelling butler process`);
      cancelled = true;
      instance.cancel();
    },
  });

  if (cancelled) {
    throw new Cancelled();
  }

  // TODO: move to buse
  logger.info(`Configuring...`);
  await configure(ctx, {
    game,
    cave,
    logger,
    runtime: currentRuntime(),
  });

  return {
    archivePath: null,
  };
}

/**
 * Mark cave as morphing, do `cb`, then clear the morphing marker
 * if it succeeded.
 */
// TODO: use for everything but first installs
export async function doMorphingOperation<T>(
  ctx: Context,
  caveId: string,
  cb: () => Promise<T>
): Promise<T> {
  ctx.db.saveOne("caves", caveId, { morphing: true });
  const res = await cb();
  // if cb throws, we'll never reach here
  ctx.db.saveOne("caves", caveId, { morphing: false });
  return res;
}

{'fraction_non_alphanumeric': 0.13405341421913147, 'fraction_numerical': 0.0017755419101871718, 'mean_word_length': 2.1040183696900114, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2755196', 'n_tokens_mistral': 3880, 'n_tokens_neox': 3861, 'n_words': 991}