<commit_before>import * as Skins from "../data/skins";
import S3 from "../s3";
import { addSkinFromBuffer } from "../addSkin";
import { EventHandler } from "./app";

async function* reportedUploads() {
  const upload = await Skins.getReportedUpload();
  if (upload != null) {
    yield upload;
  }
}

let processing = false;

export async function processUserUploads(eventHandler: EventHandler) {
  // Ensure we only have one worker processing requests.
  if (processing) {
    return;
  }
  processing = true;
  const uploads = reportedUploads();
  for await (const upload of uploads) {
    try {
      const buffer = await S3.getUploadedSkin(upload.id);
      const result = await addSkinFromBuffer(
        buffer,
        upload.filename,
        "Web API"
      );
      await Skins.recordUserUploadArchived(upload.id);
      if (result.status === "ADDED") {
        const action = {
          type:
            result.skinType === "CLASSIC"
              ? "CLASSIC_SKIN_UPLOADED"
              : "MODERN_SKIN_UPLOADED",
          md5: result.md5,
        } as const;
        eventHandler(action);
      }
    } catch (e) {
      await Skins.recordUserUploadErrored(upload.id);
      const action = {
        type: "SKIN_UPLOAD_ERROR",
        uploadId: upload.id,
        message: e.message,
      } as const;
      eventHandler(action);
      console.error(e);
    }
  }

  processing = false;
}<commit_msg>Improve user uploads<commit_after>import * as Skins from "../data/skins";
import S3 from "../s3";
import { addSkinFromBuffer } from "../addSkin";
import { EventHandler } from "./app";

async function* reportedUploads() {
  const seen = new Set();
  while (true) {
    const upload = await Skins.getReportedUpload();
    if (upload == null) {
      return;
    }
    if (seen.has(upload.id)) {
      await Skins.recordUserUploadErrored(upload.id);
      console.error("Saw the same upload twice. It didn't get handled?");
      return;
    }
    seen.add(upload.id);
    yield upload;
  }
}

let processing = false;

function log(...args: any[]) {
  console.log(...args);
}

export async function processUserUploads(eventHandler: EventHandler) {
  log("process user uploads");
  // Ensure we only have one worker processing requests.
  if (processing) {
    return;
  }
  processing = true;
  const uploads = reportedUploads();
  log("Uploads to process: ", uploads);
  for await (const upload of uploads) {
    log("Going to try: ", upload);
    try {
      const buffer = await S3.getUploadedSkin(upload.id);
      log("Got buffer: ", upload);
      const result = await addSkinFromBuffer(
        buffer,
        upload.filename,
        "Web API"
      );
      log("Added skin from buffer: ", upload, result);
      await Skins.recordUserUploadArchived(upload.id);
      log("Recorded upload archived: ", upload);
      if (result.status === "ADDED") {
        const action = {
          type:
            result.skinType === "CLASSIC"
              ? "CLASSIC_SKIN_UPLOADED"
              : "MODERN_SKIN_UPLOADED",
          md5: result.md5,
        } as const;
        log("Skin was added, sending action:", action);
        eventHandler(action);
        log("Action sent:", action);
      }
    } catch (e) {
      log("Upload errored, going to report it:", upload);
      await Skins.recordUserUploadErrored(upload.id);
      log("Recorded upload errored:", upload);
      const action = {
        type: "SKIN_UPLOAD_ERROR",
        uploadId: upload.id,
        message: e.message,
      } as const;
      eventHandler(action);
      console.error(e);
    }
  }

  processing = false;
}
{'fraction_non_alphanumeric': 0.12441119423663065, 'fraction_numerical': 0.002770850651149903, 'mean_word_length': 2.4545454545454546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '127772', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1106, 'n_words': 286}