<commit_before>package org.mercycorps.translationcards.txcmaker.task;

import com.google.api.client.http.InputStreamContent;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.ParentReference;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Reader;
import java.nio.channels.Channels;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Random;
import java.util.regex.Pattern;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.mercycorps.translationcards.txcmaker.auth.AuthUtils;

public class TxcBuilderTaskHandler extends HttpServlet {

  private static final String CSV_EXPORT_TYPE = "text/csv";

  private static final String SRC_HEADER_LANGUAGE = "Language";
  private static final String SRC_HEADER_LABEL = "Label";
  private static final String SRC_HEADER_TRANSLATION_TEXT = "Translation";
  private static final String SRC_HEADER_FILENAME = "Filename";

  private static final int BUFFER_SIZE = 1024;
  private static final String GCS_BUCKET_NAME = "translation-cards-dev.appspot.com";

  private static final Pattern FILE_URL_MATCHER = Pattern.compile(
      "https?://docs.google.com/spreadsheets/d/(.*?)(/.*)?$");
  private static final Pattern DIR_URL_MATCHER = Pattern.compile(
      "https?://drive.google.com/corp/drive/folders/(.*)$");

  private byte[] buffer = new byte[BUFFER_SIZE];
  private GcsService gcsService = GcsServiceFactory.createGcsService();

  @Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    produceTxcJson(req, resp);
  }

  private void produceTxcJson(HttpServletRequest req, HttpServletResponse resp)
      throws IOException {
    Drive drive = AuthUtils.getDriveOrOAuth(
        getServletContext(), req, resp, req.getParameter("sessionId"), false);
    Random random = new Random();
    GcsFilename gcsFilename = new GcsFilename(
        GCS_BUCKET_NAME, String.format("tmp-txc-%d", random.nextInt()));
    OutputStream gcsOutput = Channels.newOutputStream(
        gcsService.createOrReplace(gcsFilename, GcsFileOptions.getDefaultInstance()));
    TxcPortingUtility.ExportSpec exportSpec = new TxcPortingUtility.ExportSpec()
        .setDeckLabel(req.getParameter("deckName"))
        .setPublisher(req.getParameter("publisher"))
        .setDeckId(req.getParameter("deckId"))
        .setTimestamp(System.currentTimeMillis())
        .setLicenseUrl(req.getParameter("licenseUrl"))
        .setLocked(req.getParameter("locked") != null);
    String audioDirId = TxcPortingUtility.getAudioDirId(req);
    ChildList audioList = drive.children().list(audioDirId).execute();
    Map<String, String> audioFileIds = new HashMap<String, String>();
    for (ChildReference audioRef : audioList.getItems()) {
      File audioFile = drive.files().get(audioRef.getId()).execute();
      audioFileIds.put(audioFile.getOriginalFilename(), audioRef.getId());
    }
    String spreadsheetFileId = TxcPortingUtility.getSpreadsheetId(req);
    Drive.Files.Export sheetExport = drive.files().export(spreadsheetFileId, CSV_EXPORT_TYPE);
    Reader reader = new InputStreamReader(sheetExport.executeMediaAsInputStream());
    CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withHeader());
    Set<String> includedAudioFiles = new HashSet<String>();
    ZipOutputStream zipOutput = new ZipOutputStream(gcsOutput);
    try {
      for (CSVRecord row : parser) {
        String language = row.get(SRC_HEADER_LANGUAGE);
        String filename = row.get(SRC_HEADER_FILENAME);
        TxcPortingUtility.CardSpec card = new TxcPortingUtility.CardSpec()
            .setLabel(row.get(SRC_HEADER_LABEL))
            .setFilename(filename)
            .setTranslationText(row.get(SRC_HEADER_TRANSLATION_TEXT));
        exportSpec.addCard(language, card);
        if (includedAudioFiles.contains(filename)) {
          continue;
        }
        includedAudioFiles.add(filename);
        zipOutput.putNextEntry(new ZipEntry(filename));
        drive.files().get(audioFileIds.get(filename)).executeMediaAndDownloadTo(zipOutput);
        zipOutput.closeEntry();
      }
      zipOutput.putNextEntry(new ZipEntry("card_deck.json"));
      zipOutput.write(TxcPortingUtility.buildTxcJson(exportSpec).getBytes());
      zipOutput.closeEntry();
    } finally {
      parser.close();
      reader.close();
      zipOutput.close();
    }
    File targetFileInfo = new File();
    String targetFilename = req.getParameter("deckName")
        .replaceAll(" ", "_")
        .replaceAll("[^a-zA-Z_]", "");
    targetFilename += ".txc";
    targetFileInfo.setTitle(targetFilename);
    targetFileInfo.setParents(Collections.singletonList(new ParentReference().setId(audioDirId)));
    InputStream txcContentStream = Channels.newInputStream(
        gcsService.openPrefetchingReadChannel(gcsFilename, 0, BUFFER_SIZE));
    drive.files().insert(targetFileInfo, new InputStreamContent(null, txcContentStream)).execute();
    resp.getWriter().println(
        "Your TXC is being assembled and the file should arrive in Drive in a minute or two.");
  }
}
<commit_msg>Austin - refactor build txc task<commit_after>package org.mercycorps.translationcards.txcmaker.task;

import com.google.api.client.http.InputStreamContent;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.ParentReference;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Reader;
import java.nio.channels.Channels;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Random;
import java.util.regex.Pattern;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.mercycorps.translationcards.txcmaker.auth.AuthUtils;

public class TxcBuilderTaskHandler extends HttpServlet {

  private static final String CSV_EXPORT_TYPE = "text/csv";

  private static final String SRC_HEADER_LANGUAGE = "Language";
  private static final String SRC_HEADER_LABEL = "Label";
  private static final String SRC_HEADER_TRANSLATION_TEXT = "Translation";
  private static final String SRC_HEADER_FILENAME = "Filename";

  private static final int BUFFER_SIZE = 1024;
  private static final String GCS_BUCKET_NAME = "translation-cards-dev.appspot.com";

  private static final Pattern FILE_URL_MATCHER = Pattern.compile(
      "https?://docs.google.com/spreadsheets/d/(.*?)(/.*)?$");
  private static final Pattern DIR_URL_MATCHER = Pattern.compile(
      "https?://drive.google.com/corp/drive/folders/(.*)$");

  private byte[] buffer = new byte[BUFFER_SIZE];
  private GcsService gcsService = GcsServiceFactory.createGcsService();

  @Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    produceTxcJson(req, resp);
  }

  private void produceTxcJson(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    Drive drive = AuthUtils.getDriveOrOAuth(
        getServletContext(), req, resp, req.getParameter("sessionId"), false);

    Random random = new Random();
    GcsFilename gcsFilename = new GcsFilename(
        GCS_BUCKET_NAME, String.format("tmp-txc-%d", random.nextInt()));
    OutputStream gcsOutput = Channels.newOutputStream(
        gcsService.createOrReplace(gcsFilename, GcsFileOptions.getDefaultInstance()));

    String audioDirId = assembleTxc(req, drive, gcsOutput);

    pushTxcToDrive(req, drive, gcsFilename, audioDirId);

    resp.getWriter().println(
        "Your TXC is being assembled and the file should arrive in Drive in a minute or two.");
  }

  private String assembleTxc(HttpServletRequest req, Drive drive, OutputStream gcsOutput) throws IOException {
    TxcPortingUtility.ExportSpec exportSpec = createExportSpec(req);
    String audioDirId = TxcPortingUtility.getAudioDirId(req);
    ChildList audioList = drive.children().list(audioDirId).execute();
    Map<String, String> audioFileIds = readAudioFileIdsFromCsv(drive, audioList);
    String spreadsheetFileId = TxcPortingUtility.getSpreadsheetId(req);
    Drive.Files.Export sheetExport = drive.files().export(spreadsheetFileId, CSV_EXPORT_TYPE);
    Reader reader = new InputStreamReader(sheetExport.executeMediaAsInputStream());
    CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withHeader());
    Set<String> includedAudioFiles = new HashSet<String>();
    ZipOutputStream zipOutput = new ZipOutputStream(gcsOutput);
    try {
      for (CSVRecord row : parser) {
        String language = row.get(SRC_HEADER_LANGUAGE);
        String filename = row.get(SRC_HEADER_FILENAME);
        TxcPortingUtility.CardSpec card = createCardSpec(row, filename);
        exportSpec.addCard(language, card);
        if (includedAudioFiles.contains(filename)) {
          continue;
        }
        includedAudioFiles.add(filename);
        zipOutput.putNextEntry(new ZipEntry(filename));
        drive.files().get(audioFileIds.get(filename)).executeMediaAndDownloadTo(zipOutput);
        zipOutput.closeEntry();
      }
      zipOutput.putNextEntry(new ZipEntry("card_deck.json"));
      zipOutput.write(TxcPortingUtility.buildTxcJson(exportSpec).getBytes());
      zipOutput.closeEntry();
    } finally {
      parser.close();
      reader.close();
      zipOutput.close();
    }
    return audioDirId;
  }

  private TxcPortingUtility.CardSpec createCardSpec(CSVRecord row, String filename) {
    return new TxcPortingUtility.CardSpec()
            .setLabel(row.get(SRC_HEADER_LABEL))
            .setFilename(filename)
            .setTranslationText(row.get(SRC_HEADER_TRANSLATION_TEXT));
  }

  private TxcPortingUtility.ExportSpec createExportSpec(HttpServletRequest req) {
    return new TxcPortingUtility.ExportSpec()
        .setDeckLabel(req.getParameter("deckName"))
        .setPublisher(req.getParameter("publisher"))
        .setDeckId(req.getParameter("deckId"))
        .setTimestamp(System.currentTimeMillis())
        .setLicenseUrl(req.getParameter("licenseUrl"))
        .setLocked(req.getParameter("locked") != null);
  }

  private Map<String, String> readAudioFileIdsFromCsv(Drive drive, ChildList audioList) throws IOException {
    Map<String, String> audioFileIds = new HashMap<String, String>();
    for (ChildReference audioRef : audioList.getItems()) {
      File audioFile = drive.files().get(audioRef.getId()).execute();
      audioFileIds.put(audioFile.getOriginalFilename(), audioRef.getId());
    }
    return audioFileIds;
  }

  private void pushTxcToDrive(HttpServletRequest req, Drive drive, GcsFilename gcsFilename, String audioDirId) throws IOException {
    File targetFileInfo = new File();
    String targetFilename = req.getParameter("deckName")
        .replaceAll(" ", "_")
        .replaceAll("[^a-zA-Z_]", "");
    targetFilename += ".txc";
    targetFileInfo.setTitle(targetFilename);
    targetFileInfo.setParents(Collections.singletonList(new ParentReference().setId(audioDirId)));
    InputStream txcContentStream = Channels.newInputStream(
        gcsService.openPrefetchingReadChannel(gcsFilename, 0, BUFFER_SIZE));
    drive.files().insert(targetFileInfo, new InputStreamContent(null, txcContentStream)).execute();
  }
}

{'fraction_non_alphanumeric': 0.11017945302092312, 'fraction_numerical': 0.0007836376459525115, 'mean_word_length': 4.631950573698147, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7063657', 'n_tokens_mistral': 3727, 'n_tokens_neox': 3683, 'n_words': 694}