<commit_before>package org.ambraproject.rhino;

import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
import org.ambraproject.models.ArticleAsset;
import org.ambraproject.rhino.content.xml.ArticleXml;
import org.ambraproject.rhino.content.xml.XmlContentException;
import org.ambraproject.rhino.identity.ArticleIdentity;
import org.ambraproject.rhino.identity.AssetFileIdentity;
import org.ambraproject.rhino.identity.AssetIdentity;
import org.ambraproject.rhino.service.impl.AmbraService;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * Utilities for creating test ingestibles.
 */
public class IngestibleUtil {

  /**
   * Create a file name that uniquely identifies an asset file in an ingestible.
   */
  private static String getDummyFileName(ArticleAsset asset) {
    // Since the file name should be independent of the asset identity anyway, we might as well use consistent nonsense.
    return Hashing.sha1().newHasher()
        .putUnencodedChars(asset.getDoi())
        .putLong(0x51168181d9999aefL)
        .putUnencodedChars(asset.getExtension())
        .hash().toString();
  }

  private static Document buildManifest(ArticleIdentity identity, List<ArticleAsset> expectedAssets) {
    Document manifest = AmbraService.newDocumentBuilder().newDocument();
    Element manifestElement = (Element) manifest.appendChild(manifest.createElement("manifest"));
    Element articleBundle = (Element) manifestElement.appendChild(manifest.createElement("articleBundle"));

    Element articleElement = (Element) articleBundle.appendChild(manifest.createElement("article"));
    String xmlFileName = identity.forXmlAsset().getFileName();
    articleElement.setAttribute("uri", identity.getKey());
    articleElement.setAttribute("main-entry", xmlFileName);

    Element mainEntryRepresentation = (Element) articleElement.appendChild(manifest.createElement("representation"));
    mainEntryRepresentation.setAttribute("name", "XML");
    mainEntryRepresentation.setAttribute("entry", xmlFileName);

    Map<String, List<ArticleAsset>> assetGroups = expectedAssets.stream().collect(Collectors.groupingBy(ArticleAsset::getDoi));
    for (Map.Entry<String, List<ArticleAsset>> entry : assetGroups.entrySet()) {
      AssetIdentity assetId = AssetIdentity.create(entry.getKey());

      final Element parentElement;
      if (identity.getIdentifier().equals(assetId.getIdentifier())) {
        parentElement = articleElement;
      } else {
        parentElement = (Element) articleBundle.appendChild(manifest.createElement("object"));
        parentElement.setAttribute("uri", assetId.getKey());
      }

      for (ArticleAsset asset : entry.getValue()) {
        AssetFileIdentity fileId = AssetFileIdentity.create(asset);
        if (fileId.getFileExtension().equals("XML")) {
          continue; // already created above
        }
        Element assetRepresentation = (Element) parentElement.appendChild(manifest.createElement("representation"));
        assetRepresentation.setAttribute("name", asset.getExtension());
        assetRepresentation.setAttribute("entry", getDummyFileName(asset));
      }
    }

    return manifest;
  }

  public static InputStream buildMockIngestible(InputStream xml, List<ArticleAsset> expectedAssets) throws IOException {
    byte[] xmlData;<commit_msg>DPRO-2107: Debug creation of mock ingestible<commit_after>package org.ambraproject.rhino;

import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
import org.ambraproject.models.ArticleAsset;
import org.ambraproject.rhino.content.xml.ArticleXml;
import org.ambraproject.rhino.content.xml.XmlContentException;
import org.ambraproject.rhino.identity.ArticleIdentity;
import org.ambraproject.rhino.identity.AssetFileIdentity;
import org.ambraproject.rhino.identity.AssetIdentity;
import org.ambraproject.rhino.service.impl.AmbraService;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * Utilities for creating test ingestibles.
 */
public class IngestibleUtil {

  /**
   * Create a file name that uniquely identifies an asset file in an ingestible.
   */
  private static String getDummyFileName(ArticleAsset asset) {
    // Since the file name should be independent of the asset identity anyway, we might as well use consistent nonsense.
    return Hashing.sha1().newHasher()
        .putUnencodedChars(asset.getDoi())
        .putLong(0x51168181d9999aefL)
        .putUnencodedChars(asset.getExtension())
        .hash().toString();
  }

  private static Document buildManifest(ArticleIdentity identity, List<ArticleAsset> expectedAssets) {
    Document manifest = AmbraService.newDocumentBuilder().newDocument();
    Element manifestElement = (Element) manifest.appendChild(manifest.createElement("manifest"));
    Element articleBundle = (Element) manifestElement.appendChild(manifest.createElement("articleBundle"));

    Element articleElement = (Element) articleBundle.appendChild(manifest.createElement("article"));
    String xmlFileName = identity.forXmlAsset().getFileName();
    articleElement.setAttribute("uri", identity.getKey());
    articleElement.setAttribute("main-entry", xmlFileName);

    Element mainEntryRepresentation = (Element) articleElement.appendChild(manifest.createElement("representation"));
    mainEntryRepresentation.setAttribute("name", "XML");
    mainEntryRepresentation.setAttribute("entry", xmlFileName);

    Map<String, List<ArticleAsset>> assetGroups = expectedAssets.stream().collect(Collectors.groupingBy(ArticleAsset::getDoi));
    assetGroups = new TreeMap<>(assetGroups); // DEBUG!
    for (Map.Entry<String, List<ArticleAsset>> entry : assetGroups.entrySet()) {
      AssetIdentity assetId = AssetIdentity.create(entry.getKey());

      final Element parentElement;
      boolean isRootAsset = identity.getIdentifier().equals(assetId.getIdentifier());
      if (isRootAsset) {
        parentElement = articleElement;
      } else {
        parentElement = (Element) articleBundle.appendChild(manifest.createElement("object"));
        parentElement.setAttribute("uri", assetId.getKey());
      }

      for (ArticleAsset asset : entry.getValue()) {
        AssetFileIdentity fileId = AssetFileIdentity.create(asset);
        if (isRootAsset && fileId.getFileExtension().equals("XML")) {
          continue; // already created above
        }
        Element assetRepresentation = (Element) parentElement.appendChild(manifest.createElement("representation"));
        assetRepresentation.setAttribute("name", asset.getExtension());
        assetRepresentation.setAttribute("entry", getDummyFileName(asset));
      }
    }

    return manifest;
  }

  public static InputStream buildMockIngestible(InputStream xml, List<ArticleAsset> expectedAssets) throws IOException {
    byte[] xmlData;
{'fraction_non_alphanumeric': 0.10881283690610505, 'fraction_numerical': 0.0045129748025573525, 'mean_word_length': 5.012057272042201, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 26}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1560673', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2091, 'n_words': 420}