<commit_before>package org.ambraproject.rhino.service.impl;

import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import org.ambraproject.ApplicationException;
import org.ambraproject.models.Article;
import org.ambraproject.models.ArticleAsset;
import org.ambraproject.models.ArticleRelationship;
import org.ambraproject.models.Category;
import org.ambraproject.rhino.content.xml.ArticleXml;
import org.ambraproject.rhino.content.xml.AssetBuilder;
import org.ambraproject.rhino.content.xml.AssetNodesByDoi;
import org.ambraproject.rhino.content.xml.AssetXml;
import org.ambraproject.rhino.content.xml.ManifestXml;
import org.ambraproject.rhino.content.xml.XmlContentException;
import org.ambraproject.rhino.identity.ArticleIdentity;
import org.ambraproject.rhino.identity.AssetFileIdentity;
import org.ambraproject.rhino.identity.AssetIdentity;
import org.ambraproject.rhino.identity.DoiBasedIdentity;
import org.ambraproject.rhino.rest.RestClientException;
import org.ambraproject.rhino.service.DoiBasedCrudService;
import org.ambraproject.rhino.service.taxonomy.WeightedTerm;
import org.ambraproject.rhino.util.Archive;
import org.ambraproject.rhino.util.HibernateEntityUtil;
import org.ambraproject.service.article.NoSuchArticleIdException;
import org.hibernate.Criteria;
import org.hibernate.FetchMode;
import org.hibernate.Query;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Restrictions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.support.DataAccessUtils;
import org.springframework.http.HttpStatus;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

class LegacyIngestionService {

  private static final Logger log = LoggerFactory.getLogger(LegacyIngestionService.class);

  private final ArticleCrudServiceImpl parentService;

  LegacyIngestionService(ArticleCrudServiceImpl parentService) {
    this.parentService = Preconditions.checkNotNull(parentService);
  }

  /**
   * Query for an article by its identifier.
   *
   * @param id the article's identity
   * @return the article, or {@code null} if not found
   */
  public Article findArticleById(DoiBasedIdentity id) {
    return (Article) DataAccessUtils.uniqueResult((List<?>)
        parentService.hibernateTemplate.findByCriteria(DetachedCriteria
                .forClass(Article.class)
                .add(Restrictions.eq("doi", id.getKey()))
                .setFetchMode("articleType", FetchMode.JOIN)
                .setFetchMode("citedArticles", FetchMode.JOIN)
                .setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY)
        ));
  }

  /**
   * Creates or updates an Article instance based on the given Document.  Does not persist the Article; that is the
   * responsibility of the caller.
   *
   * @param doc           Document describing the article XML
   * @param manifestXml   The manifestXml document
   * @param suppliedId    the indentifier supplied for the article by the external caller, if any
   * @param mode          whether to attempt a create or update
   * @param xmlDataLength the number of bytes in the uploaded XML file
   * @return the created Article
   * @throws IOException
   */

  private Article populateArticleFromXml(ArticleXml xml,
                                         Optional<ArticleIdentity> suppliedId,
                                         DoiBasedCrudService.WriteMode mode) {
    ArticleIdentity doi;
    try {
      doi = xml.readDoi();
    } catch (XmlContentException e) {
      throw ArticleCrudServiceImpl.complainAboutXml(e);
    }

    if (suppliedId.isPresent() && !doi.equals(suppliedId.get())) {
      String message = String.format("Article XML with DOI=\"%s\" uploaded to mismatched address: \"%s\"",
          doi.getIdentifier(), suppliedId.get().getIdentifier());
      throw new RestClientException(message, HttpStatus.BAD_REQUEST);
    }

    Article article = findArticleById(doi);
    final boolean creating = (article == null);
    if ((creating && mode == DoiBasedCrudService.WriteMode.UPDATE_ONLY) || (!creating && mode == DoiBasedCrudService.WriteMode.CREATE_ONLY)) {
      String messageStub = (creating ?
          "Can't update; article does not exist at " : "Can't create; article already exists at ");
      throw new RestClientException(messageStub + doi.getIdentifier(), HttpStatus.METHOD_NOT_ALLOWED);
    }
    if (creating) {
      article = new Article();
      article.setDoi(doi.getKey());
    }

    try {
      article = xml.build(article);
    } catch (XmlContentException e) {
      throw ArticleCrudServiceImpl.complainAboutXml(e);
    }

    // TODO: If an article should have multiple journals, how does it get them?
    article.setJournals(Sets.newHashSet(parentService.getPublicationJournal(article)));
    initializeCategories(article, xml.getDocument());
    populateRelatedArticles(article, xml);

    return article;
  }

  private List<ArticleAsset> createAssets(List<AssetBuilder> assetBuilders, Archive archive, ManifestXml manifest) {
    ImmutableMap<AssetIdentity, AssetBuilder> assetMap = Maps.uniqueIndex(assetBuilders,
        assetBuilder -> AssetIdentity.create(assetBuilder.getDoi()));
    List<ArticleAsset> assets = new ArrayList<>();

    for (ManifestXml.Asset manifestAsset : manifest.parse()) {
      AssetIdentity assetIdentity = AssetIdentity.create(manifestAsset.getUri());
      AssetBuilder assetBuilder = assetMap.get(assetIdentity);
      if (assetBuilder == null) {
        // TODO: Handle manuscript, striking images, etc. Convert this to a hard exception when all cases are handled.
        log.error("Asset in manifest not matched to manuscript: {}", assetIdentity);
        continue;
      }

      for (ManifestXml.Representation representation : manifestAsset.getRepresentations()) {
        ArticleAsset asset = new ArticleAsset();
        asset.setDoi(assetIdentity.getKey());

        asset.setTitle(assetBuilder.getTitle());
        asset.setDescription(assetBuilder.getDescription());
        asset.setContextElement(assetBuilder.getContextElement());

        asset.setExtension(representation.getName());
        asset.setContentType(AssetFileIdentity.create(assetIdentity.getIdentifier(), representation.getName())
            .inferContentType().toString());

        assets.add(asset);
      }
    }
    return assets;
  }

  /**
   * Saves the hibernate entity representing an article.
   *
   * @param article the new or updated Article instance to save
   */
  private void saveArticleToHibernate(Article article) {
    if (article.getID() == null) {
      parentService.hibernateTemplate.save(article);
    } else {
      parentService.hibernateTemplate.update(article);
    }<commit_msg>DPRO-2107: Rearrange createAssets to prevent spurious errors<commit_after>package org.ambraproject.rhino.service.impl;

import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import org.ambraproject.ApplicationException;
import org.ambraproject.models.Article;
import org.ambraproject.models.ArticleAsset;
import org.ambraproject.models.ArticleRelationship;
import org.ambraproject.models.Category;
import org.ambraproject.rhino.content.xml.ArticleXml;
import org.ambraproject.rhino.content.xml.AssetBuilder;
import org.ambraproject.rhino.content.xml.AssetNodesByDoi;
import org.ambraproject.rhino.content.xml.AssetXml;
import org.ambraproject.rhino.content.xml.ManifestXml;
import org.ambraproject.rhino.content.xml.XmlContentException;
import org.ambraproject.rhino.identity.ArticleIdentity;
import org.ambraproject.rhino.identity.AssetFileIdentity;
import org.ambraproject.rhino.identity.AssetIdentity;
import org.ambraproject.rhino.identity.DoiBasedIdentity;
import org.ambraproject.rhino.rest.RestClientException;
import org.ambraproject.rhino.service.DoiBasedCrudService;
import org.ambraproject.rhino.service.taxonomy.WeightedTerm;
import org.ambraproject.rhino.util.Archive;
import org.ambraproject.rhino.util.HibernateEntityUtil;
import org.ambraproject.service.article.NoSuchArticleIdException;
import org.hibernate.Criteria;
import org.hibernate.FetchMode;
import org.hibernate.Query;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Restrictions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.support.DataAccessUtils;
import org.springframework.http.HttpStatus;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

class LegacyIngestionService {

  private static final Logger log = LoggerFactory.getLogger(LegacyIngestionService.class);

  private final ArticleCrudServiceImpl parentService;

  LegacyIngestionService(ArticleCrudServiceImpl parentService) {
    this.parentService = Preconditions.checkNotNull(parentService);
  }

  /**
   * Query for an article by its identifier.
   *
   * @param id the article's identity
   * @return the article, or {@code null} if not found
   */
  public Article findArticleById(DoiBasedIdentity id) {
    return (Article) DataAccessUtils.uniqueResult((List<?>)
        parentService.hibernateTemplate.findByCriteria(DetachedCriteria
                .forClass(Article.class)
                .add(Restrictions.eq("doi", id.getKey()))
                .setFetchMode("articleType", FetchMode.JOIN)
                .setFetchMode("citedArticles", FetchMode.JOIN)
                .setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY)
        ));
  }

  /**
   * Creates or updates an Article instance based on the given Document.  Does not persist the Article; that is the
   * responsibility of the caller.
   *
   * @param doc           Document describing the article XML
   * @param manifestXml   The manifestXml document
   * @param suppliedId    the indentifier supplied for the article by the external caller, if any
   * @param mode          whether to attempt a create or update
   * @param xmlDataLength the number of bytes in the uploaded XML file
   * @return the created Article
   * @throws IOException
   */

  private Article populateArticleFromXml(ArticleXml xml,
                                         Optional<ArticleIdentity> suppliedId,
                                         DoiBasedCrudService.WriteMode mode) {
    ArticleIdentity doi;
    try {
      doi = xml.readDoi();
    } catch (XmlContentException e) {
      throw ArticleCrudServiceImpl.complainAboutXml(e);
    }

    if (suppliedId.isPresent() && !doi.equals(suppliedId.get())) {
      String message = String.format("Article XML with DOI=\"%s\" uploaded to mismatched address: \"%s\"",
          doi.getIdentifier(), suppliedId.get().getIdentifier());
      throw new RestClientException(message, HttpStatus.BAD_REQUEST);
    }

    Article article = findArticleById(doi);
    final boolean creating = (article == null);
    if ((creating && mode == DoiBasedCrudService.WriteMode.UPDATE_ONLY) || (!creating && mode == DoiBasedCrudService.WriteMode.CREATE_ONLY)) {
      String messageStub = (creating ?
          "Can't update; article does not exist at " : "Can't create; article already exists at ");
      throw new RestClientException(messageStub + doi.getIdentifier(), HttpStatus.METHOD_NOT_ALLOWED);
    }
    if (creating) {
      article = new Article();
      article.setDoi(doi.getKey());
    }

    try {
      article = xml.build(article);
    } catch (XmlContentException e) {
      throw ArticleCrudServiceImpl.complainAboutXml(e);
    }

    // TODO: If an article should have multiple journals, how does it get them?
    article.setJournals(Sets.newHashSet(parentService.getPublicationJournal(article)));
    initializeCategories(article, xml.getDocument());
    populateRelatedArticles(article, xml);

    return article;
  }

  private List<ArticleAsset> createAssets(List<AssetBuilder> assetBuilders, Archive archive, ManifestXml manifest) {
    ImmutableMap<AssetIdentity, ManifestXml.Asset> manifestAssets = Maps.uniqueIndex(manifest.parse(),
        manifestAsset -> AssetIdentity.create(manifestAsset.getUri()));

    List<ArticleAsset> assets = new ArrayList<>();
    for (AssetBuilder assetBuilder : assetBuilders) {
      AssetIdentity assetIdentity = AssetIdentity.create(assetBuilder.getDoi());
      ManifestXml.Asset manifestAsset = manifestAssets.get(assetIdentity);
      if (manifestAsset == null) {
        throw new RestClientException("Asset in manuscript not matched to manifest: " + assetIdentity,
            HttpStatus.BAD_REQUEST);
      }

      for (ManifestXml.Representation representation : manifestAsset.getRepresentations()) {
        ArticleAsset asset = new ArticleAsset();
        asset.setDoi(assetIdentity.getKey());

        asset.setTitle(assetBuilder.getTitle());
        asset.setDescription(assetBuilder.getDescription());
        asset.setContextElement(assetBuilder.getContextElement());

        asset.setExtension(representation.getName());
        asset.setContentType(AssetFileIdentity.create(assetIdentity.getIdentifier(), representation.getName())
            .inferContentType().toString());

        assets.add(asset);
      }
    }
    // TODO: Handle manuscript, striking images, etc.
    return assets;
  }

  /**
   * Saves the hibernate entity representing an article.
   *
   * @param article the new or updated Article instance to save
   */
  private void saveArticleToHibernate(Article article) {
    if (article.getID() == null) {
      parentService.hibernateTemplate.save(article);
    } else {
      parentService.hibernateTemplate.update(article);
    }
{'fraction_non_alphanumeric': 0.09885172241637544, 'fraction_numerical': 0.000873689465801298, 'mean_word_length': 4.12635956493922, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 40}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3143417', 'n_tokens_mistral': 4627, 'n_tokens_neox': 4406, 'n_words': 955}