<commit_before>// Copyright 2008 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.codereview.manager.unpack;

import com.google.codereview.internal.NextReceivedBundle.BundleSegmentRequest;
import com.google.codereview.internal.NextReceivedBundle.BundleSegmentResponse;
import com.google.codereview.internal.NextReceivedBundle.NextReceivedBundleResponse;
import com.google.codereview.internal.SubmitChange.SubmitChangeRequest;
import com.google.codereview.internal.SubmitChange.SubmitChangeResponse;
import com.google.codereview.internal.UpdateReceivedBundle.UpdateReceivedBundleRequest;
import com.google.codereview.internal.UpdateReceivedBundle.UpdateReceivedBundleRequest.CodeType;
import com.google.codereview.manager.Backend;
import com.google.codereview.manager.InvalidRepositoryException;
import com.google.codereview.rpc.SimpleController;
import com.google.codereview.util.GitMetaUtil;
import com.google.codereview.util.MutableBoolean;
import com.google.protobuf.RpcCallback;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.spearce.jgit.errors.MissingBundlePrerequisiteException;
import org.spearce.jgit.lib.NullProgressMonitor;
import org.spearce.jgit.lib.ObjectId;
import org.spearce.jgit.lib.Ref;
import org.spearce.jgit.lib.RefUpdate;
import org.spearce.jgit.lib.Repository;
import org.spearce.jgit.revwalk.RevCommit;
import org.spearce.jgit.revwalk.RevSort;
import org.spearce.jgit.revwalk.RevWalk;
import org.spearce.jgit.transport.FetchConnection;
import org.spearce.jgit.transport.Transport;
import org.spearce.jgit.transport.TransportBundleStream;
import org.spearce.jgit.transport.URIish;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

/** Unpacks a bundle and imports the commits into the code review system. */
class UnpackBundleOp {
  private static final Log LOG = LogFactory.getLog(UnpackBundleOp.class);

  private static String refOf(final int changeId, final int patchsetId) {
    final StringBuilder b = new StringBuilder();
    final int dh = changeId % 100;
    b.append("refs/changes/");
    if (dh < 10) {
      b.append('0');
    }
    b.append(dh);
    b.append('/');
    b.append(changeId);
    b.append('/');
    b.append(patchsetId);
    return b.toString();
  }

  private final Backend server;
  private final NextReceivedBundleResponse in;
  private Repository db;
  private ObjectId tip;
  private int changeId;
  private int patchsetId;
  private String patchsetKey;

  UnpackBundleOp(final Backend be, final NextReceivedBundleResponse bundleInfo) {
    server = be;
    in = bundleInfo;
  }

  UpdateReceivedBundleRequest unpack() {
    final UpdateReceivedBundleRequest.Builder update;
    update = UpdateReceivedBundleRequest.newBuilder();
    update.setBundleKey(in.getBundleKey());
    try {
      unpackImpl();
      update.setStatusCode(CodeType.UNPACKED_OK);
    } catch (UnpackException ue) {
      update.setStatusCode(ue.status);
      update.setErrorDetails(ue.details);
      LOG.error("Unpacking bundle " + in.getBundleKey() + " failed.", ue);
    }
    return update.build();
  }

  private void unpackImpl() throws UnpackException {
    LOG.debug("Unpacking bundle " + in.getBundleKey());
    openRepository();
    unpackTip();
    createChanges(newCommits());
  }

  private void openRepository() throws UnpackException {
    try {
      db = server.getRepositoryCache().get(in.getDestProject());
    } catch (InvalidRepositoryException notGit) {
      final String m = "Repository \"" + in.getDestProject() + "\" unknown.";
      throw new UnpackException(CodeType.UNKNOWN_PROJECT, m, notGit);
    }
  }

  private void unpackTip() throws UnpackException {
    final Transport bundleTransport = openBundle();
    try {
      final FetchConnection fc = bundleTransport.openFetch();
      if (fc.getRefs().size() > 1) {
        final String m = "Bundle contains more than one head";
        throw new UnpackException(CodeType.INVALID_BUNDLE, m);
      } else if (fc.getRefs().size() == 0) {
        final String m = "Bundle contains no heads";
        throw new UnpackException(CodeType.INVALID_BUNDLE, m);
      }

      fc.fetch(NullProgressMonitor.INSTANCE, fc.getRefs());
      tip = fc.getRefs().iterator().next().getObjectId();
      LOG.debug("Unpacked " + tip.name() + " from " + in.getBundleKey());
    } catch (MissingBundlePrerequisiteException e) {
      throw new UnpackException(CodeType.MISSING_BASE, e.getMessage(), e);
    } catch (IOException readError) {
      final String m = "Processing the bundle stream failed";
      throw new UnpackException(CodeType.INVALID_BUNDLE, m, readError);
    } finally {
      bundleTransport.close();
    }
  }

  private List<RevCommit> newCommits() throws UnpackException {
    final RevWalk rw = new RevWalk(db);
    rw.sort(RevSort.REVERSE, true);
    rw.sort(RevSort.TOPO, true);

    try {
      rw.markStart(rw.parseCommit(tip));
    } catch (IOException e) {
      final String m = "Chain " + tip.name() + " is corrupt";
      throw new UnpackException(CodeType.INVALID_BUNDLE, m, e);
    }

    for (final Ref r : db.getAllRefs().values()) {
      try {
        rw.markUninteresting(rw.parseCommit(r.getObjectId()));
      } catch (IOException err) {
        final String m = "Local ref is invalid";
        throw new UnpackException(CodeType.SUSPEND_BUNDLE, m, err);
      }<commit_msg>Correctly handle linux-2.6.git's 'weird' v2.6.11-tree tag<commit_after>// Copyright 2008 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.codereview.manager.unpack;

import com.google.codereview.internal.NextReceivedBundle.BundleSegmentRequest;
import com.google.codereview.internal.NextReceivedBundle.BundleSegmentResponse;
import com.google.codereview.internal.NextReceivedBundle.NextReceivedBundleResponse;
import com.google.codereview.internal.SubmitChange.SubmitChangeRequest;
import com.google.codereview.internal.SubmitChange.SubmitChangeResponse;
import com.google.codereview.internal.UpdateReceivedBundle.UpdateReceivedBundleRequest;
import com.google.codereview.internal.UpdateReceivedBundle.UpdateReceivedBundleRequest.CodeType;
import com.google.codereview.manager.Backend;
import com.google.codereview.manager.InvalidRepositoryException;
import com.google.codereview.rpc.SimpleController;
import com.google.codereview.util.GitMetaUtil;
import com.google.codereview.util.MutableBoolean;
import com.google.protobuf.RpcCallback;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.spearce.jgit.errors.IncorrectObjectTypeException;
import org.spearce.jgit.errors.MissingBundlePrerequisiteException;
import org.spearce.jgit.lib.NullProgressMonitor;
import org.spearce.jgit.lib.ObjectId;
import org.spearce.jgit.lib.Ref;
import org.spearce.jgit.lib.RefUpdate;
import org.spearce.jgit.lib.Repository;
import org.spearce.jgit.revwalk.RevCommit;
import org.spearce.jgit.revwalk.RevSort;
import org.spearce.jgit.revwalk.RevWalk;
import org.spearce.jgit.transport.FetchConnection;
import org.spearce.jgit.transport.Transport;
import org.spearce.jgit.transport.TransportBundleStream;
import org.spearce.jgit.transport.URIish;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

/** Unpacks a bundle and imports the commits into the code review system. */
class UnpackBundleOp {
  private static final Log LOG = LogFactory.getLog(UnpackBundleOp.class);

  private static String refOf(final int changeId, final int patchsetId) {
    final StringBuilder b = new StringBuilder();
    final int dh = changeId % 100;
    b.append("refs/changes/");
    if (dh < 10) {
      b.append('0');
    }
    b.append(dh);
    b.append('/');
    b.append(changeId);
    b.append('/');
    b.append(patchsetId);
    return b.toString();
  }

  private final Backend server;
  private final NextReceivedBundleResponse in;
  private Repository db;
  private ObjectId tip;
  private int changeId;
  private int patchsetId;
  private String patchsetKey;

  UnpackBundleOp(final Backend be, final NextReceivedBundleResponse bundleInfo) {
    server = be;
    in = bundleInfo;
  }

  UpdateReceivedBundleRequest unpack() {
    final UpdateReceivedBundleRequest.Builder update;
    update = UpdateReceivedBundleRequest.newBuilder();
    update.setBundleKey(in.getBundleKey());
    try {
      unpackImpl();
      update.setStatusCode(CodeType.UNPACKED_OK);
    } catch (UnpackException ue) {
      update.setStatusCode(ue.status);
      update.setErrorDetails(ue.details);
      LOG.error("Unpacking bundle " + in.getBundleKey() + " failed.", ue);
    }
    return update.build();
  }

  private void unpackImpl() throws UnpackException {
    LOG.debug("Unpacking bundle " + in.getBundleKey());
    openRepository();
    unpackTip();
    createChanges(newCommits());
  }

  private void openRepository() throws UnpackException {
    try {
      db = server.getRepositoryCache().get(in.getDestProject());
    } catch (InvalidRepositoryException notGit) {
      final String m = "Repository \"" + in.getDestProject() + "\" unknown.";
      throw new UnpackException(CodeType.UNKNOWN_PROJECT, m, notGit);
    }
  }

  private void unpackTip() throws UnpackException {
    final Transport bundleTransport = openBundle();
    try {
      final FetchConnection fc = bundleTransport.openFetch();
      if (fc.getRefs().size() > 1) {
        final String m = "Bundle contains more than one head";
        throw new UnpackException(CodeType.INVALID_BUNDLE, m);
      } else if (fc.getRefs().size() == 0) {
        final String m = "Bundle contains no heads";
        throw new UnpackException(CodeType.INVALID_BUNDLE, m);
      }

      fc.fetch(NullProgressMonitor.INSTANCE, fc.getRefs());
      tip = fc.getRefs().iterator().next().getObjectId();
      LOG.debug("Unpacked " + tip.name() + " from " + in.getBundleKey());
    } catch (MissingBundlePrerequisiteException e) {
      throw new UnpackException(CodeType.MISSING_BASE, e.getMessage(), e);
    } catch (IOException readError) {
      final String m = "Processing the bundle stream failed";
      throw new UnpackException(CodeType.INVALID_BUNDLE, m, readError);
    } finally {
      bundleTransport.close();
    }
  }

  private List<RevCommit> newCommits() throws UnpackException {
    final RevWalk rw = new RevWalk(db);
    rw.sort(RevSort.REVERSE, true);
    rw.sort(RevSort.TOPO, true);

    try {
      rw.markStart(rw.parseCommit(tip));
    } catch (IOException e) {
      final String m = "Chain " + tip.name() + " is corrupt";
      throw new UnpackException(CodeType.INVALID_BUNDLE, m, e);
    }

    for (final Ref r : db.getAllRefs().values()) {
      try {
        rw.markUninteresting(rw.parseCommit(r.getObjectId()));
      } catch (IncorrectObjectTypeException notCommit) {
        // These happen in some repositories like linux-2.6.git where
        // there is an annotated tag pointing at a tree, or in git.git
        // where there is an annotated tag pointing at a blob.
        //
        continue;
      } catch (IOException err) {
        final String m = "Local ref is invalid";
        throw new UnpackException(CodeType.SUSPEND_BUNDLE, m, err);
      }
{'fraction_non_alphanumeric': 0.10883524343656019, 'fraction_numerical': 0.0032512395350727465, 'mean_word_length': 3.9000398247710075, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2561544', 'n_tokens_mistral': 3692, 'n_tokens_neox': 3600, 'n_words': 883}