<commit_before>// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.notedb;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.server.PatchLineCommentsUtil.setCommentRevId;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_HASHTAGS;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_PATCH_SET;
import static java.util.concurrent.TimeUnit.SECONDS;

import com.google.common.base.MoreObjects;
import com.google.common.base.Splitter;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchLineComment.Status;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.client.StarredChange;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.InternalUser;
import com.google.gerrit.server.PatchLineCommentsUtil;
import com.google.gerrit.server.git.ChainedReceiveCommands;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import com.google.inject.util.Providers;

import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;

import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ChangeRebuilder {
  /**
   * The maximum amount of time between the ReviewDb timestamp of the first and
   * last events batched together into a single NoteDb update.
   * <p>
   * Used to account for the fact that different records with their own
   * timestamps (e.g. {@link PatchSetApproval} and {@link ChangeMessage})
   * historically didn't necessarily use the same timestamp, and tended to call
   * {@code System.currentTimeMillis()} independently.
   */
  static final long MAX_WINDOW_MS = SECONDS.toMillis(3);

  /**
   * The maximum amount of time between two consecutive events to consider them
   * to be in the same batch.
   */
  private static final long MAX_DELTA_MS = SECONDS.toMillis(1);

  private final SchemaFactory<ReviewDb> schemaFactory;
  private final GitRepositoryManager repoManager;
  private final ChangeControl.GenericFactory controlFactory;
  private final IdentifiedUser.GenericFactory userFactory;
  private final InternalUser.Factory internalUserFactory;
  private final PatchListCache patchListCache;
  private final ChangeUpdate.Factory updateFactory;
  private final ChangeDraftUpdate.Factory draftUpdateFactory;
  private final NoteDbUpdateManager.Factory updateManagerFactory;
  private final ChangeNoteUtil changeNoteUtil;

  @Inject
  ChangeRebuilder(SchemaFactory<ReviewDb> schemaFactory,
      GitRepositoryManager repoManager,
      ChangeControl.GenericFactory controlFactory,
      IdentifiedUser.GenericFactory userFactory,
      InternalUser.Factory internalUserFactory,
      PatchListCache patchListCache,
      ChangeUpdate.Factory updateFactory,
      ChangeDraftUpdate.Factory draftUpdateFactory,
      NoteDbUpdateManager.Factory updateManagerFactory,
      ChangeNoteUtil changeNoteUtil) {
    this.schemaFactory = schemaFactory;
    this.repoManager = repoManager;
    this.controlFactory = controlFactory;
    this.userFactory = userFactory;
    this.internalUserFactory = internalUserFactory;
    this.patchListCache = patchListCache;
    this.updateFactory = updateFactory;
    this.draftUpdateFactory = draftUpdateFactory;
    this.updateManagerFactory = updateManagerFactory;
    this.changeNoteUtil = changeNoteUtil;
  }

  public ListenableFuture<?> rebuildAsync(final Change.Id id,
      ListeningExecutorService executor) {
    return executor.submit(new Callable<Void>() {
        @Override
      public Void call() throws Exception {
        try (ReviewDb db = schemaFactory.open()) {
          rebuild(db, id);
        }
        return null;
      }
    });
  }

  public void rebuild(ReviewDb db, Change.Id changeId)
      throws NoSuchChangeException, IOException, OrmException,
      ConfigInvalidException {
    Change change = db.changes().get(changeId);
    if (change == null) {
      return;
    }
    NoteDbUpdateManager manager =
        updateManagerFactory.create(change.getProject());

    // We will rebuild all events, except for draft comments, in buckets based
    // on author and timestamp.
    List<Event> events = Lists.newArrayList();
    Multimap<Account.Id, PatchLineCommentEvent> draftCommentEvents =
        ArrayListMultimap.create();

    Repository changeMetaRepo = manager.getChangeRepo();
    events.addAll(getHashtagsEvents(change, manager));

    // Delete ref only after hashtags have been read
    deleteRef(change, changeMetaRepo, manager.getChangeCommands());

    try (Repository codeRepo = repoManager.openRepository(change.getProject());
        RevWalk codeRw = new RevWalk(codeRepo)) {
      for (PatchSet ps : db.patchSets().byChange(changeId)) {
        events.add(new PatchSetEvent(change, ps, codeRw));
        for (PatchLineComment c : db.patchComments().byPatchSet(ps.getId())) {
          PatchLineCommentEvent e =
              new PatchLineCommentEvent(c, change, ps, patchListCache);
          if (c.getStatus() == Status.PUBLISHED) {
            events.add(e);<commit_msg>ChangeRebuilder: Sort PatchLineCommentEvents<commit_after>// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.notedb;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.server.PatchLineCommentsUtil.setCommentRevId;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_HASHTAGS;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_PATCH_SET;
import static java.util.concurrent.TimeUnit.SECONDS;

import com.google.common.base.MoreObjects;
import com.google.common.base.Splitter;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchLineComment.Status;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.client.StarredChange;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.InternalUser;
import com.google.gerrit.server.PatchLineCommentsUtil;
import com.google.gerrit.server.git.ChainedReceiveCommands;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import com.google.inject.util.Providers;

import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;

import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ChangeRebuilder {
  /**
   * The maximum amount of time between the ReviewDb timestamp of the first and
   * last events batched together into a single NoteDb update.
   * <p>
   * Used to account for the fact that different records with their own
   * timestamps (e.g. {@link PatchSetApproval} and {@link ChangeMessage})
   * historically didn't necessarily use the same timestamp, and tended to call
   * {@code System.currentTimeMillis()} independently.
   */
  static final long MAX_WINDOW_MS = SECONDS.toMillis(3);

  /**
   * The maximum amount of time between two consecutive events to consider them
   * to be in the same batch.
   */
  private static final long MAX_DELTA_MS = SECONDS.toMillis(1);

  private final SchemaFactory<ReviewDb> schemaFactory;
  private final GitRepositoryManager repoManager;
  private final ChangeControl.GenericFactory controlFactory;
  private final IdentifiedUser.GenericFactory userFactory;
  private final InternalUser.Factory internalUserFactory;
  private final PatchListCache patchListCache;
  private final ChangeUpdate.Factory updateFactory;
  private final ChangeDraftUpdate.Factory draftUpdateFactory;
  private final NoteDbUpdateManager.Factory updateManagerFactory;
  private final ChangeNoteUtil changeNoteUtil;

  @Inject
  ChangeRebuilder(SchemaFactory<ReviewDb> schemaFactory,
      GitRepositoryManager repoManager,
      ChangeControl.GenericFactory controlFactory,
      IdentifiedUser.GenericFactory userFactory,
      InternalUser.Factory internalUserFactory,
      PatchListCache patchListCache,
      ChangeUpdate.Factory updateFactory,
      ChangeDraftUpdate.Factory draftUpdateFactory,
      NoteDbUpdateManager.Factory updateManagerFactory,
      ChangeNoteUtil changeNoteUtil) {
    this.schemaFactory = schemaFactory;
    this.repoManager = repoManager;
    this.controlFactory = controlFactory;
    this.userFactory = userFactory;
    this.internalUserFactory = internalUserFactory;
    this.patchListCache = patchListCache;
    this.updateFactory = updateFactory;
    this.draftUpdateFactory = draftUpdateFactory;
    this.updateManagerFactory = updateManagerFactory;
    this.changeNoteUtil = changeNoteUtil;
  }

  public ListenableFuture<?> rebuildAsync(final Change.Id id,
      ListeningExecutorService executor) {
    return executor.submit(new Callable<Void>() {
        @Override
      public Void call() throws Exception {
        try (ReviewDb db = schemaFactory.open()) {
          rebuild(db, id);
        }
        return null;
      }
    });
  }

  public void rebuild(ReviewDb db, Change.Id changeId)
      throws NoSuchChangeException, IOException, OrmException,
      ConfigInvalidException {
    Change change = db.changes().get(changeId);
    if (change == null) {
      return;
    }
    NoteDbUpdateManager manager =
        updateManagerFactory.create(change.getProject());

    // We will rebuild all events, except for draft comments, in buckets based
    // on author and timestamp.
    List<Event> events = Lists.newArrayList();
    Multimap<Account.Id, PatchLineCommentEvent> draftCommentEvents =
        ArrayListMultimap.create();

    Repository changeMetaRepo = manager.getChangeRepo();
    events.addAll(getHashtagsEvents(change, manager));

    // Delete ref only after hashtags have been read
    deleteRef(change, changeMetaRepo, manager.getChangeCommands());

    try (Repository codeRepo = repoManager.openRepository(change.getProject());
        RevWalk codeRw = new RevWalk(codeRepo)) {
      for (PatchSet ps : db.patchSets().byChange(changeId)) {
        events.add(new PatchSetEvent(change, ps, codeRw));
        List<PatchLineComment> comments =
            PatchLineCommentsUtil.PLC_ORDER.sortedCopy(
                db.patchComments().byPatchSet(ps.getId()));
        for (PatchLineComment c : comments) {
          PatchLineCommentEvent e =
              new PatchLineCommentEvent(c, change, ps, patchListCache);
          if (c.getStatus() == Status.PUBLISHED) {
            events.add(e);
{'fraction_non_alphanumeric': 0.0893215576073866, 'fraction_numerical': 0.0013381506757660913, 'mean_word_length': 4.523651145602365, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3113059', 'n_tokens_mistral': 4241, 'n_tokens_neox': 4137, 'n_words': 999}