<commit_before>/*
 * Copyright 2015, <NAME>, <NAME>, <NAME>
 *                 and Iowa State University of Science and Technology
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package boa.datagen.scm;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.diff.RawTextComparator;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.io.NullOutputStream;

import boa.types.Diff.ChangedFile;
import boa.types.Diff.ChangedFile.FileKind;
import boa.types.Shared.ChangeKind;

/**
 * Concrete implementation of a commit for Git.
 *
 * @author rdyer
 * @author josephb
 */
public class GitCommit extends AbstractCommit {
	// the repository the commit lives in - should already be connected!
	private Repository repository;
	private RevWalk revwalk;
	private HashMap<String, ObjectId> filePathGitObjectIds = new HashMap<String, ObjectId>();

	public GitCommit(final GitConnector cnn, final Repository repository, final RevWalk revwalk) {
		super(cnn);
		this.repository = repository;
		this.revwalk = revwalk;
	}

	@Override
	/** {@inheritDoc} */
	protected String getFileContents(final String path) {
		try {
			ObjectId fileid = filePathGitObjectIds.get(path);

			try {
				buffer.reset();
				buffer.write(repository.open(fileid, Constants.OBJ_BLOB).getCachedBytes());
			} catch (final IOException e) {
				if (debug)
					System.err.println("Git Error getting contents for '" + path + "' at revision " + id + ": " + e.getMessage());
			}
			return buffer.toString();
		} catch (final Exception e) {
			if (debug)
				System.err.println("Git Error getting contents for '" + path + "' at revision " + id + ": " + e.getMessage());
			e.printStackTrace();
		}
		return "";
	}

	void getChangeFiles(RevCommit rc) {
		if (rc.getParentCount() == 0) {
			TreeWalk tw = new TreeWalk(repository);
			tw.reset();
			try {
				tw.addTree(rc.getTree());
				tw.setRecursive(true);
				while (tw.next()) {
					if (!tw.isSubtree()) {
						String path = tw.getPathString();
						ChangedFile.Builder cfb = ChangedFile.newBuilder();
						cfb.setChange(ChangeKind.ADDED);
						cfb.setName(path);
						cfb.setKind(FileKind.OTHER);
						cfb.setKey(-1);
						fileNameIndices.put(path, changedFiles.size());
						changedFiles.add(cfb);
					}
				}
			} catch (IOException e) {
				System.err.println(e.getMessage());
			}
			tw.close();
		} else {
			parentIndices = new int[rc.getParentCount()];
			for (int i = 0; i < rc.getParentCount(); i++) {
				try {
					getChangeFiles(revwalk.parseCommit(rc.getParent(i).getId()), rc);
				} catch (IOException e) {
					if (debug)
						System.err.println("Git Error parsing parent commit. " + e.getMessage());
				}
				parentIndices[i] = connector.revisionMap.get(rc.getParent(i).getName());
			}
		}
	}

	private void getChangeFiles(final RevCommit parent, final RevCommit rc) {
		final DiffFormatter df = new DiffFormatter(NullOutputStream.INSTANCE);<commit_msg>cache object ids for first commit also<commit_after>/*
 * Copyright 2015, <NAME>, <NAME>, <NAME>
 *                 and Iowa State University of Science and Technology
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package boa.datagen.scm;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.diff.RawTextComparator;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.io.NullOutputStream;

import boa.types.Diff.ChangedFile;
import boa.types.Diff.ChangedFile.FileKind;
import boa.types.Shared.ChangeKind;

/**
 * Concrete implementation of a commit for Git.
 *
 * @author rdyer
 * @author josephb
 */
public class GitCommit extends AbstractCommit {
	// the repository the commit lives in - should already be connected!
	private Repository repository;
	private RevWalk revwalk;
	private HashMap<String, ObjectId> filePathGitObjectIds = new HashMap<String, ObjectId>();

	public GitCommit(final GitConnector cnn, final Repository repository, final RevWalk revwalk) {
		super(cnn);
		this.repository = repository;
		this.revwalk = revwalk;
	}

	@Override
	/** {@inheritDoc} */
	protected String getFileContents(final String path) {
		ObjectId fileid = filePathGitObjectIds.get(path);
		try {
			buffer.reset();
			buffer.write(repository.open(fileid, Constants.OBJ_BLOB).getCachedBytes());
		} catch (final IOException e) {
			if (debug)
				System.err.println("Git Error getting contents for '" + path + "' at revision " + id + ": " + e.getMessage());
		}
		return buffer.toString();
	}

	void getChangeFiles(RevCommit rc) {
		if (rc.getParentCount() == 0) {
			TreeWalk tw = new TreeWalk(repository);
			tw.reset();
			try {
				tw.addTree(rc.getTree());
				tw.setRecursive(true);
				while (tw.next()) {
					if (!tw.isSubtree()) {
						String path = tw.getPathString();
						ChangedFile.Builder cfb = ChangedFile.newBuilder();
						cfb.setChange(ChangeKind.ADDED);
						cfb.setName(path);
						cfb.setKind(FileKind.OTHER);
						cfb.setKey(-1);
						fileNameIndices.put(path, changedFiles.size());
						changedFiles.add(cfb);
						filePathGitObjectIds.put(path, tw.getObjectId(0));
					}
				}
			} catch (IOException e) {
				System.err.println(e.getMessage());
			}
			tw.close();
		} else {
			parentIndices = new int[rc.getParentCount()];
			for (int i = 0; i < rc.getParentCount(); i++) {
				try {
					getChangeFiles(revwalk.parseCommit(rc.getParent(i).getId()), rc);
				} catch (IOException e) {
					if (debug)
						System.err.println("Git Error parsing parent commit. " + e.getMessage());
				}
				parentIndices[i] = connector.revisionMap.get(rc.getParent(i).getName());
			}
		}
	}

	private void getChangeFiles(final RevCommit parent, final RevCommit rc) {
		final DiffFormatter df = new DiffFormatter(NullOutputStream.INSTANCE);
{'fraction_non_alphanumeric': 0.12075226977950713, 'fraction_numerical': 0.0029831387808041503, 'mean_word_length': 3.8588531821045997, 'pattern_counts': {'":': 3, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3855861', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2377, 'n_words': 614}