<commit_before>/*
 * Copyright 2015-2021, <NAME>, <NAME>, <NAME>
 *                 Iowa State University of Science and Technology
 *                 and University of Nebraska Board of Regents
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package boa.datagen;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import boa.datagen.forges.github.RepoMetadata;
import boa.datagen.forges.github.RepositoryCloner;
import boa.datagen.scm.AbstractConnector;
import boa.datagen.scm.GitConnector;
import boa.datagen.util.FileIO;
import boa.datagen.util.Properties;
import boa.types.Code.CodeRepository;
import boa.types.Code.Revision;
import boa.types.Toplevel.Project;

public class SeqRepoImporter {
	private final static boolean debug = Properties.getBoolean("debug", DefaultProperties.DEBUG);
	private final static boolean cache = Properties.getBoolean("cache", DefaultProperties.CACHE);
	private final static long skips = Long.parseLong(Properties.getProperty("skip", DefaultProperties.SKIPS)) + 1;
	private final static long offset = Long.parseLong(Properties.getProperty("offset", DefaultProperties.OFFSET));

	private final static File gitRootPath = new File(Properties.getProperty("gh.svn.path", DefaultProperties.GH_GIT_PATH));
	final static String jsonPath = Properties.getProperty("gh.json.path", DefaultProperties.GH_JSON_PATH);
	final static String jsonCachePath = Properties.getProperty("output.path", DefaultProperties.OUTPUT);

	private final static HashSet<String> processedProjectIds = new HashSet<String>();

	private static Configuration conf = null;
	private static FileSystem fileSystem = null;
	private static String base = null;
	private static boolean done = false;

	private final static int POOL_SIZE = Integer.parseInt(Properties.getProperty("num.threads", DefaultProperties.NUM_THREADS));
	private final static int MAX_SIZE_FOR_PROJECT_WITH_COMMITS = Integer.valueOf(DefaultProperties.MAX_SIZE_FOR_PROJECT_WITH_COMMITS);
	private final static long TOTAL_MAX_PROJECTS = Long.parseLong(DefaultProperties.TOTAL_MAX_PROJECTS);
	private final static int MAX_PROJECTS = Integer.parseInt(DefaultProperties.MAX_PROJECTS);
	private final static boolean STORE_COMMITS = DefaultProperties.STORE_COMMITS;
	private static final ImportTask[] workers = new ImportTask[POOL_SIZE];
	private static int counter = 0;

	public static void main(final String[] args) throws IOException, InterruptedException {
		conf = new Configuration();
		fileSystem = FileSystem.get(conf);
		base = Properties.getProperty("output.path", DefaultProperties.OUTPUT);

		getProcessedProjects();

		// assign each thread with a worker
		final Thread[] threads = new Thread[POOL_SIZE];
		for (int i = 0; i < POOL_SIZE; i++) {
			workers[i] = new ImportTask(i);
			threads[i] = new Thread(workers[i]);
			threads[i].start();
			Thread.sleep(10);
		}

		processJSONdir(new File(jsonPath));

		for (int j = 0; j < POOL_SIZE; j++)
			while (workers[j].isAssigned())
				Thread.sleep(100);
		setDone(true);

		// wait for workers to close writers and finish
		for (final Thread thread : threads)
			while (thread.isAlive())
				Thread.sleep(1000);
	}

	static void processJSONdir(final File dir) {
		if (getDone() || !dir.isDirectory())
			return;

		for (final File file : dir.listFiles()) {
			if (file.isDirectory())
				processJSONdir(file);
			else if (file.getName().endsWith(".json"))
				processJSON(file);
		}
	}

	static void processJSON(final File file) {
		final String content = FileIO.readFileContents(file);
		final Gson parser = new Gson();

		JsonArray repoArray = null;
		try {
			repoArray = parser.fromJson(content, JsonElement.class).getAsJsonArray();
		} catch (final Exception e) {
			System.err.println("Error proccessing page: " + file.getPath());
			e.printStackTrace();
			return;
		}
		for (int i = 0; !getDone() && i < repoArray.size(); i++) {
			if (counter >= TOTAL_MAX_PROJECTS) {
				setDone(true);
				return;
			}
			try {
				final JsonObject rp = repoArray.get(i).getAsJsonObject();
				final RepoMetadata repo = new RepoMetadata(rp);
				if (counter >= offset && (skips <= 0 || counter % skips == 0)) {
					if (repo.id != null && repo.name != null && !processedProjectIds.contains(repo.id)) {
						final Project project = repo.toBoaMetaDataProtobuf(); // current project instance only contains metadata

						// System.out.println(jRepo.toString());
						boolean assigned = false;
						while (!getDone() && !assigned) {
							for (int j = 0; !getDone() && j < POOL_SIZE; j++) {
								if (workers[j].isReady() && !workers[j].isAssigned()) {
									workers[j].setProject(project);
									workers[j].setAssigned(true);
									assigned = true;
									break;
								}
							}
							// Thread.sleep(100);
						}<commit_msg>fix issue with datagen when using -skip and -offset together<commit_after>/*
 * Copyright 2015-2021, <NAME>, <NAME>, <NAME>
 *                 Iowa State University of Science and Technology
 *                 and University of Nebraska Board of Regents
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package boa.datagen;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import boa.datagen.forges.github.RepoMetadata;
import boa.datagen.forges.github.RepositoryCloner;
import boa.datagen.scm.AbstractConnector;
import boa.datagen.scm.GitConnector;
import boa.datagen.util.FileIO;
import boa.datagen.util.Properties;
import boa.types.Code.CodeRepository;
import boa.types.Code.Revision;
import boa.types.Toplevel.Project;

public class SeqRepoImporter {
	private final static boolean debug = Properties.getBoolean("debug", DefaultProperties.DEBUG);
	private final static boolean cache = Properties.getBoolean("cache", DefaultProperties.CACHE);
	private final static long skips = Long.parseLong(Properties.getProperty("skip", DefaultProperties.SKIPS)) + 1;
	private final static long offset = Long.parseLong(Properties.getProperty("offset", DefaultProperties.OFFSET));

	private final static File gitRootPath = new File(Properties.getProperty("gh.svn.path", DefaultProperties.GH_GIT_PATH));
	final static String jsonPath = Properties.getProperty("gh.json.path", DefaultProperties.GH_JSON_PATH);
	final static String jsonCachePath = Properties.getProperty("output.path", DefaultProperties.OUTPUT);

	private final static HashSet<String> processedProjectIds = new HashSet<String>();

	private static Configuration conf = null;
	private static FileSystem fileSystem = null;
	private static String base = null;
	private static boolean done = false;

	private final static int POOL_SIZE = Integer.parseInt(Properties.getProperty("num.threads", DefaultProperties.NUM_THREADS));
	private final static int MAX_SIZE_FOR_PROJECT_WITH_COMMITS = Integer.valueOf(DefaultProperties.MAX_SIZE_FOR_PROJECT_WITH_COMMITS);
	private final static long TOTAL_MAX_PROJECTS = Long.parseLong(DefaultProperties.TOTAL_MAX_PROJECTS);
	private final static int MAX_PROJECTS = Integer.parseInt(DefaultProperties.MAX_PROJECTS);
	private final static boolean STORE_COMMITS = DefaultProperties.STORE_COMMITS;
	private static final ImportTask[] workers = new ImportTask[POOL_SIZE];
	private static int counter = 0;

	public static void main(final String[] args) throws IOException, InterruptedException {
		conf = new Configuration();
		fileSystem = FileSystem.get(conf);
		base = Properties.getProperty("output.path", DefaultProperties.OUTPUT);

		getProcessedProjects();

		// assign each thread with a worker
		final Thread[] threads = new Thread[POOL_SIZE];
		for (int i = 0; i < POOL_SIZE; i++) {
			workers[i] = new ImportTask(i);
			threads[i] = new Thread(workers[i]);
			threads[i].start();
			Thread.sleep(10);
		}

		processJSONdir(new File(jsonPath));

		for (int j = 0; j < POOL_SIZE; j++)
			while (workers[j].isAssigned())
				Thread.sleep(100);
		setDone(true);

		// wait for workers to close writers and finish
		for (final Thread thread : threads)
			while (thread.isAlive())
				Thread.sleep(1000);
	}

	static void processJSONdir(final File dir) {
		if (getDone() || !dir.isDirectory())
			return;

		for (final File file : dir.listFiles()) {
			if (file.isDirectory())
				processJSONdir(file);
			else if (file.getName().endsWith(".json"))
				processJSON(file);
		}
	}

	static void processJSON(final File file) {
		final String content = FileIO.readFileContents(file);
		final Gson parser = new Gson();

		JsonArray repoArray = null;
		try {
			repoArray = parser.fromJson(content, JsonElement.class).getAsJsonArray();
		} catch (final Exception e) {
			System.err.println("Error proccessing page: " + file.getPath());
			e.printStackTrace();
			return;
		}
		for (int i = 0; !getDone() && i < repoArray.size(); i++) {
			if (counter >= TOTAL_MAX_PROJECTS) {
				setDone(true);
				return;
			}
			try {
				final JsonObject rp = repoArray.get(i).getAsJsonObject();
				final RepoMetadata repo = new RepoMetadata(rp);
				if (counter >= offset && (skips <= 0 || (counter - offset) % skips == 0)) {
					if (repo.id != null && repo.name != null && !processedProjectIds.contains(repo.id)) {
						final Project project = repo.toBoaMetaDataProtobuf(); // current project instance only contains metadata

						// System.out.println(jRepo.toString());
						boolean assigned = false;
						while (!getDone() && !assigned) {
							for (int j = 0; !getDone() && j < POOL_SIZE; j++) {
								if (workers[j].isReady() && !workers[j].isAssigned()) {
									workers[j].setProject(project);
									workers[j].setAssigned(true);
									assigned = true;
									break;
								}
							}
							// Thread.sleep(100);
						}
{'fraction_non_alphanumeric': 0.11409796893667862, 'fraction_numerical': 0.00546168288103772, 'mean_word_length': 4.3099229723606705, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1447148', 'n_tokens_mistral': 3847, 'n_tokens_neox': 3427, 'n_words': 950}