<commit_before>/*
 * Copyright (c) 2014. <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.jordanwilliams.heftydb.test.endurance;

import com.jordanwilliams.heftydb.db.DB;
import com.jordanwilliams.heftydb.db.HeftyDB;
import com.jordanwilliams.heftydb.db.Record;
import com.jordanwilliams.heftydb.db.Snapshot;
import com.jordanwilliams.heftydb.test.generator.ConfigGenerator;
import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator;
import com.jordanwilliams.heftydb.test.helper.StopWatch;
import com.jordanwilliams.heftydb.test.helper.TestFileHelper;
import com.jordanwilliams.heftydb.util.ByteBuffers;

import java.util.Iterator;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

public class EnduranceTest {

    private static final int THREAD_COUNT = 16;
    private static final int RUNTIME_MINUTES = 30;

    public static void main(String[] args) throws Exception {
        TestFileHelper.createTestDirectory();
        TestFileHelper.cleanUpTestFiles();

        final AtomicLong maxSnapshotId = new AtomicLong();
        final AtomicInteger maxKey = new AtomicInteger();
        final AtomicBoolean finished = new AtomicBoolean();

        final KeyValueGenerator keyValueGenerator = new KeyValueGenerator();
        final Random random = new Random(System.nanoTime());

        final ExecutorService writeExecutor = Executors.newFixedThreadPool(THREAD_COUNT);
        final ExecutorService readExecutor = Executors.newFixedThreadPool(THREAD_COUNT);
        final ExecutorService scanExecutor = Executors.newFixedThreadPool(THREAD_COUNT);

        final DB db = HeftyDB.open(ConfigGenerator.defaultConfig());

        for (int i = 0; i < THREAD_COUNT; i++) {
            writeExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < 10; i++) {
                                String nextKey = Long.toString(maxKey.incrementAndGet());
                                Snapshot maxSnapshot = db.put(ByteBuffers.fromString(nextKey),
                                        keyValueGenerator.testValue(100));

                                long currentMaxSnapshotId = maxSnapshotId.get();

                                if (maxSnapshot.id() > currentMaxSnapshotId) {
                                    maxSnapshotId.compareAndSet(currentMaxSnapshotId, maxSnapshot.id());
                                }
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });

            readExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < 10; i++) {
                                String nextKey = Long.toString(random.nextInt(maxKey.get()));
                                db.get(ByteBuffers.fromString(nextKey));
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });

            scanExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        Iterator<Record> scanIterator = db.ascendingIterator(new Snapshot(maxSnapshotId.get()));
                        long maxScanSnapshotId = 0;

                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < 10; i++) {
                                if (scanIterator.hasNext()) {
                                    Record record = scanIterator.next();
                                    maxScanSnapshotId = Math.max(maxScanSnapshotId, record.snapshot().id());
                                } else {
                                    scanIterator = db.ascendingIterator(new Snapshot(maxSnapshotId.get()));
                                }
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });<commit_msg>Better parameterize endurance test<commit_after>/*
 * Copyright (c) 2014. <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.jordanwilliams.heftydb.test.endurance;

import com.jordanwilliams.heftydb.db.DB;
import com.jordanwilliams.heftydb.db.HeftyDB;
import com.jordanwilliams.heftydb.db.Record;
import com.jordanwilliams.heftydb.db.Snapshot;
import com.jordanwilliams.heftydb.test.generator.ConfigGenerator;
import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator;
import com.jordanwilliams.heftydb.test.helper.StopWatch;
import com.jordanwilliams.heftydb.test.helper.TestFileHelper;
import com.jordanwilliams.heftydb.util.ByteBuffers;

import java.util.Iterator;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

public class EnduranceTest {

    private static final int THREAD_COUNT = 16;
    private static final int RUNTIME_MINUTES = 30;
    private static final int LOAD_LEVEL = 10;

    public static void main(String[] args) throws Exception {
        TestFileHelper.createTestDirectory();
        TestFileHelper.cleanUpTestFiles();

        final AtomicLong maxSnapshotId = new AtomicLong();
        final AtomicInteger maxKey = new AtomicInteger();
        final AtomicBoolean finished = new AtomicBoolean();

        final KeyValueGenerator keyValueGenerator = new KeyValueGenerator();
        final Random random = new Random(System.nanoTime());

        final ExecutorService writeExecutor = Executors.newFixedThreadPool(THREAD_COUNT);
        final ExecutorService readExecutor = Executors.newFixedThreadPool(THREAD_COUNT);
        final ExecutorService scanExecutor = Executors.newFixedThreadPool(THREAD_COUNT);

        final DB db = HeftyDB.open(ConfigGenerator.defaultConfig());

        for (int i = 0; i < THREAD_COUNT; i++) {
            writeExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < LOAD_LEVEL; i++) {
                                String nextKey = Long.toString(maxKey.incrementAndGet());
                                Snapshot maxSnapshot = db.put(ByteBuffers.fromString(nextKey),
                                        keyValueGenerator.testValue(100));

                                long currentMaxSnapshotId = maxSnapshotId.get();

                                if (maxSnapshot.id() > currentMaxSnapshotId) {
                                    maxSnapshotId.compareAndSet(currentMaxSnapshotId, maxSnapshot.id());
                                }
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });

            readExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < LOAD_LEVEL; i++) {
                                String nextKey = Long.toString(random.nextInt(maxKey.get()));
                                db.get(ByteBuffers.fromString(nextKey));
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });

            scanExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        Iterator<Record> scanIterator = db.ascendingIterator(new Snapshot(maxSnapshotId.get()));
                        long maxScanSnapshotId = 0;

                        while (true) {
                            if (finished.get()) {
                                return;
                            }

                            for (int i = 0; i < LOAD_LEVEL; i++) {
                                if (scanIterator.hasNext()) {
                                    Record record = scanIterator.next();
                                    maxScanSnapshotId = Math.max(maxScanSnapshotId, record.snapshot().id());
                                } else {
                                    scanIterator = db.ascendingIterator(new Snapshot(maxSnapshotId.get()));
                                }
                            }

                            Thread.sleep(10);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });
{'fraction_non_alphanumeric': 0.08149256358356068, 'fraction_numerical': 0.0052802956965590075, 'mean_word_length': 1.4091583633665465, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4742693', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2590, 'n_words': 623}