<commit_before>/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.jet.impl.connector;

import com.hazelcast.jet.AbstractProcessor;
import com.hazelcast.jet.Processor;
import com.hazelcast.jet.ProcessorSupplier;
import com.hazelcast.logging.ILogger;
import com.hazelcast.util.ExceptionUtil;

import javax.annotation.Nonnull;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.sun.nio.file.SensitivityWatchEventModifier.HIGH;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;

/**
 * A producer reads files from directory and emits them lines by lines to the next processor
 */
public class FileStreamReader extends AbstractProcessor {

    private static final ILogger LOGGER = com.hazelcast.logging.Logger.getLogger(FileStreamReader.class);
    private final String filePath;
    private final WatchType watchType;
    private final int parallelism;
    private final int id;
    private WatchService watcher;
    private Path path;

    public enum WatchType {

        /**
         * Process only new files.
         */
        NEW,

        /**
         * Re-process files from beginning when a modification occurs on them.
         */
        REPROCESS,

        /**
         * Process only appended content to the files.
         */
        APPENDED_ONLY
    }

    private Map<String, Long> fileOffsets = new HashMap<>();


    protected FileStreamReader(String filePath, WatchType watchType, int parallelism, int id) {
        this.filePath = filePath;
        this.watchType = watchType;
        this.parallelism = parallelism;
        this.id = id;
    }

    @Override
    protected void init(@Nonnull Context context) {
        try {
            watcher = FileSystems.getDefault().newWatchService();
            path = Paths.get(filePath);
            path.register(watcher, new WatchEvent.Kind[]{ENTRY_CREATE, ENTRY_MODIFY, ENTRY_DELETE}, HIGH);
            LOGGER.info("Started to watch the directory : " + filePath);
        } catch (IOException e) {
            LOGGER.severe("Error occurred while watching directories, error : " + e.getMessage());
            throw ExceptionUtil.rethrow(e);
        }
    }

    @Override
    public boolean complete() {
        try {
            while (true) {
                WatchKey key;
                try {
                    key = watcher.take();
                } catch (InterruptedException x) {
                    return true;
                }

                for (WatchEvent<?> event : key.pollEvents()) {
                    WatchEvent.Kind<?> kind = event.kind();
                    WatchEvent<Path> ev = (WatchEvent<Path>) event;
                    Path file = ev.context();
                    Path resolved = path.resolve(file);
                    if (kind == OVERFLOW) {
                        continue;
                    }
                    if (kind == ENTRY_DELETE) {
                        LOGGER.info("Directory (" + filePath + ") deleted, stopped watching");
                        watcher.close();
                        return true;
                    }

                    int filenameHash = resolved.toFile().getPath().hashCode();
                    if (((filenameHash & Integer.MAX_VALUE) % parallelism) != id) {
                        continue;
                    }

                    if (kind == ENTRY_CREATE) {
                        LOGGER.info("New file (" + resolved + ") added");
                        readFile(resolved, 0L);
                        return !key.reset();
                    } else if (kind == ENTRY_MODIFY) {
                        LOGGER.info("The file (" + resolved + ") updated");
                        if (watchType == WatchType.NEW) {
                            continue;
                        } else if (watchType == WatchType.REPROCESS) {
                            LOGGER.info("Re-processing the file (" + resolved + ")");
                            fileOffsets.put(resolved.toString(), 0L);
                            readFile(resolved, 0L);
                            return !key.reset();
                        } else if (watchType == WatchType.APPENDED_ONLY) {
                            LOGGER.info("Processing only the appended content on the file (" + resolved + ")");
                            readFile(resolved, fileOffsets.computeIfAbsent(resolved.toString(), s -> 0L));
                            return !key.reset();
                        }
                    }
                }<commit_msg>Simplify logic in FileStreamReader<commit_after>/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.jet.impl.connector;

import com.hazelcast.jet.AbstractProcessor;
import com.hazelcast.jet.Processor;
import com.hazelcast.jet.ProcessorSupplier;
import com.hazelcast.logging.ILogger;
import com.hazelcast.util.ExceptionUtil;

import javax.annotation.Nonnull;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.sun.nio.file.SensitivityWatchEventModifier.HIGH;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;

/**
 * A producer reads files from directory and emits them lines by lines to the next processor
 */
public class FileStreamReader extends AbstractProcessor {

    private static final ILogger LOGGER = com.hazelcast.logging.Logger.getLogger(FileStreamReader.class);
    private final String filePath;
    private final WatchType watchType;
    private final int parallelism;
    private final int id;
    private WatchService watcher;
    private Path path;

    public enum WatchType {

        /**
         * Process only new files.
         */
        NEW,

        /**
         * Re-process files from beginning when a modification occurs on them.
         */
        REPROCESS,

        /**
         * Process only appended content to the files.
         */
        APPENDED_ONLY
    }

    private Map<String, Long> fileOffsets = new HashMap<>();


    protected FileStreamReader(String filePath, WatchType watchType, int parallelism, int id) {
        this.filePath = filePath;
        this.watchType = watchType;
        this.parallelism = parallelism;
        this.id = id;
    }

    @Override
    protected void init(@Nonnull Context context) {
        try {
            watcher = FileSystems.getDefault().newWatchService();
            path = Paths.get(filePath);
            path.register(watcher, new WatchEvent.Kind[]{ENTRY_CREATE, ENTRY_MODIFY, ENTRY_DELETE}, HIGH);
            LOGGER.info("Started to watch the directory : " + filePath);
        } catch (IOException e) {
            LOGGER.severe("Error occurred while watching directories, error : " + e.getMessage());
            throw ExceptionUtil.rethrow(e);
        }
    }

    @Override
    public boolean complete() {
        try {
            while (true) {
                final WatchKey key;
                try {
                    key = watcher.take();
                } catch (InterruptedException x) {
                    return true;
                }

                for (WatchEvent<?> event : key.pollEvents()) {
                    final WatchEvent.Kind<?> kind = event.kind();
                    final WatchEvent<Path> ev = (WatchEvent<Path>) event;
                    final Path file = ev.context();
                    final Path resolved = path.resolve(file);
                    if (kind == OVERFLOW) {
                        continue;
                    }
                    if (kind == ENTRY_DELETE) {
                        LOGGER.info("Directory (" + filePath + ") deleted, stopped watching");
                        watcher.close();
                        return true;
                    }

                    final int filenameHash = resolved.toFile().getPath().hashCode();
                    if (((filenameHash & Integer.MAX_VALUE) % parallelism) != id) {
                        continue;
                    }

                    if (kind == ENTRY_CREATE) {
                        LOGGER.info("Detected new file: " + resolved);
                        readFile(resolved, 0L);
                        return !key.reset();
                    }
                    if (kind != ENTRY_MODIFY) {
                        continue;
                    }
                    LOGGER.info("Detected file change: " + resolved);
                    switch (watchType) {
                        case REPROCESS:
                            LOGGER.info("Re-processing the file (" + resolved + ')');
                            fileOffsets.put(resolved.toString(), 0L);
                            readFile(resolved, 0L);
                            return !key.reset();
                        case APPENDED_ONLY:
                            LOGGER.info("Processing only the appended content on the file (" + resolved + ')');
                            readFile(resolved, fileOffsets.computeIfAbsent(resolved.toString(), s -> 0L));
                            return !key.reset();
                        case NEW:
                        default:
                    }
                }
{'fraction_non_alphanumeric': 0.09222486982614067, 'fraction_numerical': 0.0028241108463507194, 'mean_word_length': 1.8580075662042874, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5538316', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2812, 'n_words': 790}