<commit_before>package ru.aifgi.recognizer.model;

/*
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import ru.aifgi.recognizer.api.ImageWrapper;
import ru.aifgi.recognizer.api.ModelFacade;
import ru.aifgi.recognizer.api.ProgressListener;
import ru.aifgi.recognizer.api.Rectangle;
import ru.aifgi.recognizer.api.neural_network.NeuralNetwork;
import ru.aifgi.recognizer.model.neural_network.NeuralNetworkImpl;
import ru.aifgi.recognizer.model.neural_network.NeuralNetworkStructureImpl;
import ru.aifgi.recognizer.model.preprosessing.Binarizer;
import ru.aifgi.recognizer.model.preprosessing.ImageComponentsFinder;
import ru.aifgi.recognizer.model.thread_factories.MyThreadFactory;

import java.awt.*;
import java.io.*;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @author aifgi
 */

class ModelFacadeImpl implements ModelFacade {
    private final ExecutorService myExecutorService = Executors.newCachedThreadPool(new MyThreadFactory("Recognition"));

    private final Set<ProgressListener> myProgressListeners = new HashSet<>();
    private final Binarizer myBinarizer = new Binarizer();
    private NeuralNetwork myNeuralNetwork = new NeuralNetworkImpl(new NeuralNetworkStructureImpl());
    private Labels myLabels = new Labels(new char[] {
            'а', 'б', 'в', 'г', 'д', 'Е', 'ж', 'и', 'к', 'л', 'м', 'н', 'п', 'р', 'с',
            'т', 'у', 'ф', 'х', 'ц', 'ш', 'щ', 'ъ', 'ь', 'э', 'ю', 'я', 'е',
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
    });

    @Override
    public void addProgressListener(final ProgressListener progressListener) {
        myProgressListeners.add(progressListener);
    }

    @Override
    public void removeProgressListener(final ProgressListener progressListener) {
        myProgressListeners.remove(progressListener);
    }

    @Override
    public String recognize(final ImageWrapper inputImage) {
        if (EventQueue.isDispatchThread()) {
            // TODO: exceptions
            throw new RuntimeException("Recognition couldn't work in event dispatch thread");
        }
        notifyStarted("Recognition started");

        final String result = doRecognize(inputImage);

        notifyDone("Recognition done");
        return result;
    }

    private String doRecognize(final ImageWrapper inputImage) {
        double[][] input = inputImage.getBrightnesses();
        notifyProgress(1, "Binarization");
        input = myBinarizer.apply(input);
        notifyProgress(5, "Find connected components");
        final ImageComponentsFinder imageComponentsFinder = new ImageComponentsFinder(input);
        final Collection<Rectangle> words = imageComponentsFinder.getWords();
        notifyProgress(20, "Word components recognition");

        final String[] recognizedWords = recognizeWords(input, words);

        return buildResult(recognizedWords);
    }

    private String[] recognizeWords(final double[][] input, final Collection<Rectangle> words) {
        final WordRecognizer wordRecognizer = new WordRecognizer(input, myLabels, myNeuralNetwork);
        final int size = words.size();
        final CountDownLatch latch = new CountDownLatch(size);
        final String[] recognizedWords = new String[size];
        int i = 0;
        for (final Rectangle word : words) {
            final int pos = i++;
            myExecutorService.execute(new Runnable() {
                @Override
                public void run() {
                    recognizedWords[pos] = wordRecognizer.recognize(word);
                    latch.countDown();
                    notifyProgress((int) (20 + 80 * ((double) size - latch.getCount()) / size));
            }
            });
        }
        try {
            latch.await();
        }
        catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        return recognizedWords;
    }

    private String buildResult(final String[] recognizedWords) {
        final StringBuilder builder = new StringBuilder();
        for (final String word : recognizedWords) {
            builder.append(word).append(' ');
        }
        return builder.toString();
    }

    // TODO: another thread?
    private void notifyStarted(final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.started(message);
        }
    }

    private void notifyProgress(final int progress) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.progress(progress);
        }
    }

    private void notifyProgress(final int progress, final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.progress(progress, message);
        }
    }

    private void notifyDone(final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.done(message);
        }
    }

    @Override
    public void study(File file) {
        //To change body of implemented methods use File | Settings | File Templates.
    }

    @Override
    public void saveRecognizer(final File file) {
        try (final ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(file))) {
            stream.writeObject(myLabels);
            stream.writeObject(myNeuralNetwork);
        }
        catch (IOException e) {
            // TODO:
            throw new RuntimeException(e);<commit_msg>eat memory when it need<commit_after>package ru.aifgi.recognizer.model;

/*
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import ru.aifgi.recognizer.api.ImageWrapper;
import ru.aifgi.recognizer.api.ModelFacade;
import ru.aifgi.recognizer.api.ProgressListener;
import ru.aifgi.recognizer.api.Rectangle;
import ru.aifgi.recognizer.api.neural_network.NeuralNetwork;
import ru.aifgi.recognizer.model.neural_network.NeuralNetworkImpl;
import ru.aifgi.recognizer.model.neural_network.NeuralNetworkStructureImpl;
import ru.aifgi.recognizer.model.preprosessing.Binarizer;
import ru.aifgi.recognizer.model.preprosessing.ImageComponentsFinder;
import ru.aifgi.recognizer.model.thread_factories.MyThreadFactory;

import java.awt.*;
import java.io.*;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @author aifgi
 */

class ModelFacadeImpl implements ModelFacade {
    private final ExecutorService myExecutorService = Executors.newCachedThreadPool(new MyThreadFactory("Recognition"));

    private final Set<ProgressListener> myProgressListeners = new HashSet<>();
    private NeuralNetwork myNeuralNetwork;
    private Labels myLabels;

    @Override
    public void addProgressListener(final ProgressListener progressListener) {
        myProgressListeners.add(progressListener);
    }

    @Override
    public void removeProgressListener(final ProgressListener progressListener) {
        myProgressListeners.remove(progressListener);
    }

    @Override
    public String recognize(final ImageWrapper inputImage) {
        if (EventQueue.isDispatchThread()) {
            // TODO: exceptions
            throw new RuntimeException("Recognition couldn't work in event dispatch thread");
        }
        notifyStarted("Recognition started");

        final String result = doRecognize(inputImage);

        notifyDone("Recognition done");
        return result;
    }

    private String doRecognize(final ImageWrapper inputImage) {
        init();
        double[][] input = inputImage.getBrightnesses();
        notifyProgress(1, "Binarization");
        input = new Binarizer().apply(input);
        notifyProgress(5, "Find connected components");
        final ImageComponentsFinder imageComponentsFinder = new ImageComponentsFinder(input);
        final Collection<Rectangle> words = imageComponentsFinder.getWords();
        notifyProgress(20, "Word components recognition");

        final String[] recognizedWords = recognizeWords(input, words);

        return buildResult(recognizedWords);
    }

    private void init() {
        if (myNeuralNetwork == null) {
            myNeuralNetwork = new NeuralNetworkImpl(new NeuralNetworkStructureImpl());
        }
        if (myLabels == null) {
            myLabels = new Labels(new char[] {
                    'а', 'б', 'в', 'г', 'д', 'Е', 'ж', 'и', 'к', 'л', 'м', 'н', 'п', 'р', 'с',
                    'т', 'у', 'ф', 'х', 'ц', 'ш', 'щ', 'ъ', 'ь', 'э', 'ю', 'я', 'е',
                    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
            });
        }
    }

    private String[] recognizeWords(final double[][] input, final Collection<Rectangle> words) {
        final WordRecognizer wordRecognizer = new WordRecognizer(input, myLabels, myNeuralNetwork);
        final int size = words.size();
        final CountDownLatch latch = new CountDownLatch(size);
        final String[] recognizedWords = new String[size];
        int i = 0;
        for (final Rectangle word : words) {
            final int pos = i++;
            myExecutorService.execute(new Runnable() {
                @Override
                public void run() {
                    recognizedWords[pos] = wordRecognizer.recognize(word);
                    latch.countDown();
                    notifyProgress((int) (20 + 80 * ((double) size - latch.getCount()) / size));
            }
            });
        }
        try {
            latch.await();
        }
        catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        return recognizedWords;
    }

    private String buildResult(final String[] recognizedWords) {
        final StringBuilder builder = new StringBuilder();
        for (final String word : recognizedWords) {
            builder.append(word).append(' ');
        }
        return builder.toString();
    }

    // TODO: another thread?
    private void notifyStarted(final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.started(message);
        }
    }

    private void notifyProgress(final int progress) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.progress(progress);
        }
    }

    private void notifyProgress(final int progress, final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.progress(progress, message);
        }
    }

    private void notifyDone(final String message) {
        for (final ProgressListener progressListener : myProgressListeners) {
            progressListener.done(message);
        }
    }

    @Override
    public void study(File file) {
        init();
    }

    @Override
    public void saveRecognizer(final File file) {
        try (final ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(file))) {
            stream.writeObject(myLabels);
            stream.writeObject(myNeuralNetwork);
        }
        catch (IOException e) {
            // TODO:
            throw new RuntimeException(e);
{'fraction_non_alphanumeric': 0.10229460847240052, 'fraction_numerical': 0.004332477535301669, 'mean_word_length': 2.9210443535703052, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7388386', 'n_tokens_mistral': 3339, 'n_tokens_neox': 3233, 'n_words': 892}