<commit_before>/*
 * Copyright (C) 2009 <NAME> <<EMAIL>>.
 *
 * This file is part of Rygel.
 *
 * Rygel is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Rygel is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

using GLib;
using Gee;

public class Rygel.MediaExport.Harvester : GLib.Object {
    private MetadataExtractor extractor;
    private MediaCache media_db;
    private GLib.Queue<MediaContainer> containers;
    private Gee.Queue<File> files;
    private File origin;
    private MediaContainer parent;
    private RecursiveFileMonitor monitor;
    private Regex file_filter;
    public Cancellable cancellable;
    private string flag;
    private const string HARVESTER_ATTRIBUTES =
                                        FILE_ATTRIBUTE_STANDARD_NAME + "," +
                                        FILE_ATTRIBUTE_STANDARD_TYPE + "," +
                                        FILE_ATTRIBUTE_TIME_MODIFIED + "," +
                                        FILE_ATTRIBUTE_STANDARD_SIZE;


    public Harvester (MediaContainer       parent,
                      MediaCache           media_db,
                      MetadataExtractor    extractor,
                      RecursiveFileMonitor monitor,
                      string?              flag = null) {
        this.parent = parent;
        this.extractor = extractor;
        this.media_db = media_db;
        this.extractor.extraction_done.connect (on_extracted_cb);
        this.extractor.error.connect (on_extractor_error_cb);
        this.files = new LinkedList<File> ();
        this.containers = new GLib.Queue<MediaContainer> ();
        this.origin = null;
        this.monitor = monitor;
        this.cancellable = new Cancellable ();
        this.flag = flag;
        var config = MetaConfig.get_default ();

        try {
            var extensions = config.get_string_list ("MediaExport",
                                                     "include-filter");

            // never trust user input
            string[] escaped_extensions = new string[0];
            foreach (var extension in extensions) {
                escaped_extensions += Regex.escape_string (extension);
            }

            var list = string.joinv ("|", escaped_extensions);
            file_filter = new Regex ("(%s)$".printf (list),
                                     RegexCompileFlags.CASELESS |
                                     RegexCompileFlags.OPTIMIZE);
        } catch (Error error) {
            file_filter = null;
        }
    }

    private bool push_if_changed_or_unknown (File       file,
                                             FileInfo   info,
                                             out string id) {
        id = Checksum.compute_for_string (ChecksumType.MD5, file.get_uri ());
        int64 timestamp;
        try {
            if (this.media_db.exists (id, out timestamp)) {
                int64 mtime = (int64) info.get_attribute_uint64 (
                                        FILE_ATTRIBUTE_TIME_MODIFIED);

                if (mtime > timestamp) {
                    this.files.offer (file);

                    return true;
                } else {
                    // check size
                    var size = info.get_size ();
                    var item = media_db.get_item (id);
                    if (item.size != size) {
                        this.files.offer (file);

                        return true;
                    }
                }
            } else {
                this.files.offer (file);

                return true;
            }
        } catch (Error error) {
            warning (_("Failed to query database: %s"), error.message);
        }

        return false;
    }

    private bool process_children (GLib.List<FileInfo>? list) {
        if (list == null || this.cancellable.is_cancelled ()) {
            return false;
        }

        foreach (var info in list) {
            if (info.get_name ()[0] == '.') {
                continue;
            }
            var parent_container =
                (DummyContainer) this.containers.peek_head ();

            var dir = parent_container.file;
            var file = dir.get_child (info.get_name ());
            if (info.get_file_type () == FileType.DIRECTORY) {
                monitor.monitor (file);
                var container = new DummyContainer (file,
                                                    parent_container);
                this.containers.push_tail (container);
                parent_container.seen (container.id);
                try {
                    int64 timestamp;
                    if (!this.media_db.exists (container.id,
                                               out timestamp)) {
                        this.media_db.save_container (container);
                    }
                } catch (Error err) {
                    warning (_("Failed to update database: %s"), err.message);
                }
            } else {
                string id;

                // Let's see if the file is wanted
                if (file_filter != null &&
                    !file_filter.match (file.get_uri ())) {
                    continue;
                }

                push_if_changed_or_unknown (file, info, out id);
                parent_container.seen (id);
            }
        }

        return true;
    }

    private async void enumerate_directory (File directory) {
        try {
            var enumerator = yield directory.enumerate_children_async (
                                        HARVESTER_ATTRIBUTES,
                                        FileQueryInfoFlags.NONE,
                                        Priority.DEFAULT,
                                        this.cancellable);

            GLib.List<FileInfo> list = null;
            do {
                list = yield enumerator.next_files_async (10,
                                                          Priority.DEFAULT,
                                                          this.cancellable);
            } while (process_children (list));

            yield enumerator.close_async (Priority.DEFAULT, this.cancellable);
        } catch (Error err) {
            warning (_("failed to enumerate folder: %s"), err.message);
        }

        cleanup_database (this.containers.peek_head () as DummyContainer);
        this.do_update ();
    }

    void cleanup_database (DummyContainer container) {<commit_msg>media-export: Process 256 children at a time<commit_after>/*
 * Copyright (C) 2009 <NAME> <<EMAIL>>.
 *
 * This file is part of Rygel.
 *
 * Rygel is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Rygel is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

using GLib;
using Gee;

public class Rygel.MediaExport.Harvester : GLib.Object {
    private MetadataExtractor extractor;
    private MediaCache media_db;
    private GLib.Queue<MediaContainer> containers;
    private Gee.Queue<File> files;
    private File origin;
    private MediaContainer parent;
    private RecursiveFileMonitor monitor;
    private Regex file_filter;
    public Cancellable cancellable;
    private string flag;
    private const string HARVESTER_ATTRIBUTES =
                                        FILE_ATTRIBUTE_STANDARD_NAME + "," +
                                        FILE_ATTRIBUTE_STANDARD_TYPE + "," +
                                        FILE_ATTRIBUTE_TIME_MODIFIED + "," +
                                        FILE_ATTRIBUTE_STANDARD_SIZE;


    public Harvester (MediaContainer       parent,
                      MediaCache           media_db,
                      MetadataExtractor    extractor,
                      RecursiveFileMonitor monitor,
                      string?              flag = null) {
        this.parent = parent;
        this.extractor = extractor;
        this.media_db = media_db;
        this.extractor.extraction_done.connect (on_extracted_cb);
        this.extractor.error.connect (on_extractor_error_cb);
        this.files = new LinkedList<File> ();
        this.containers = new GLib.Queue<MediaContainer> ();
        this.origin = null;
        this.monitor = monitor;
        this.cancellable = new Cancellable ();
        this.flag = flag;
        var config = MetaConfig.get_default ();

        try {
            var extensions = config.get_string_list ("MediaExport",
                                                     "include-filter");

            // never trust user input
            string[] escaped_extensions = new string[0];
            foreach (var extension in extensions) {
                escaped_extensions += Regex.escape_string (extension);
            }

            var list = string.joinv ("|", escaped_extensions);
            file_filter = new Regex ("(%s)$".printf (list),
                                     RegexCompileFlags.CASELESS |
                                     RegexCompileFlags.OPTIMIZE);
        } catch (Error error) {
            file_filter = null;
        }
    }

    private bool push_if_changed_or_unknown (File       file,
                                             FileInfo   info,
                                             out string id) {
        id = Checksum.compute_for_string (ChecksumType.MD5, file.get_uri ());
        int64 timestamp;
        try {
            if (this.media_db.exists (id, out timestamp)) {
                int64 mtime = (int64) info.get_attribute_uint64 (
                                        FILE_ATTRIBUTE_TIME_MODIFIED);

                if (mtime > timestamp) {
                    this.files.offer (file);

                    return true;
                } else {
                    // check size
                    var size = info.get_size ();
                    var item = media_db.get_item (id);
                    if (item.size != size) {
                        this.files.offer (file);

                        return true;
                    }
                }
            } else {
                this.files.offer (file);

                return true;
            }
        } catch (Error error) {
            warning (_("Failed to query database: %s"), error.message);
        }

        return false;
    }

    private bool process_children (GLib.List<FileInfo>? list) {
        if (list == null || this.cancellable.is_cancelled ()) {
            return false;
        }

        foreach (var info in list) {
            if (info.get_name ()[0] == '.') {
                continue;
            }
            var parent_container =
                (DummyContainer) this.containers.peek_head ();

            var dir = parent_container.file;
            var file = dir.get_child (info.get_name ());
            if (info.get_file_type () == FileType.DIRECTORY) {
                monitor.monitor (file);
                var container = new DummyContainer (file,
                                                    parent_container);
                this.containers.push_tail (container);
                parent_container.seen (container.id);
                try {
                    int64 timestamp;
                    if (!this.media_db.exists (container.id,
                                               out timestamp)) {
                        this.media_db.save_container (container);
                    }
                } catch (Error err) {
                    warning (_("Failed to update database: %s"), err.message);
                }
            } else {
                string id;

                // Let's see if the file is wanted
                if (file_filter != null &&
                    !file_filter.match (file.get_uri ())) {
                    continue;
                }

                push_if_changed_or_unknown (file, info, out id);
                parent_container.seen (id);
            }
        }

        return true;
    }

    private async void enumerate_directory (File directory) {
        try {
            var enumerator = yield directory.enumerate_children_async (
                                        HARVESTER_ATTRIBUTES,
                                        FileQueryInfoFlags.NONE,
                                        Priority.DEFAULT,
                                        this.cancellable);

            GLib.List<FileInfo> list = null;
            do {
                list = yield enumerator.next_files_async (256,
                                                          Priority.DEFAULT,
                                                          this.cancellable);
            } while (process_children (list));

            yield enumerator.close_async (Priority.DEFAULT, this.cancellable);
        } catch (Error err) {
            warning (_("failed to enumerate folder: %s"), err.message);
        }

        cleanup_database (this.containers.peek_head () as DummyContainer);
        this.do_update ();
    }

    void cleanup_database (DummyContainer container) {
{'fraction_non_alphanumeric': 0.07769458963130385, 'fraction_numerical': 0.004661675377878231, 'mean_word_length': 1.3426538716082066, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3106449', 'n_tokens_mistral': 3440, 'n_tokens_neox': 3241, 'n_words': 928}