<commit_before>#!/usr/bin/python
import os
import subprocess
import shutil
import configuration as cfg

import gobject
import glib
import dbus
from dbus.mainloop.glib import DBusGMainLoop
import time

import options
from dconf import DConfClient

# Don't use /tmp (not enough space there)

# Add this after fixing the backup/restore and ontology changes tests
#"G_DEBUG" : "fatal_criticals",

TEST_ENV_DIRS =  { "XDG_DATA_HOME" : os.path.join (cfg.TEST_TMP_DIR, "xdg-data-home"),
                   "XDG_CACHE_HOME": os.path.join (cfg.TEST_TMP_DIR, "xdg-cache-home")}

TEST_ENV_VARS = {  "TRACKER_DISABLE_MEEGOTOUCH_LOCALE": "",
                   "LC_COLLATE": "en_GB.utf8",
                   "DCONF_PROFILE": "trackertest"}

EXTRA_DIRS = [os.path.join (cfg.TEST_TMP_DIR, "xdg-data-home", "tracker"),
              os.path.join (cfg.TEST_TMP_DIR, "xdg-cache-home", "tracker")]

REASONABLE_TIMEOUT = 30

class UnableToBootException (Exception):
    pass

class TrackerStoreLifeCycle ():

    def __init__ (self):
        self.timeout_id = 0
        self.available = False
        
    def start (self):
        """
        call this method to start and instance of tracker-store. It will return when the store is ready
        """
        self.loop = gobject.MainLoop()
        dbus_loop = DBusGMainLoop(set_as_default=True)
        self.bus = dbus.SessionBus (dbus_loop)

        obj = self.bus.get_object ("org.freedesktop.DBus",
                                   "/org/freedesktop/DBus")
        self.admin = dbus.Interface (obj, dbus_interface="org.freedesktop.DBus")
        if (self.admin.NameHasOwner (cfg.TRACKER_BUSNAME)):
            raise Exception ("Store is already running! kill it before starting this one")

        self.name_owner_match = self.bus.add_signal_receiver (self.__name_owner_changed_cb,
                                                         signal_name="NameOwnerChanged",
                                                         path="/org/freedesktop/DBus",
                                                         dbus_interface="org.freedesktop.DBus")
        self.store_proc = self.__start_tracker_store ()

        # It should step out of this loop when the miner is visible in DBus
        self.loop.run ()

        tracker = self.bus.get_object (cfg.TRACKER_BUSNAME, cfg.TRACKER_OBJ_PATH)
        tracker_status = self.bus.get_object (cfg.TRACKER_BUSNAME, cfg.TRACKER_STATUS_OBJ_PATH)
        self.status_iface = dbus.Interface (tracker_status, dbus_interface=cfg.STATUS_IFACE)
        print "[store] booting..."
        self.status_iface.Wait ()
        print "[store] ready."


    def stop (self):
        self.__stop_tracker_store ()
        if self.available:
            # It should step out of this loop when the miner disappear from the bus
            self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
            self.loop.run ()

        print "[store] stop."
        # Disconnect the signals of the next start we get duplicated messages
        self.bus._clean_up_signal_match (self.name_owner_match)

    def kill (self):
        self.store_proc.kill ()
        self.loop.run ()
        # Name owner changed cb should take us out from this loop

        print "[store] killed."
        self.bus._clean_up_signal_match (self.name_owner_match)

    def __timeout_on_idle (self):
        print "[store] Timeout waiting... asumming idle."
        self.loop.quit ()
        return False

    def __name_owner_changed_cb (self, name, old_owner, new_owner):
        if name == cfg.TRACKER_BUSNAME:
            if old_owner == '' and new_owner != '':
                print "[store] appears in the bus"
                self.available = True
            elif old_owner != ''  and new_owner == '':
                print "[store] disappears from the bus"
                self.available = False
            else:
                print "[store] name change %s -> %s" % (old_owner, new_owner)
            self.loop.quit ()

    def __start_tracker_store (self):
        tracker_binary = os.path.join (cfg.EXEC_PREFIX, "tracker-store")
        tracker = [tracker_binary]
        # The env variables can be passed as parameters!
        FNULL = open('/dev/null', 'w')
        if options.is_manual_start ():
            print "Start tracker-store manually"
        else:
            return subprocess.Popen (tracker, stdout=FNULL, stderr=FNULL)

    def __stop_tracker_store (self):
        #control_binary = os.path.join (cfg.BINDIR, "tracker-control")
        #FNULL = open('/dev/null', 'w')
        #subprocess.call ([control_binary, "-t"], stdout=FNULL)
        if options.is_manual_start ():
            if self.available:
                print "Kill tracker-store manually"
                # Quit when disappearing from the bus
                self.loop.run ()
        else:
            self.store_proc.terminate ()


class TrackerMinerFsLifeCycle():
    """
    Starts and monitors the miner-fs life cycle
    """
    def __init__ (self):
        self.timeout_id = 0
        self.available = False

    def start (self):
        """
        call this method to start and instance of miner-fs. It will return when the miner is 'Idle'
        after all the initial crawling
        """
        self.loop = gobject.MainLoop()
        dbus_loop = DBusGMainLoop(set_as_default=True)
        self.bus = dbus.SessionBus (dbus_loop)

        obj = self.bus.get_object ("org.freedesktop.DBus",
                                   "/org/freedesktop/DBus")
        self.admin = dbus.Interface (obj, dbus_interface="org.freedesktop.DBus")
        if (self.admin.NameHasOwner (cfg.MINERFS_BUSNAME)):
            raise Exception ("Miner is already running! kill it before starting this one")

        self.name_owner_match = self.bus.add_signal_receiver (self.__name_owner_changed_cb,
                                                         signal_name="NameOwnerChanged",
                                                         path="/org/freedesktop/DBus",
                                                         dbus_interface="org.freedesktop.DBus")
        self.__start_tracker_miner_fs ()

        # It should step out of this loop when the miner is visible in DBus
        self.loop.run ()

        self.status_match = self.bus.add_signal_receiver (self.__minerfs_status_cb,
                                                     signal_name="Progress",
                                                     path=cfg.MINERFS_OBJ_PATH,
                                                     dbus_interface=cfg.MINER_IFACE)
        # It should step out of this loop after to "Idle" progress changes
        self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
        self.loop.run ()


    def stop (self):
        self.__stop_tracker_miner_fs ()
        # It should step out of this loop when the miner disappear from the bus
        if (self.available):
            self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
            self.loop.run ()

        # Disconnect the signals of the next start we get duplicated messages
        self.bus._clean_up_signal_match (self.name_owner_match)
        self.bus._clean_up_signal_match (self.status_match)
        print "[miner-fs] stop."


    def wait_for_idle (self, timeout=REASONABLE_TIMEOUT):
        # The signal is already connected
        print "\n[miner-fs] waiting for Idle"
        self.timeout_id = glib.timeout_add_seconds (timeout, self.__timeout_on_idle)
        self.loop.run ()

    def __timeout_on_idle (self):
        print "[miner-fs] timeout... asumming idle"
        self.loop.quit ()
        return False

    def __minerfs_status_cb (self, status, handle):
        print "[miner-fs] status is now", status.encode ("utf-8")
        if (status == "Idle"):
            if (self.timeout_id != 0):
                glib.source_remove (self.timeout_id)
                self.timeout_id = 0
            self.loop.quit ()


    def __name_owner_changed_cb (self, name, old_owner, new_owner):
        if name == cfg.MINERFS_BUSNAME:
            if old_owner == '' and new_owner != '':<commit_msg>functional-tests: Update miner progress signal handler<commit_after>#!/usr/bin/python
import os
import subprocess
import shutil
import configuration as cfg

import gobject
import glib
import dbus
from dbus.mainloop.glib import DBusGMainLoop
import time

import options
from dconf import DConfClient

# Don't use /tmp (not enough space there)

# Add this after fixing the backup/restore and ontology changes tests
#"G_DEBUG" : "fatal_criticals",

TEST_ENV_DIRS =  { "XDG_DATA_HOME" : os.path.join (cfg.TEST_TMP_DIR, "xdg-data-home"),
                   "XDG_CACHE_HOME": os.path.join (cfg.TEST_TMP_DIR, "xdg-cache-home")}

TEST_ENV_VARS = {  "TRACKER_DISABLE_MEEGOTOUCH_LOCALE": "",
                   "LC_COLLATE": "en_GB.utf8",
                   "DCONF_PROFILE": "trackertest"}

EXTRA_DIRS = [os.path.join (cfg.TEST_TMP_DIR, "xdg-data-home", "tracker"),
              os.path.join (cfg.TEST_TMP_DIR, "xdg-cache-home", "tracker")]

REASONABLE_TIMEOUT = 30

class UnableToBootException (Exception):
    pass

class TrackerStoreLifeCycle ():

    def __init__ (self):
        self.timeout_id = 0
        self.available = False
        
    def start (self):
        """
        call this method to start and instance of tracker-store. It will return when the store is ready
        """
        self.loop = gobject.MainLoop()
        dbus_loop = DBusGMainLoop(set_as_default=True)
        self.bus = dbus.SessionBus (dbus_loop)

        obj = self.bus.get_object ("org.freedesktop.DBus",
                                   "/org/freedesktop/DBus")
        self.admin = dbus.Interface (obj, dbus_interface="org.freedesktop.DBus")
        if (self.admin.NameHasOwner (cfg.TRACKER_BUSNAME)):
            raise Exception ("Store is already running! kill it before starting this one")

        self.name_owner_match = self.bus.add_signal_receiver (self.__name_owner_changed_cb,
                                                         signal_name="NameOwnerChanged",
                                                         path="/org/freedesktop/DBus",
                                                         dbus_interface="org.freedesktop.DBus")
        self.store_proc = self.__start_tracker_store ()

        # It should step out of this loop when the miner is visible in DBus
        self.loop.run ()

        tracker = self.bus.get_object (cfg.TRACKER_BUSNAME, cfg.TRACKER_OBJ_PATH)
        tracker_status = self.bus.get_object (cfg.TRACKER_BUSNAME, cfg.TRACKER_STATUS_OBJ_PATH)
        self.status_iface = dbus.Interface (tracker_status, dbus_interface=cfg.STATUS_IFACE)
        print "[store] booting..."
        self.status_iface.Wait ()
        print "[store] ready."


    def stop (self):
        self.__stop_tracker_store ()
        if self.available:
            # It should step out of this loop when the miner disappear from the bus
            self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
            self.loop.run ()

        print "[store] stop."
        # Disconnect the signals of the next start we get duplicated messages
        self.bus._clean_up_signal_match (self.name_owner_match)

    def kill (self):
        self.store_proc.kill ()
        self.loop.run ()
        # Name owner changed cb should take us out from this loop

        print "[store] killed."
        self.bus._clean_up_signal_match (self.name_owner_match)

    def __timeout_on_idle (self):
        print "[store] Timeout waiting... asumming idle."
        self.loop.quit ()
        return False

    def __name_owner_changed_cb (self, name, old_owner, new_owner):
        if name == cfg.TRACKER_BUSNAME:
            if old_owner == '' and new_owner != '':
                print "[store] appears in the bus"
                self.available = True
            elif old_owner != ''  and new_owner == '':
                print "[store] disappears from the bus"
                self.available = False
            else:
                print "[store] name change %s -> %s" % (old_owner, new_owner)
            self.loop.quit ()

    def __start_tracker_store (self):
        tracker_binary = os.path.join (cfg.EXEC_PREFIX, "tracker-store")
        tracker = [tracker_binary]
        # The env variables can be passed as parameters!
        FNULL = open('/dev/null', 'w')
        if options.is_manual_start ():
            print "Start tracker-store manually"
        else:
            return subprocess.Popen (tracker, stdout=FNULL, stderr=FNULL)

    def __stop_tracker_store (self):
        #control_binary = os.path.join (cfg.BINDIR, "tracker-control")
        #FNULL = open('/dev/null', 'w')
        #subprocess.call ([control_binary, "-t"], stdout=FNULL)
        if options.is_manual_start ():
            if self.available:
                print "Kill tracker-store manually"
                # Quit when disappearing from the bus
                self.loop.run ()
        else:
            self.store_proc.terminate ()


class TrackerMinerFsLifeCycle():
    """
    Starts and monitors the miner-fs life cycle
    """
    def __init__ (self):
        self.timeout_id = 0
        self.available = False

    def start (self):
        """
        call this method to start and instance of miner-fs. It will return when the miner is 'Idle'
        after all the initial crawling
        """
        self.loop = gobject.MainLoop()
        dbus_loop = DBusGMainLoop(set_as_default=True)
        self.bus = dbus.SessionBus (dbus_loop)

        obj = self.bus.get_object ("org.freedesktop.DBus",
                                   "/org/freedesktop/DBus")
        self.admin = dbus.Interface (obj, dbus_interface="org.freedesktop.DBus")
        if (self.admin.NameHasOwner (cfg.MINERFS_BUSNAME)):
            raise Exception ("Miner is already running! kill it before starting this one")

        self.name_owner_match = self.bus.add_signal_receiver (self.__name_owner_changed_cb,
                                                         signal_name="NameOwnerChanged",
                                                         path="/org/freedesktop/DBus",
                                                         dbus_interface="org.freedesktop.DBus")
        self.__start_tracker_miner_fs ()

        # It should step out of this loop when the miner is visible in DBus
        self.loop.run ()

        self.status_match = self.bus.add_signal_receiver (self.__minerfs_status_cb,
                                                     signal_name="Progress",
                                                     path=cfg.MINERFS_OBJ_PATH,
                                                     dbus_interface=cfg.MINER_IFACE)
        # It should step out of this loop after to "Idle" progress changes
        self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
        self.loop.run ()


    def stop (self):
        self.__stop_tracker_miner_fs ()
        # It should step out of this loop when the miner disappear from the bus
        if (self.available):
            self.timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self.__timeout_on_idle)
            self.loop.run ()

        # Disconnect the signals of the next start we get duplicated messages
        self.bus._clean_up_signal_match (self.name_owner_match)
        self.bus._clean_up_signal_match (self.status_match)
        print "[miner-fs] stop."


    def wait_for_idle (self, timeout=REASONABLE_TIMEOUT):
        # The signal is already connected
        print "\n[miner-fs] waiting for Idle"
        self.timeout_id = glib.timeout_add_seconds (timeout, self.__timeout_on_idle)
        self.loop.run ()

    def __timeout_on_idle (self):
        print "[miner-fs] timeout... asumming idle"
        self.loop.quit ()
        return False

    def __minerfs_status_cb (self, status, progress, remaining_time):
        print "[miner-fs] status is now", status.encode ("utf-8")
        if (status == "Idle"):
            if (self.timeout_id != 0):
                glib.source_remove (self.timeout_id)
                self.timeout_id = 0
            self.loop.quit ()


    def __name_owner_changed_cb (self, name, old_owner, new_owner):
        if name == cfg.MINERFS_BUSNAME:
            if old_owner == '' and new_owner != '':
{'fraction_non_alphanumeric': 0.09289750288595905, 'fraction_numerical': 0.0009721125220244244, 'mean_word_length': 2.029076186970924, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3824388', 'n_tokens_mistral': 4850, 'n_tokens_neox': 4725, 'n_words': 1174}