<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
Manage reports
'''

'''
Copyright (c) 2013, <NAME>, Abundo AB
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
   * Redistributions of source code must retain the above copyright
     notice, this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.
   * Neither the name of the <organization> nor the
     names of its contributors may be used to endorse or promote products
     derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

import datetime
import queue
import threading

import PyQt5.QtCore as QtCore

from myglobals import *
from logger import log
from settings import sett

import util
import lib.db as db
import lib.network as network

class ReportMgr(QtCore.QObject):
    sig = QtCore.pyqtSignal()
    
    def __init__(self, localdb=None):
        super().__init__()
        self.localdb = localdb

        self._autosync = False
        self.reports = []   # local cache for todays reports
        self.toThreadQ = queue.Queue()
        self.t = threading.Thread(target=self.runThread)
        self.t.setName("ReportMgr")
        self.t.daemon = True
        self.t.start()

    def init(self):
        """Load the list of reports from local db"""
        self.sig.emit()

    def get(self, _id):
        for report in self.reports:
            if report._id == _id:
                return report
        try:
            sql = "SELECT * FROM report WHERE _id=?"
            report = self.localdb.select_one(sql, (_id,))
        except db.DbException as err: 
            log.error("Cant load report with _id %s from local database %s" % (_id, err))
            return None
        return report

    def getList(self, start=None):
        stop = start + datetime.timedelta(days=1)
        try:
            sql = "SELECT * FROM report WHERE start >= ? AND start < ? ORDER BY start"
            reports = self.localdb.select_all(sql, (start, stop))
            self.reports.clear()
            for r in reports:
                self.reports.append(r)
        except db.DbException as err:
            log.error("Cannot load list of reports from local database %s" % err)

        return self.reports

    def getUnsyncronisedCount(self):
        """
        Count number of reports in local database that is not syncronised with server
        """
        # todo, use count(*)
        sql = "SELECT count(*) FROM report WHERE server_id < 0"
        unsync_reports_count = self.localdb.count(sql) 
        return len(unsync_reports_count)
    
    def store(self, report):
        try:
            if report._id < 0:
                self.localdb.insert("report", d=report, primary_key="_id")
            else:
                self.localdb.update("report", d=report, primary_key="_id")
        except db.DbException as err:
            log.error("Cannot store report in local database %s" % err)
            return False
        self.sig.emit()
        if self._autosync:
            self.sync()
        return True

    def remove(self, report):
        ret = False
        try:
            if report.server_id != None and report.server_id >= 0:
                # Report exist on server, mark for removal - next sync will remove the row
                report.deleted = 1
                self.localdb.update("report", d=report, primary_key="_id")
                ret = True
            else:
                # Report does not exist on server, can be removed directly
                sql = "DELETE FROM report WHERE _id=?"
                self.localdb.delete(sql, (report._id))
            self.sig.emit()
            if self._autosync:
                self.sync()
        except db.DbException as err:
            log.error("Can't update local database when deleting report %s" % err)
        return ret

    def setAutosync(self, value):
        self._autosync = value
        if self._autosync:
            self.sync()

    def sync(self):
        """Sync the local database with the one on the server"""
        self.toThreadQ.put( ["sync"] )

    def stop(self):
        self.toThreadQ.put( ["quit"] )


##############################################################################
#<commit_msg>Fixes #16 Periodic sync of reports<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
Manage reports
'''

'''
Copyright (c) 2013, <NAME>, Abundo AB
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
   * Redistributions of source code must retain the above copyright
     notice, this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.
   * Neither the name of the <organization> nor the
     names of its contributors may be used to endorse or promote products
     derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

import datetime
import queue
import threading
import random

import PyQt5.QtCore as QtCore

from myglobals import *
from logger import log
from settings import sett

import util
import lib.db as db
import lib.network as network

class ReportMgr(QtCore.QObject):
    sig = QtCore.pyqtSignal()
    
    def __init__(self, localdb=None):
        super().__init__()
        self.localdb = localdb

        self.periodicsync_timer = None
        self._autosync = False
        self.reports = []   # local cache for todays reports
        self.toThreadQ = queue.Queue()
        self.t = threading.Thread(target=self.runThread)
        self.t.setName("ReportMgr")
        self.t.daemon = True
        self.t.start()
        
        sett.updated.connect(self.handle_settings)
        self.handle_settings()

    def handle_settings(self):
        """
        Handle changes in settings
        """
        if sett.report_sync_interval:
            if self.periodicsync_timer:
                # has the interval changed?
                if self.periodicsync_timer.interval != sett.report_sync_interval:
                    self.periodicsync_timer.cancel()
                    self.periodicsync_timer = None
            if self.periodicsync_timer is None:
                log.debug("ReportMgr starting autosync timer, interval %s" % sett.report_sync_interval)
                self._start_periodicsync_timer()
        else:
            if self.periodicsync_timer:
                log.debug("ReportMgr stopping autosync timer")
                self.periodicsync_timer.cancel()
            self.periodicsync_timer = None

    def _start_periodicsync_timer(self):
        jitter = sett.report_sync_interval // 10    # 10% jitter
        if jitter < 1:
            jitter = 1
        interval = sett.report_sync_interval + random.randint(-jitter, jitter) 
        log.debug("ReportMgr interval %s jitter %s" % (interval, jitter))
        self.periodicsync_timer = threading.Timer(interval, self.periodic_sync)
        self.periodicsync_timer.setName("ReportMgr.Timer")
        self.periodicsync_timer.daemon = True
        self.periodicsync_timer.start()

    def periodic_sync(self):
        log.debug("ReportMgr.periodic_sync triggered")
        self.sync()
        self._start_periodicsync_timer()

    def init(self):
        """Load the list of reports from local db"""
        self.sig.emit()

    def get(self, _id):
        for report in self.reports:
            if report._id == _id:
                return report
        try:
            sql = "SELECT * FROM report WHERE _id=?"
            report = self.localdb.select_one(sql, (_id,))
        except db.DbException as err: 
            log.error("Cant load report with _id %s from local database %s" % (_id, err))
            return None
        return report

    def getList(self, start=None):
        stop = start + datetime.timedelta(days=1)
        try:
            sql = "SELECT * FROM report WHERE start >= ? AND start < ? ORDER BY start"
            reports = self.localdb.select_all(sql, (start, stop))
            self.reports.clear()
            for r in reports:
                self.reports.append(r)
        except db.DbException as err:
            log.error("Cannot load list of reports from local database %s" % err)

        return self.reports

    def getUnsyncronisedCount(self):
        """
        Count number of reports in local database that is not syncronised with server
        """
        # todo, use count(*)
        sql = "SELECT count(*) FROM report WHERE server_id < 0"
        unsync_reports_count = self.localdb.count(sql) 
        return len(unsync_reports_count)
    
    def store(self, report):
        try:
            if report._id < 0:
                self.localdb.insert("report", d=report, primary_key="_id")
            else:
                self.localdb.update("report", d=report, primary_key="_id")
        except db.DbException as err:
            log.error("Cannot store report in local database %s" % err)
            return False
        self.sig.emit()
        if self._autosync:
            self.sync()
        return True

    def remove(self, report):
        ret = False
        try:
            if report.server_id != None and report.server_id >= 0:
                # Report exist on server, mark for removal - next sync will remove the row
                report.deleted = 1
                self.localdb.update("report", d=report, primary_key="_id")
                ret = True
            else:
                # Report does not exist on server, can be removed directly
                sql = "DELETE FROM report WHERE _id=?"
                self.localdb.delete(sql, (report._id))
            self.sig.emit()
            if self._autosync:
                self.sync()
        except db.DbException as err:
            log.error("Can't update local database when deleting report %s" % err)
        return ret

    def setAutosync(self, value):
        self._autosync = value
        if self._autosync:
            self.sync()

    def sync(self):
        """Sync the local database with the one on the server"""
        self.toThreadQ.put( ["sync"] )

    def stop(self):
        if self.periodicsync_timer and self.periodicsync_timer.is_alive():
            self.periodicsync_timer.cancel()
        self.toThreadQ.put( ["quit"] )


##############################################################################
#
{'fraction_non_alphanumeric': 0.09248132510555375, 'fraction_numerical': 0.0025982461838259174, 'mean_word_length': 2.3497416372042426, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '874829', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3194, 'n_words': 1170}