<commit_before># -*- coding: utf-8 -*-

import logging
import os
import threading
from datetime import datetime as dt

from odoo import _, api, fields, models

from .pymediainfo import MediaInfo

_logger = logging.getLogger(__name__)


class VideoFolderScan(models.TransientModel):
    _name = 'oovideo.folder.scan'
    _description = 'Video Folder Scan'

    ALLOWED_FILE_EXTENSIONS = {
        'avi',
        'm4v',
        'mkv',
        'mov',
        'mp4',
        'mpg',
        'webm',
    }

    def _lock_folder(self, folder_id):
        '''
        Check if a folder is locked. If it is not locked, lock it. If it is locked, log an error.

        :param int folder_id: ID of the folder to scan
        :return bool: False if the folder was already locked, True if it was not
        '''
        Folder = self.env['oovideo.folder'].browse([folder_id])
        if Folder.locked:
            _logger.error(
                "\"%s\" is locked! It probably means that a scan is ongoing.", Folder.path)
            res = False
        else:
            Folder.write({'locked': True})
            res = True

        self.env.cr.commit()
        return res

    def _clean_directory(self, path, user_id):
        '''
        Clean a directory. It removes folders and media which are not on the disk anymore. This
        can potentially deletes the folder linked to the given path if the path doesn't exist
        anymore.

        :param str path: path of the folder to clean
        :param int user_id: ID of the user to whom belongs the folder
        '''
        _logger.debug("Cleaning folder \"%s\"...", path)

        # List existing directories and files
        filelist = set(' ')
        folderlist = set(' ')
        for rootdir, dirnames, filenames in os.walk(path):
            folderlist.add(rootdir)
            for fn in filenames:
                fn_ext = fn.split('.')[-1]
                if fn_ext and fn_ext.lower() in self.ALLOWED_FILE_EXTENSIONS:
                    filelist.add(os.path.join(rootdir, fn))

        track_data = [
            (('id', 'path'), 'oovideo_folder', folderlist),
            (('id', 'path'), 'oovideo_media', filelist),
        ]

        # Cleaning part:
        # - select existing paths in table
        # - compare with paths actually used
        # - deletes the ones which are not used anymore
        for td in track_data:
            field_list = ','.join([x for x in td[0]])
            query = 'SELECT ' + field_list + ' FROM ' + td[1] + ' WHERE user_id = ' + str(user_id)
            self.env.cr.execute(query)
            res = self.env.cr.fetchall()
            to_clean = {r[1] for r in res if path in r[1]} - td[2]

            if to_clean:
                to_clean = {r[1]: r[0] for r in res if r[1] in to_clean}.values()
                self.env[td[1].replace('_', '.')].browse(to_clean).sudo().unlink()

    def _build_cache(self, folder_id, user_id):
        '''
        Builds the cache for a given folder. This avoids using the ORM cache which does not show
        the required performances for a large number of files. Only the necessary data is set in
        cache.

        :param int folder_id: ID of the folder to scan
        :param int user_id: ID of the user to whom belongs the folder
        :return dict: content of the cache
        '''
        _logger.debug("Building cache for folder_id \"%s\"...", folder_id)
        cache = {}
        cache['user_id'] = user_id
        params = (user_id, )

        query = "SELECT path, id, last_modification FROM oovideo_folder WHERE user_id = %s;"
        self.env.cr.execute(query, params)
        res = self.env.cr.fetchall()
        cache['folder'] = {r[0]: (r[1], r[2] or 0) for r in res}

        params = (user_id, folder_id)
        query = '''
            SELECT path, id, last_modification FROM oovideo_media
            WHERE user_id = %s AND root_folder_id = %s;
        '''
        self.env.cr.execute(query, params)
        res = self.env.cr.fetchall()
        cache['media'] = {r[0]: (r[1], r[2]) for r in res}

        return cache

    def _manage_dir(self, rootdir, cache):
        '''
        For a given directory, checks that it is already in the cache.
        - If not in the cache, create the associated folder
        - If in the cache, check the last modification date
        If the last modification date is older than the one recorded, the folder is skipped

        :param str rootdir: folder to check
        :param dict cache: reading cache
        :return dict, bool: tuple with content of the cache updated, and boolean which indicates if
            the folder can be skipped
        '''
        skip = False
        folder = cache['folder'].get(rootdir)
        mtime = int(os.path.getmtime(rootdir))

        if not folder:
            cache = self._create_folder(rootdir, cache)
        elif folder[1] >= mtime:
            skip = True
        else:
            parent_dir = os.sep.join(rootdir.split(os.sep)[:-1])
            parent_dir = cache['folder'].get(parent_dir, [False])
            Folder = self.env['oovideo.folder'].browse(folder[0])
            Folder.write({
                'last_modification': mtime,
                'parent_id': parent_dir[0],
            })
        return cache, skip

    def _create_folder(self, rootdir, cache):
        '''
        Create the directory rootdir, and updates the cache.

        :param str rootdir: path of the folder to create
        :param dict cache: reading cache
        :return dict: cache updated
        '''
        parent_dir = os.sep.join(rootdir.split(os.sep)[:-1])
        parent_dir = cache['folder'].get(parent_dir)
        if parent_dir:
            mtime = int(os.path.getmtime(rootdir))
            vals = {
                'root': False,
                'path': rootdir,
                'parent_id': parent_dir[0],
                'last_modification': mtime,
                'user_id': cache['user_id'],
            }
            Folder = self.env['oovideo.folder'].create(vals)
            cache['folder'][rootdir] = (Folder.id, mtime)

        return cache

    def _get_media_info(self, file_path):
        '''
        Get the infos of a media, thanks to MediaInfo.

        :param str file_path: path of the media to get the data from
        :return dict: media data
        '''
        try:
            media_info = MediaInfo.parse(file_path)
            vals = {
                'audio_tracks': 0,
                'audio_tracks_lang': [],
            }
            for track in media_info.tracks:
                if track.track_type == 'General':
                    vals['duration'] = int(float(track.duration))
                elif track.track_type == 'Video':
                    vals['height'] = track.height
                    vals['width'] = track.width
                    vals['bitrate'] = int(
                        (track.bit_rate or track.nominal_bit_rate or 0.0) / 1000.0)
                elif track.track_type == 'Audio':
                    vals['audio_tracks'] += 1
                    vals['audio_tracks_lang'] +=\
                        [str(vals['audio_tracks']) + ': ' + (track.language or _('Unknown'))]
        except:
            _logger.warning('Error while opening file "%s"', file_path, exc_info=1)
        return vals

    def _scan_folder(self, folder_id):
        '''
        The folder scanning method. It walks in all sub-directories of the folder. If the
        modification date is more recent than the recorded date, the directory is scanned.

        A file is scanned if these conditions are met:
        - the extension matches the allowed file extensions;
        - the last modification date is more recent than the recorded date, in order to update data
          of an existing record.
        During the scan, any new album or artists will be created as well.

        There is an arbitrary commit every 100 medias, which should allow a regular update of the
        database.

        :param int folder_id: ID of the folder to scan
        '''
        with api.Environment.manage(), self.pool.cursor() as cr:
            time_start = dt.now()<commit_msg>Improve robustness of scanning<commit_after># -*- coding: utf-8 -*-

import logging
import os
import threading
from datetime import datetime as dt

from odoo import _, api, fields, models

from .pymediainfo import MediaInfo

_logger = logging.getLogger(__name__)


class VideoFolderScan(models.TransientModel):
    _name = 'oovideo.folder.scan'
    _description = 'Video Folder Scan'

    ALLOWED_FILE_EXTENSIONS = {
        'avi',
        'm4v',
        'mkv',
        'mov',
        'mp4',
        'mpg',
        'webm',
    }

    def _lock_folder(self, folder_id):
        '''
        Check if a folder is locked. If it is not locked, lock it. If it is locked, log an error.

        :param int folder_id: ID of the folder to scan
        :return bool: False if the folder was already locked, True if it was not
        '''
        Folder = self.env['oovideo.folder'].browse([folder_id])
        if Folder.locked:
            _logger.error(
                "\"%s\" is locked! It probably means that a scan is ongoing.", Folder.path)
            res = False
        else:
            Folder.write({'locked': True})
            res = True

        self.env.cr.commit()
        return res

    def _clean_directory(self, path, user_id):
        '''
        Clean a directory. It removes folders and media which are not on the disk anymore. This
        can potentially deletes the folder linked to the given path if the path doesn't exist
        anymore.

        :param str path: path of the folder to clean
        :param int user_id: ID of the user to whom belongs the folder
        '''
        _logger.debug("Cleaning folder \"%s\"...", path)

        # List existing directories and files
        filelist = set(' ')
        folderlist = set(' ')
        for rootdir, dirnames, filenames in os.walk(path):
            folderlist.add(rootdir)
            for fn in filenames:
                fn_ext = fn.split('.')[-1]
                if fn_ext and fn_ext.lower() in self.ALLOWED_FILE_EXTENSIONS:
                    filelist.add(os.path.join(rootdir, fn))

        track_data = [
            (('id', 'path'), 'oovideo_folder', folderlist),
            (('id', 'path'), 'oovideo_media', filelist),
        ]

        # Cleaning part:
        # - select existing paths in table
        # - compare with paths actually used
        # - deletes the ones which are not used anymore
        for td in track_data:
            field_list = ','.join([x for x in td[0]])
            query = 'SELECT ' + field_list + ' FROM ' + td[1] + ' WHERE user_id = ' + str(user_id)
            self.env.cr.execute(query)
            res = self.env.cr.fetchall()
            to_clean = {r[1] for r in res if path in r[1]} - td[2]

            if to_clean:
                to_clean = {r[1]: r[0] for r in res if r[1] in to_clean}.values()
                self.env[td[1].replace('_', '.')].browse(to_clean).sudo().unlink()

    def _build_cache(self, folder_id, user_id):
        '''
        Builds the cache for a given folder. This avoids using the ORM cache which does not show
        the required performances for a large number of files. Only the necessary data is set in
        cache.

        :param int folder_id: ID of the folder to scan
        :param int user_id: ID of the user to whom belongs the folder
        :return dict: content of the cache
        '''
        _logger.debug("Building cache for folder_id \"%s\"...", folder_id)
        cache = {}
        cache['user_id'] = user_id
        params = (user_id, )

        query = "SELECT path, id, last_modification FROM oovideo_folder WHERE user_id = %s;"
        self.env.cr.execute(query, params)
        res = self.env.cr.fetchall()
        cache['folder'] = {r[0]: (r[1], r[2] or 0) for r in res}

        params = (user_id, folder_id)
        query = '''
            SELECT path, id, last_modification FROM oovideo_media
            WHERE user_id = %s AND root_folder_id = %s;
        '''
        self.env.cr.execute(query, params)
        res = self.env.cr.fetchall()
        cache['media'] = {r[0]: (r[1], r[2]) for r in res}

        return cache

    def _manage_dir(self, rootdir, cache):
        '''
        For a given directory, checks that it is already in the cache.
        - If not in the cache, create the associated folder
        - If in the cache, check the last modification date
        If the last modification date is older than the one recorded, the folder is skipped

        :param str rootdir: folder to check
        :param dict cache: reading cache
        :return dict, bool: tuple with content of the cache updated, and boolean which indicates if
            the folder can be skipped
        '''
        skip = False
        folder = cache['folder'].get(rootdir)
        mtime = int(os.path.getmtime(rootdir))

        if not folder:
            cache = self._create_folder(rootdir, cache)
        elif folder[1] >= mtime:
            skip = True
        else:
            parent_dir = os.sep.join(rootdir.split(os.sep)[:-1])
            parent_dir = cache['folder'].get(parent_dir, [False])
            Folder = self.env['oovideo.folder'].browse(folder[0])
            Folder.write({
                'last_modification': mtime,
                'parent_id': parent_dir[0],
            })
        return cache, skip

    def _create_folder(self, rootdir, cache):
        '''
        Create the directory rootdir, and updates the cache.

        :param str rootdir: path of the folder to create
        :param dict cache: reading cache
        :return dict: cache updated
        '''
        parent_dir = os.sep.join(rootdir.split(os.sep)[:-1])
        parent_dir = cache['folder'].get(parent_dir)
        if parent_dir:
            mtime = int(os.path.getmtime(rootdir))
            vals = {
                'root': False,
                'path': rootdir,
                'parent_id': parent_dir[0],
                'last_modification': mtime,
                'user_id': cache['user_id'],
            }
            Folder = self.env['oovideo.folder'].create(vals)
            cache['folder'][rootdir] = (Folder.id, mtime)

        return cache

    def _get_media_info(self, file_path):
        '''
        Get the infos of a media, thanks to MediaInfo.

        :param str file_path: path of the media to get the data from
        :return dict: media data
        '''
        try:
            media_info = MediaInfo.parse(file_path)
            vals = {
                'audio_tracks': 0,
                'audio_tracks_lang': [],
            }
            for track in media_info.tracks:
                if track.track_type == 'General':
                    vals['duration'] = int(float(track.duration or 0.0))
                elif track.track_type == 'Video':
                    vals['height'] = track.height
                    vals['width'] = track.width
                    vals['bitrate'] = int(
                        float(track.bit_rate or track.nominal_bit_rate or 0.0) / 1000.0)
                elif track.track_type == 'Audio':
                    vals['audio_tracks'] += 1
                    vals['audio_tracks_lang'] +=\
                        [str(vals['audio_tracks']) + ': ' + (track.language or _('Unknown'))]
        except:
            _logger.warning('Error while opening file "%s"', file_path, exc_info=1)
        return vals

    def _scan_folder(self, folder_id):
        '''
        The folder scanning method. It walks in all sub-directories of the folder. If the
        modification date is more recent than the recorded date, the directory is scanned.

        A file is scanned if these conditions are met:
        - the extension matches the allowed file extensions;
        - the last modification date is more recent than the recorded date, in order to update data
          of an existing record.
        During the scan, any new album or artists will be created as well.

        There is an arbitrary commit every 100 medias, which should allow a regular update of the
        database.

        :param int folder_id: ID of the folder to scan
        '''
        with api.Environment.manage(), self.pool.cursor() as cr:
            time_start = dt.now()
{'fraction_non_alphanumeric': 0.10221079061791905, 'fraction_numerical': 0.004899258987078205, 'mean_word_length': 1.869442980144087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1528754', 'n_tokens_mistral': 4726, 'n_tokens_neox': 4511, 'n_words': 1550}