<commit_before>'''
maintains a listing of all series currently being watched.
has methods to check the filesystem/rss for new downloads to refresh its data.
also has methods to move/play the files and start the anidb add process.
basically this is the controller class for all the other classes.
'''
import sql
import rss
import os
import anidb
if os.name=='nt':
    import makeico
import re
import torrentprogress
import StringIO
import shutil
from bencode import BTFailure
import logging
import errno
from shana_interface import ShanaLink
import stat

class SeriesList:
    series=None
    SQL=None
    RSS=None
    user_settings=None
    def __init__(self):
        if os.path.exists('DEBUG_TEST'):
            logging.basicConfig(level=logging.DEBUG, filename='DEBUG.log')
        else:
            logging.basicConfig(level=logging.DEBUG, stream=StringIO.StringIO())
            logging.disable(logging.DEBUG)
        self.SQL=sql.SQLManager()
        self.SQL.connect()
        self.RSS=rss.RSSReader()
        self.series={}
        self.SHANALINK = ShanaLink()
        #just init the bare minimum things.
        self._getUserSettings()
        self._populateSeries()
##        self.doUpdates(True)

    def _populateSeries(self):
        self.series = self.SQL.getSeries()

    def DownloadTorrents(self):
        for episode in self.removeInvalidTorrents:
            self.SQL.removeEpisode(episode[0],episode[1])
        for data in self.torrentDatum:
            self.SQL.addTorrentData(data[0],data[1],data[2],data[3])
        return len(self.torrentDatum) + len(self.removeInvalidTorrents)
            
    def iDownloadTorrents(self):
        'download all new torrent files'
        self.torrentDatum = []
        self.removeInvalidTorrents = []
        for entry in self.newEntries:
            torrent = torrentprogress.download_torrent(entry[2])
            if torrent:
                torrentdata=buffer(torrent)
                try:
                    filename = torrentprogress.file_name(StringIO.StringIO(torrentdata))
                except BTFailure,e:
                    print 'initial bencode failed %r'%e
                    if str(e)=='not a valid bencoded string' or str(e)=='torrent contains multiple files':
                        print 'pre-removing %s.'%entry[1]
                        self.removeInvalidTorrents.append((entry[2],len(torrentdata)))
                        continue
                path = os.path.join(self.user_settings['Download Directory'],filename)
                self.torrentDatum.append((path,entry[2],torrentdata,filename))
            
        return len(self.torrentDatum)

    def prepLoadRSS(self):
        self.torrentBlacklist = self.SQL.getTorrentBlacklist()
    
    def loadRSS(self):
        '''downloads RSS feed and loads its contents into the db'''
        self.newEntries = []
        if not self.user_settings:
            return -1
        for item in self.rssitems:
            if item[2] not in self.torrentBlacklist:
                                                                                            #file name, rsstitle,torrent url 
                if self.SQL.addEpisode(os.path.join(self.user_settings['Download Directory'],item[1]),item[0],item[2]):
                    self.newEntries.append(item)
        return len(self.newEntries)

    def iloadRSS(self):
        if not self.user_settings:
            return -1
        self.rssitems=self.RSS.getFiles(self.user_settings['RSS Feed'])

    def checkFiles(self):
        '''Scans the download directory for any matches'''
        if not self.user_settings:
            return 0
        for episode in self.changedFiles:
            self.SQL.setDownloading(episode[0],episode[1],episode[2],episode[3])
        for episode in self.removeInvalid:
            self.SQL.removeEpisode(episode)
        return len(self.changedFiles)

    
    
    # STRONGLY recommended that you call populateseries before running this method.
    def icheckFiles(self):
##        import time
##        start = time.time()
        '''Scans the download directory for any matches'''
        self.changedFiles=[]
        self.removeInvalid=[]
        if not self.user_settings:
            return -1
        dl_dir = self.user_settings['Download Directory']#.decode('utf8')
##        potentialFiles=os.listdir(dl_dir)
        # we replace space with _ here, make sure to do this to all the strings you want to match.
        originalFiles = os.listdir(dl_dir)
        potentialFiles = map(lambda x: re.sub(ur'[ ]',ur'_',x), originalFiles)
        potentialMatches = dict(zip(potentialFiles,originalFiles))
##        contString=u'\n'.join(potentialFiles)+u'\n'
        for series in self.series.values():
            for episode in series:
                # narrow the search to undownloaded, unwatched series to save time. (also you can ignore hidden series)
                if episode['downloaded']==0 and episode['watched']==0 and episode['hidden']==0:
##                    # basically we want to match the episode name excepting '_' for ' ' if necessary
##                    # meaning 'the game' will match to 'the game' or 'the_game'
##                    workingFile = episode['file_name']#.decode('utf8')
##                    # we also need to check for invalid filename chars, which shanaproject replaces with _ (probably)
##                    # ur'\/:*?"<>|'
##                    matcher = re.compile(ur'('+
##                                         re.sub(ur'\\ ',ur'[ _]',re.escape(re.sub(ur'[\/:*?"<>|]',ur'_',workingFile)))
##                                         +u')\n')#(\.!ut)?\n')
##                    match = matcher.search(contString)
                    pattern,replacement = self.RSS.invalidCharReplacement(self.user_settings['RSS Feed'])                    
                    workingFile = re.sub(pattern,replacement,episode['file_name'])
##                    if match:
                    if workingFile in potentialMatches:
                        filename = potentialMatches[workingFile]
                        # we only want to find NEWLY downloaded files so only check those that aren't marked.
##                        if episode['downloaded']==0:# and workingFile in potentialFiles:
                        #this is a downloading file that is just not completely downloaded.
##                        path=os.path.join(dl_dir,match.group(1))
                        path=os.path.join(dl_dir,filename)
                        torrentdata=episode['torrent_data']
                        percent_downloaded=episode['download_percent']

                        if not torrentdata:
                            torrent = torrentprogress.download_torrent(episode['torrent_url'])
                            if torrent:
                                torrentdata=buffer(torrent)
                        if torrentdata:
##                            if match.group(2):
##                                path = path+match.group(2)
                            try:
                                percent_downloaded, torrentdata=torrentprogress.percentCompleted(StringIO.StringIO(torrentdata),path)
                            except BTFailure,e:
                                print 'bencode failed %r'%e
                                if str(e)=='not a valid bencoded string' and len(torrentdata):
                                    print 'episode (%s) was removed.'%episode['display_name']
                                    self.removeInvalid.append(episode['torrent_url'])
                                    continue
                                
##                            print 'torrent hashing took %f seconds'%(time.time()-star)
##                        self.changedFiles.append((episode['torrent_url'],match.group(1),torrentdata,percent_downloaded))
                        self.changedFiles.append((episode['torrent_url'],filename,torrentdata,percent_downloaded))
##        print 'checkfiles took %f seconds'%(time.time()-start)
        
    def prepCheckFiles(self):
        return
    
    def watchfile(self,filepath):
        import sys,subprocess, os
        if sys.platform.startswith('darwin'):
            subprocess.call(('open', filepath))
        elif os.name == 'nt':
            os.startfile(filepath)
        elif os.name == 'posix':
            subprocess.call(('xdg-open', filepath))

    def dropSeries(self, title, delete):
        self.SQL.hideSeries(title)
        if delete:
            paths = self.SQL.getAllPaths(title)
            for path in paths:
                directory = os.path.dirname(path[0])
                if os.path.isfile(path[0]):
                    os.remove(path[0])
                self.cleanFolder(directory)
            self.SQL.dropSeries(title)

##    def dropSeries(self, title, delete):
##        try:
##            if self.SHANALINK.delete_follow(title):
##                #only hide if successful.
##                self.SQL.hideSeries(title)
##                if delete:
##                    import shutil
##                    paths = self.SQL.getAllPaths(title)
##                    for path in paths:
##                        directory = os.path.dirname(path[0])
##                        if os.path.exists(directory):
####                            shutil.rmtree(directory, ignore_errors=True)
##                            print 'rming %s'%directory
##                    
##                return 1
##        except:
##            raise
##            return 0
##        return 0

    def validShanaProjectCredentials(self):
        return not not self.user_settings['Shana Project Username'] and not not self.user_settings['Shana Project Password']
    
    #data in this case is the list that can be obtained from self.series
    #it contains all the info available about a specific episode, making it easy to manipulate it.
    def prepPlayAndSort(self):
        '''move/sort the episode into its proper folder then play it in the default media player.
            also queue the file for anidb add and update its watched status in the db'''
        self._getUserSettings()
        self._populateSeries()
        if not self.user_settings:
            return -1
        return 0
    
    def playAndSort(self,data):
        if data['watched']:
            self.watchfile(data['path'])#.decode('utf8'))
            return 'watched','watched'
        import shutil
        shana_title = data['title']#self.SQL.getSeriesTitle(data['id'])#.decode('utf8')
        st_dir = self.user_settings['Save Directory']#.decode('utf8')
        folder_title = ur''.join(i for i in shana_title if i not in ur'\/:*?"<>|.')

        # check usersettings for usesubfolders.
        # if true/false move existing files from one to the other

        toplevel_folder = os.path.join(st_dir,folder_title)
        season=data['season']#self.SQL.getSeriesSeason(data['id'])
        if season:
            year = season.split()[1]
            seasonsorted_folder = os.path.join(st_dir,year,season,folder_title)
            
        
        if self.user_settings['Season Sort'] and season:
            dest_folder = seasonsorted_folder
        else:
            dest_folder = toplevel_folder

        dest_file = os.path.join(dest_folder,data['file_name'])
        
        if not os.path.isdir(dest_folder):
            os.makedirs(dest_folder)
        if not os.path.exists(dest_file):
            shutil.move(data['path'],dest_file)#.decode('utf8'),dest_file)
        self.watchfile(dest_file)
##        self.SQL.watchMoveQueue(data['path'],dest_file)

        # we also want to get the ed2k hash asap in case you decide to drop the series right after watching an episode.
        try:<commit_msg>fixed bug when rewatching a file<commit_after>'''
maintains a listing of all series currently being watched.
has methods to check the filesystem/rss for new downloads to refresh its data.
also has methods to move/play the files and start the anidb add process.
basically this is the controller class for all the other classes.
'''
import sql
import rss
import os
import anidb
if os.name=='nt':
    import makeico
import re
import torrentprogress
import StringIO
import shutil
from bencode import BTFailure
import logging
import errno
from shana_interface import ShanaLink
import stat

class SeriesList:
    series=None
    SQL=None
    RSS=None
    user_settings=None
    def __init__(self):
        if os.path.exists('DEBUG_TEST'):
            logging.basicConfig(level=logging.DEBUG, filename='DEBUG.log')
        else:
            logging.basicConfig(level=logging.DEBUG, stream=StringIO.StringIO())
            logging.disable(logging.DEBUG)
        self.SQL=sql.SQLManager()
        self.SQL.connect()
        self.RSS=rss.RSSReader()
        self.series={}
        self.SHANALINK = ShanaLink()
        #just init the bare minimum things.
        self._getUserSettings()
        self._populateSeries()
##        self.doUpdates(True)

    def _populateSeries(self):
        self.series = self.SQL.getSeries()

    def DownloadTorrents(self):
        for episode in self.removeInvalidTorrents:
            self.SQL.removeEpisode(episode[0],episode[1])
        for data in self.torrentDatum:
            self.SQL.addTorrentData(data[0],data[1],data[2],data[3])
        return len(self.torrentDatum) + len(self.removeInvalidTorrents)
            
    def iDownloadTorrents(self):
        'download all new torrent files'
        self.torrentDatum = []
        self.removeInvalidTorrents = []
        for entry in self.newEntries:
            torrent = torrentprogress.download_torrent(entry[2])
            if torrent:
                torrentdata=buffer(torrent)
                try:
                    filename = torrentprogress.file_name(StringIO.StringIO(torrentdata))
                except BTFailure,e:
                    print 'initial bencode failed %r'%e
                    if str(e)=='not a valid bencoded string' or str(e)=='torrent contains multiple files':
                        print 'pre-removing %s.'%entry[1]
                        self.removeInvalidTorrents.append((entry[2],len(torrentdata)))
                        continue
                path = os.path.join(self.user_settings['Download Directory'],filename)
                self.torrentDatum.append((path,entry[2],torrentdata,filename))
            
        return len(self.torrentDatum)

    def prepLoadRSS(self):
        self.torrentBlacklist = self.SQL.getTorrentBlacklist()
    
    def loadRSS(self):
        '''downloads RSS feed and loads its contents into the db'''
        self.newEntries = []
        if not self.user_settings:
            return -1
        for item in self.rssitems:
            if item[2] not in self.torrentBlacklist:
                                                                                            #file name, rsstitle,torrent url 
                if self.SQL.addEpisode(os.path.join(self.user_settings['Download Directory'],item[1]),item[0],item[2]):
                    self.newEntries.append(item)
        return len(self.newEntries)

    def iloadRSS(self):
        if not self.user_settings:
            return -1
        self.rssitems=self.RSS.getFiles(self.user_settings['RSS Feed'])

    def checkFiles(self):
        '''Scans the download directory for any matches'''
        if not self.user_settings:
            return 0
        for episode in self.changedFiles:
            self.SQL.setDownloading(episode[0],episode[1],episode[2],episode[3])
        for episode in self.removeInvalid:
            self.SQL.removeEpisode(episode)
        return len(self.changedFiles)

    
    
    # STRONGLY recommended that you call populateseries before running this method.
    def icheckFiles(self):
##        import time
##        start = time.time()
        '''Scans the download directory for any matches'''
        self.changedFiles=[]
        self.removeInvalid=[]
        if not self.user_settings:
            return -1
        dl_dir = self.user_settings['Download Directory']#.decode('utf8')
##        potentialFiles=os.listdir(dl_dir)
        # we replace space with _ here, make sure to do this to all the strings you want to match.
        originalFiles = os.listdir(dl_dir)
        potentialFiles = map(lambda x: re.sub(ur'[ ]',ur'_',x), originalFiles)
        potentialMatches = dict(zip(potentialFiles,originalFiles))
##        contString=u'\n'.join(potentialFiles)+u'\n'
        for series in self.series.values():
            for episode in series:
                # narrow the search to undownloaded, unwatched series to save time. (also you can ignore hidden series)
                if episode['downloaded']==0 and episode['watched']==0 and episode['hidden']==0:
##                    # basically we want to match the episode name excepting '_' for ' ' if necessary
##                    # meaning 'the game' will match to 'the game' or 'the_game'
##                    workingFile = episode['file_name']#.decode('utf8')
##                    # we also need to check for invalid filename chars, which shanaproject replaces with _ (probably)
##                    # ur'\/:*?"<>|'
##                    matcher = re.compile(ur'('+
##                                         re.sub(ur'\\ ',ur'[ _]',re.escape(re.sub(ur'[\/:*?"<>|]',ur'_',workingFile)))
##                                         +u')\n')#(\.!ut)?\n')
##                    match = matcher.search(contString)
                    pattern,replacement = self.RSS.invalidCharReplacement(self.user_settings['RSS Feed'])                    
                    workingFile = re.sub(pattern,replacement,episode['file_name'])
##                    if match:
                    if workingFile in potentialMatches:
                        filename = potentialMatches[workingFile]
                        # we only want to find NEWLY downloaded files so only check those that aren't marked.
##                        if episode['downloaded']==0:# and workingFile in potentialFiles:
                        #this is a downloading file that is just not completely downloaded.
##                        path=os.path.join(dl_dir,match.group(1))
                        path=os.path.join(dl_dir,filename)
                        torrentdata=episode['torrent_data']
                        percent_downloaded=episode['download_percent']

                        if not torrentdata:
                            torrent = torrentprogress.download_torrent(episode['torrent_url'])
                            if torrent:
                                torrentdata=buffer(torrent)
                        if torrentdata:
##                            if match.group(2):
##                                path = path+match.group(2)
                            try:
                                percent_downloaded, torrentdata=torrentprogress.percentCompleted(StringIO.StringIO(torrentdata),path)
                            except BTFailure,e:
                                print 'bencode failed %r'%e
                                if str(e)=='not a valid bencoded string' and len(torrentdata):
                                    print 'episode (%s) was removed.'%episode['display_name']
                                    self.removeInvalid.append(episode['torrent_url'])
                                    continue
                                
##                            print 'torrent hashing took %f seconds'%(time.time()-star)
##                        self.changedFiles.append((episode['torrent_url'],match.group(1),torrentdata,percent_downloaded))
                        self.changedFiles.append((episode['torrent_url'],filename,torrentdata,percent_downloaded))
##        print 'checkfiles took %f seconds'%(time.time()-start)
        
    def prepCheckFiles(self):
        return
    
    def watchfile(self,filepath):
        import sys,subprocess, os
        if sys.platform.startswith('darwin'):
            subprocess.call(('open', filepath))
        elif os.name == 'nt':
            os.startfile(filepath)
        elif os.name == 'posix':
            subprocess.call(('xdg-open', filepath))

    def dropSeries(self, title, delete):
        self.SQL.hideSeries(title)
        if delete:
            paths = self.SQL.getAllPaths(title)
            for path in paths:
                directory = os.path.dirname(path[0])
                if os.path.isfile(path[0]):
                    os.remove(path[0])
                self.cleanFolder(directory)
            self.SQL.dropSeries(title)

##    def dropSeries(self, title, delete):
##        try:
##            if self.SHANALINK.delete_follow(title):
##                #only hide if successful.
##                self.SQL.hideSeries(title)
##                if delete:
##                    import shutil
##                    paths = self.SQL.getAllPaths(title)
##                    for path in paths:
##                        directory = os.path.dirname(path[0])
##                        if os.path.exists(directory):
####                            shutil.rmtree(directory, ignore_errors=True)
##                            print 'rming %s'%directory
##                    
##                return 1
##        except:
##            raise
##            return 0
##        return 0

    def validShanaProjectCredentials(self):
        return not not self.user_settings['Shana Project Username'] and not not self.user_settings['Shana Project Password']
    
    #data in this case is the list that can be obtained from self.series
    #it contains all the info available about a specific episode, making it easy to manipulate it.
    def prepPlayAndSort(self):
        '''move/sort the episode into its proper folder then play it in the default media player.
            also queue the file for anidb add and update its watched status in the db'''
        self._getUserSettings()
        self._populateSeries()
        if not self.user_settings:
            return -1
        return 0
    
    def playAndSort(self,data):
        if data['watched']:
            self.watchfile(data['path'])#.decode('utf8'))
            return ('watched',)*4
        import shutil
        shana_title = data['title']#self.SQL.getSeriesTitle(data['id'])#.decode('utf8')
        st_dir = self.user_settings['Save Directory']#.decode('utf8')
        folder_title = ur''.join(i for i in shana_title if i not in ur'\/:*?"<>|.')

        # check usersettings for usesubfolders.
        # if true/false move existing files from one to the other

        toplevel_folder = os.path.join(st_dir,folder_title)
        season=data['season']#self.SQL.getSeriesSeason(data['id'])
        if season:
            year = season.split()[1]
            seasonsorted_folder = os.path.join(st_dir,year,season,folder_title)
            
        
        if self.user_settings['Season Sort'] and season:
            dest_folder = seasonsorted_folder
        else:
            dest_folder = toplevel_folder

        dest_file = os.path.join(dest_folder,data['file_name'])
        
        if not os.path.isdir(dest_folder):
            os.makedirs(dest_folder)
        if not os.path.exists(dest_file):
            shutil.move(data['path'],dest_file)#.decode('utf8'),dest_file)
        self.watchfile(dest_file)
##        self.SQL.watchMoveQueue(data['path'],dest_file)

        # we also want to get the ed2k hash asap in case you decide to drop the series right after watching an episode.
        try:
{'fraction_non_alphanumeric': 0.10153518123667378, 'fraction_numerical': 0.004051172707889125, 'mean_word_length': 1.869676945668135, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 78, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5538614', 'n_tokens_mistral': 6383, 'n_tokens_neox': 6098, 'n_words': 1512}