<commit_before># nvPY: cross-platform note-taking app with simplenote syncing
# copyright 2012 by <NAME> <<EMAIL>>
# new BSD license

import glob
import os
import json
import re
from simplenote import Simplenote
import time
import utils

class NotesDB:
    """NotesDB will take care of the local notes database and syncing with SN.
    """
    def __init__(self, db_path, sn_username, sn_password):
        # create db dir if it does not exist
        if not os.path.exists(db_path):
            os.mkdir(db_path)
            
        self.db_path = db_path
            
        # now read all .json files from disk
        fnlist = glob.glob(self.helper_key_to_fname('*'))
        self.notes = {}
        for fn in fnlist:
            n = json.load(open(fn, 'rb'))
            # filename is ALWAYS localkey; only simplenote key when available
            localkey = os.path.splitext(os.path.basename(fn))[0]
            self.notes[localkey] = n
            
        
        # initialise the simplenote instance we're going to use
        # this does not yet need network access
        self.simplenote = Simplenote(sn_username, sn_password)
        
        # first line with non-whitespace should be the title
        self.title_re = re.compile('\s*(.*)\n?')
        
    def create_note(self, title):
        # need to get a key unique to this database. not really important
        # what it is, as long as it's unique.
        new_key = utils.generate_random_key()
        while new_key in self.notes:
            new_key = utils.generate_random_key()
            
        timestamp = time.time()
            
        new_note = {'key' : new_key,
                    'content' : title,
                    'modifydate' : timestamp,
                    'createdate' : timestamp,
                    'saved' : False, # has this been written to disc?
                    'synced' : False # has it been synced with server?
                    }
        
        self.notes[new_key] = new_note
        
        return new_key
        
    def get_note_names(self, search_string=None):
        """Return 
        """
        
        note_names = []
        for k in self.notes:
            n = self.notes[k]
            c = n.get('content')
            tmo = self.title_re.match(c)
            if tmo and (not search_string or re.search(search_string, c)):
                title = tmo.groups()[0]
                
                # timestamp
                # convert to datetime with datetime.datetime.fromtimestamp(modified)
                modified = float(n.get('modifydate'))

                o = utils.KeyValueObject(key=k, title=title, modified=modified)
                note_names.append(o)
            
        # we could sort note_names here
        return note_names
    
    def get_note_content(self, key):
        return self.notes[key].get('content')
    
    def helper_key_to_fname(self, k):
        return os.path.join(self.db_path, k) + '.json'
    
    def helper_save_note(self, k, note):
        fn = self.helper_key_to_fname(k)
        json.dump(note, open(fn, 'wb'), indent=2)
        
    def helper_sync_note(self, k, note):
        uret = self.simplenote.update_note(note)
        if uret[1] == 0:
            # success!
            n = uret[0]
            # if content was unchanged, there'll be no content sent back!
            # so we have to copy our old content
            if not n.get('content', None):
                n['content'] = note['content']
                # FIXME: record that content has not changed
                # then we know GUI does not have to be updated either.
                
            if n.get('key') != k:
                # new key assigned during sync
                # 1. delete from our notes list
                del self.notes[k]
                # 2. remove from filesystem
                os.unlink(self.helper_key_to_fname(k))
                # set us up with new key
                k = n.get('key')
            
            # store the new n at k, possibly new
            self.notes[k] = n
            # return the key
            return k
            
        else:
            return None
        
        
    def save(self):
        """Write all notes that have been changed since last save to disc.
        
        This is usually called every few seconds by nvPY, so it should be quick.
        """
        nsaved = 0
        for k,n in self.notes.items():
            if not n.get('saved', True):
                n['saved'] = True
                self.helper_save_note(k, n)
                nsaved += 1
                
        return nsaved
    
    def sync_partial(self):
        """Only sync notes that have been changed / created locally since previous sync.
        """
        
        nsynced = 0
        nerrored = 0
        for k,n in self.notes.items():
            if not n.get('synced', True): # either note has Synced=False, or nothing, which means it's synced!
                k = self.helper_sync_note(k,n)
                
                if k:
                    n = self.notes[k]
                    n['synced'] = True
                    nsynced += 1
                    self.helper_save_note(k, n)
                    
                    
                else:
                    nerrored += 1
                
        return (nsynced, nerrored)
    
    def sync_full(self):
        local_updates = {}
        local_deletes = {}

        print "step 1"
        # 1. go through local notes, if anything changed or new, update to server
        for lk in self.notes.keys():
            n = self.notes[lk]
            if not n.get('key') or not n.get('synced', False):
                uret = self.simplenote.update_note(n)
                if uret[1] == 0:
                    # replace n with uret[0]
                    # if this was a new note, our local key is not valid anymore
                    del self.notes[lk]
                    # in either case (new or existing note), save note at assigned key
                    k = uret[0].get('key')
                    self.notes[k] = uret[0]
                    
                    # whatever the case may be, k is now updated
                    local_updates[k] = True
                    if lk != k:
                        # if lk was a different (purely local) key, should be deleted
                        local_deletes[lk] = True
             
        print "step 2"
        # 2. if remote syncnum > local syncnum, update our note; if key is new, add note to local.
        # this gets the FULL note list, even if multiple gets are required       
        nl = self.simplenote.get_note_list()
        if nl[1] == 0:
            nl = nl[0]
            
        else:
            raise RuntimeError('Could not get note list from server.')
        
        print "  got note list."
            
        server_keys = {}
        for n in nl:
            k = n.get('key')
            server_keys[k] = True
            if k in self.notes:
                # we already have this
                if n.get('syncnum') > self.notes[k]:
                    # and the server is newer
                    print "  getting newer note", k
                    ret = self.simplenote.get_note(k)
                    if ret[1] == 0:
                        self.notes[k] = ret[0]
                        local_updates[k] = True
                        
            else:
                # new note
                print "  getting new note", k
                ret = self.simplenote.get_note(k)
                if ret[1] == 0:
                    self.notes[k] = ret[0]
                    local_updates[k] = True
                     
        print "step 3"
        # 3. for each local note not in server index, remove.     
        for lk in self.notes.keys():
            if lk not in server_keys:
                del self.notes[lk]
                local_deletes[lk] = True
                
        # sync done, now write changes to db_path
        for uk in local_updates.keys():
            self.helper_save_note(uk, self.notes[uk])
            
        for dk in local_deletes.keys():
            os.unlink(self.helper_key_to_fname(dk))
            
        print "done syncin'"
        
        
    def do_full_get(self):
        # this returns a tuple ([], -1) if wrong password
        # on success, ([docs], 0)
        # each doc:
        #{u'createdate': u'1335860754.841000',
        # u'deleted': 0,
        # u'key': u'455f66ee936711e19657591a71011082',
        # u'minversion': 10,
        # u'modifydate': u'1337007469.836000',
        # u'syncnum': 54,
        # u'systemtags': [],
        # u'tags': [],
        # u'version': 40}
        
        note_list = self.simplenote.get_note_list()
        
        # simplenote.get_note(key) returns (doc, status)
        # where doc is all of the above with extra field content

        server_notes = []        
        if note_list[1] == 0:
            for i in note_list[0]:
                n = self.simplenote.get_note(i.get('key'))
                if n[1] == 0:
                    server_notes.append(n[0])
                    
        # write server_notes to disc
        for n in server_notes:
            f = open(os.path.join(self.db_path, n.get('key')) + '.json', 'wb')
            json.dump(n, f, indent=2)

    def set_note_content(self, key, content):
        n = self.notes[key]
        old_content = n.get('content')
        if content != old_content:
            n['content'] = content
            n['modifydate'] = time.time()
            # we've made changes, so this needs to be written to disc and synced
            n['saved'] = n['synced'] = False
<commit_msg>Testing alpha sorting for notes list.<commit_after># nvPY: cross-platform note-taking app with simplenote syncing
# copyright 2012 by <NAME> <<EMAIL>>
# new BSD license

import glob
import os
import json
import re
from simplenote import Simplenote
import time
import utils

class NotesDB:
    """NotesDB will take care of the local notes database and syncing with SN.
    """
    def __init__(self, db_path, sn_username, sn_password):
        # create db dir if it does not exist
        if not os.path.exists(db_path):
            os.mkdir(db_path)
            
        self.db_path = db_path
            
        # now read all .json files from disk
        fnlist = glob.glob(self.helper_key_to_fname('*'))
        self.notes = {}
        for fn in fnlist:
            n = json.load(open(fn, 'rb'))
            # filename is ALWAYS localkey; only simplenote key when available
            localkey = os.path.splitext(os.path.basename(fn))[0]
            self.notes[localkey] = n
            
        
        # initialise the simplenote instance we're going to use
        # this does not yet need network access
        self.simplenote = Simplenote(sn_username, sn_password)
        
        # first line with non-whitespace should be the title
        self.title_re = re.compile('\s*(.*)\n?')
        
    def create_note(self, title):
        # need to get a key unique to this database. not really important
        # what it is, as long as it's unique.
        new_key = utils.generate_random_key()
        while new_key in self.notes:
            new_key = utils.generate_random_key()
            
        timestamp = time.time()
            
        new_note = {'key' : new_key,
                    'content' : title,
                    'modifydate' : timestamp,
                    'createdate' : timestamp,
                    'saved' : False, # has this been written to disc?
                    'synced' : False # has it been synced with server?
                    }
        
        self.notes[new_key] = new_note
        
        return new_key
        
    def get_note_names(self, search_string=None):
        """Return 
        """
        
        note_names = []
        for k in self.notes:
            n = self.notes[k]
            c = n.get('content')
            tmo = self.title_re.match(c)
            if tmo and (not search_string or re.search(search_string, c)):
                title = tmo.groups()[0]
                
                # timestamp
                # convert to datetime with datetime.datetime.fromtimestamp(modified)
                modified = float(n.get('modifydate'))

                o = utils.KeyValueObject(key=k, title=title, modified=modified)
                note_names.append(o)
            
        # we could sort note_names here
        note_names.sort(key=lambda o: o.title)
        return note_names
    
    def get_note_content(self, key):
        return self.notes[key].get('content')
    
    def helper_key_to_fname(self, k):
        return os.path.join(self.db_path, k) + '.json'
    
    def helper_save_note(self, k, note):
        fn = self.helper_key_to_fname(k)
        json.dump(note, open(fn, 'wb'), indent=2)
        
    def helper_sync_note(self, k, note):
        uret = self.simplenote.update_note(note)
        if uret[1] == 0:
            # success!
            n = uret[0]
            # if content was unchanged, there'll be no content sent back!
            # so we have to copy our old content
            if not n.get('content', None):
                n['content'] = note['content']
                # FIXME: record that content has not changed
                # then we know GUI does not have to be updated either.
                
            if n.get('key') != k:
                # new key assigned during sync
                # 1. delete from our notes list
                del self.notes[k]
                # 2. remove from filesystem
                os.unlink(self.helper_key_to_fname(k))
                # set us up with new key
                k = n.get('key')
            
            # store the new n at k, possibly new
            self.notes[k] = n
            # return the key
            return k
            
        else:
            return None
        
        
    def save(self):
        """Write all notes that have been changed since last save to disc.
        
        This is usually called every few seconds by nvPY, so it should be quick.
        """
        nsaved = 0
        for k,n in self.notes.items():
            if not n.get('saved', True):
                n['saved'] = True
                self.helper_save_note(k, n)
                nsaved += 1
                
        return nsaved
    
    def sync_partial(self):
        """Only sync notes that have been changed / created locally since previous sync.
        """
        
        nsynced = 0
        nerrored = 0
        for k,n in self.notes.items():
            if not n.get('synced', True): # either note has Synced=False, or nothing, which means it's synced!
                k = self.helper_sync_note(k,n)
                
                if k:
                    n = self.notes[k]
                    n['synced'] = True
                    nsynced += 1
                    self.helper_save_note(k, n)
                    
                    
                else:
                    nerrored += 1
                
        return (nsynced, nerrored)
    
    def sync_full(self):
        local_updates = {}
        local_deletes = {}

        print "step 1"
        # 1. go through local notes, if anything changed or new, update to server
        for lk in self.notes.keys():
            n = self.notes[lk]
            if not n.get('key') or not n.get('synced', False):
                uret = self.simplenote.update_note(n)
                if uret[1] == 0:
                    # replace n with uret[0]
                    # if this was a new note, our local key is not valid anymore
                    del self.notes[lk]
                    # in either case (new or existing note), save note at assigned key
                    k = uret[0].get('key')
                    self.notes[k] = uret[0]
                    
                    # whatever the case may be, k is now updated
                    local_updates[k] = True
                    if lk != k:
                        # if lk was a different (purely local) key, should be deleted
                        local_deletes[lk] = True
             
        print "step 2"
        # 2. if remote syncnum > local syncnum, update our note; if key is new, add note to local.
        # this gets the FULL note list, even if multiple gets are required       
        nl = self.simplenote.get_note_list()
        if nl[1] == 0:
            nl = nl[0]
            
        else:
            raise RuntimeError('Could not get note list from server.')
        
        print "  got note list."
            
        server_keys = {}
        for n in nl:
            k = n.get('key')
            server_keys[k] = True
            if k in self.notes:
                # we already have this
                if n.get('syncnum') > self.notes[k]:
                    # and the server is newer
                    print "  getting newer note", k
                    ret = self.simplenote.get_note(k)
                    if ret[1] == 0:
                        self.notes[k] = ret[0]
                        local_updates[k] = True
                        
            else:
                # new note
                print "  getting new note", k
                ret = self.simplenote.get_note(k)
                if ret[1] == 0:
                    self.notes[k] = ret[0]
                    local_updates[k] = True
                     
        print "step 3"
        # 3. for each local note not in server index, remove.     
        for lk in self.notes.keys():
            if lk not in server_keys:
                del self.notes[lk]
                local_deletes[lk] = True
                
        # sync done, now write changes to db_path
        for uk in local_updates.keys():
            self.helper_save_note(uk, self.notes[uk])
            
        for dk in local_deletes.keys():
            os.unlink(self.helper_key_to_fname(dk))
            
        print "done syncin'"
        
        
    def do_full_get(self):
        # this returns a tuple ([], -1) if wrong password
        # on success, ([docs], 0)
        # each doc:
        #{u'createdate': u'1335860754.841000',
        # u'deleted': 0,
        # u'key': u'455f66ee936711e19657591a71011082',
        # u'minversion': 10,
        # u'modifydate': u'1337007469.836000',
        # u'syncnum': 54,
        # u'systemtags': [],
        # u'tags': [],
        # u'version': 40}
        
        note_list = self.simplenote.get_note_list()
        
        # simplenote.get_note(key) returns (doc, status)
        # where doc is all of the above with extra field content

        server_notes = []        
        if note_list[1] == 0:
            for i in note_list[0]:
                n = self.simplenote.get_note(i.get('key'))
                if n[1] == 0:
                    server_notes.append(n[0])
                    
        # write server_notes to disc
        for n in server_notes:
            f = open(os.path.join(self.db_path, n.get('key')) + '.json', 'wb')
            json.dump(n, f, indent=2)

    def set_note_content(self, key, content):
        n = self.notes[key]
        old_content = n.get('content')
        if content != old_content:
            n['content'] = content
            n['modifydate'] = time.time()
            # we've made changes, so this needs to be written to disc and synced
            n['saved'] = n['synced'] = False

{'fraction_non_alphanumeric': 0.09207752814765308, 'fraction_numerical': 0.011618939900262198, 'mean_word_length': 1.3351740696278511, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541497', 'n_tokens_mistral': 5712, 'n_tokens_neox': 5380, 'n_words': 1646}