<commit_before>#!/usr/bin/python

from gdata.photos.service import  *
import gdata.media
import gdata.geo
import os
import re
import pprint
import sys
import argparse
import mimetypes
import hashlib

# Class to store details of an album

supportedImageFormats = frozenset(["image/bmp", "image/gif",  "image/jpeg",  "image/png"])
activityLevels = ["none", "upload", "replace", "delete", "overwrite"]

class Albums:
    def __init__(self, rootDir, albumNaming):
        self.albums = Albums.scanFileSystem(rootDir, albumNaming)
    # walk the directory tree populating the list of files we have locally
    @staticmethod
    def scanFileSystem(rootDir, albumNaming):
        fileAlbums = {}
        for dirName,subdirList,fileList in os.walk( rootDir ) :
            albumName = convertDirToAlbum(albumNaming, rootDir,  dirName)
            # have we already seen this album? If so append our path to it's list
            if albumName in fileAlbums:
                album = fileAlbums[album.getAlbumName()]
                album.paths.append(dirName)
            else:
                # create a new album
                album = AlbumEntry(dirName,  albumName)
                fileAlbums[album.getAlbumName()] = album
            # now iterate it's files to add them to our list
            for fname in fileList :
                fullFilename = os.path.join(dirName, fname)
                # figure out the filename relative to the root dir of the album (to ensure uniqeness) 
                relFileName = re.sub("^/","", fullFilename[len(album.rootPath):])
                fileEntry = FileEntry(relFileName, fullFilename,  False, True)
                album.entries[relFileName] = fileEntry
        print "Found "+str(len(fileAlbums))+" albums on the filesystem"
        return fileAlbums;
    def scanWebAlbums(self):
        # walk the web album finding albums there
        webAlbums = gd_client.GetUserFeed()
        for webAlbum in webAlbums.entry:
            webAlbumTitle = Albums.flatten(webAlbum.title.text)
            if webAlbumTitle in self.albums:
                foundAlbum = self.albums[webAlbumTitle]
                photos = gd_client.GetFeed(webAlbum.GetPhotosUri())
                foundAlbum.webAlbum.append(WebAlbum(webAlbum, int(photos.total_results.text)))
                for photo in photos.entry:
                    # if photo.checksum.text:
                        # print "Found checksum %s" % photo.checksum.text
                    if photo.title.text in foundAlbum.entries: 
                        foundAlbum.entries[photo.title.text].isWeb = True
                    else:
                        print "skipping web only photo "+photo.title.text
            else:
                print "skipping web only album "+webAlbum.title.text 
            print 'Checked: %s (containing %s files)' % (webAlbum.title.text, webAlbum.numphotos.text)
    def uploadMissingAlbumsAndFiles(self,  remoteLevel, metadataLevel):
        for album in self.albums.itervalues():
            for file in album.entries.itervalues():
                if file.isLocal: 
                    if file.isWeb:
                        print "Nothing to do with %s as local and remote" % file.path
                    else:
                        album.upload(file, remoteLevel)
                else:
                    if file.isWeb:
                        print "Download to local not yet supported for %s" % file.path
                    else:
                        print "There is no way a file can be neither remote or local. Error! %s" % file.path
                            


    @staticmethod 
    def createAlbumName(name,  index):
        if index == 0:
            return name
        else:
            return "%s #%s" % (name, index)
    @staticmethod
    def flatten(name):
        return re.sub("#[0-9]*$","",name)
        

class AlbumEntry:
    def __init__(self, fileName,  albumName):
        self.paths = [fileName]
        self.rootPath= fileName
        self.albumName = albumName
        self.entries = {}
        self.webAlbum = []
        self.webAlbumIndex = 0
    def __str__(self):
        return (self.albumName+" starting at "+rootPath+" total "+str(len(self.entries))+" entries "+\
            ["exists","doesn't exist"][not self.webAlbum]+" online")
    def getAlbumName(self):
        if (len(self.albumName) > 0):
            return self.albumName
        else:
            return "Home"
    def getPathsAsString(self):
        return ",".join(self.paths)
    def upload(self, file,  remoteLevel):
        if(remoteLevel >= 1):
            while (self.webAlbumIndex<len(self.webAlbum) and self.webAlbum[self.webAlbumIndex].numberFiles >= 999):
                self.webAlbumIndex = self.webAlbumIndex + 1                        
            if self.webAlbumIndex>=len(self.webAlbum):
                subAlbum = WebAlbum(gd_client.InsertAlbum(title=Albums.createAlbumName(self.getAlbumName(), self.webAlbumIndex), access='private', summary='synced from '+self.rootPath), 0)
                self.webAlbum.append(subAlbum)
                print 'Created album %s to sync %s' % (subAlbum.album.title.text, self.rootPath)
            else:
                subAlbum = self.webAlbum[self.webAlbumIndex]
            photo = self.upload2(subAlbum, file,  remoteLevel)    
        else:
            print "Skipped upload of %s as not instructed to upload" % file.path
    def upload2(self,  subAlbum,  file,  remoteLevel):
        try:
            mimeType = mimetypes.guess_type(file.path)[0]
            if mimeType in supportedImageFormats:
                metadata = gdata.photos.PhotoEntry()
                metadata.title=atom.Title(text=file.name)
                metadata.summary = atom.Summary(text='synced from '+file.path, summary_type='text')
                md5=AlbumEntry.md5sum(file.path)
                metadata.checksum= gdata.photos.Checksum(text=md5)
                photo = gd_client.InsertPhoto(subAlbum.album, metadata, file.path, mimeType)
                print "uploaded "+file.path
                subAlbum.numberFiles = subAlbum.numberFiles + 1
                return photo
            else:
                print "Skipped %s (because can't upload file of type %s)." % (file.path, mimeType)
        except GooglePhotosException:
            print "Skipping upload of %s due to exception" % file.path 
    @staticmethod 
    def md5sum(filename):
        md5 = hashlib.md5()
        with open(filename,'rb') as f: 
            for chunk in iter(lambda: f.read(128*md5.block_size), b''): 
                 md5.update(chunk)
        return md5.hexdigest()
    
# Class to store web album details

class WebAlbum:
    def __init__(self, album,  numberFiles):
        self.album = album
        self.numberFiles = numberFiles


# Class to store details of an individual file

class FileEntry:
    def __init__(self, name, path,  isWeb,  isLocal):
        self.name = name
        self.path=path
        self.isWeb=isWeb
        self.isLocal=isLocal
    
    
# Method to translate directory name to an album name   
    
def convertDirToAlbum(form,  root,  name):
    formElements = re.split("~", form)
    nameElements = re.split("/", re.sub("^/","",name[len(root):]))
    which = min(len(formElements), len(nameElements))
    work = formElements[which-1].format(*nameElements)
    return work<commit_msg>Hashing largely working - but no way of switching it off<commit_after>#!/usr/bin/python

from gdata.photos.service import  *
import gdata.media
import gdata.geo
import os
import re
import pprint
import sys
import argparse
import mimetypes
import hashlib
import urllib

# Class to store details of an album

supportedImageFormats = frozenset(["image/bmp", "image/gif",  "image/jpeg",  "image/png"])
activityLevels = ["none", "upload", "replace", "delete", "overwrite"]

class Albums:
    def __init__(self, rootDir, albumNaming):
        self.albums = Albums.scanFileSystem(rootDir, albumNaming)
    # walk the directory tree populating the list of files we have locally
    @staticmethod
    def scanFileSystem(rootDir, albumNaming):
        fileAlbums = {}
        for dirName,subdirList,fileList in os.walk( rootDir ) :
            albumName = convertDirToAlbum(albumNaming, rootDir,  dirName)
            # have we already seen this album? If so append our path to it's list
            if albumName in fileAlbums:
                album = fileAlbums[album.getAlbumName()]
                album.paths.append(dirName)
            else:
                # create a new album
                album = AlbumEntry(dirName,  albumName)
                fileAlbums[album.getAlbumName()] = album
            # now iterate it's files to add them to our list
            for fname in fileList :
                fullFilename = os.path.join(dirName, fname)
                # figure out the filename relative to the root dir of the album (to ensure uniqeness) 
                relFileName = re.sub("^/","", fullFilename[len(album.rootPath):])
                fileEntry = FileEntry(relFileName, fullFilename,  False, True)
                album.entries[relFileName] = fileEntry
        print "Found "+str(len(fileAlbums))+" albums on the filesystem"
        return fileAlbums;
    def scanWebAlbums(self):
        # walk the web album finding albums there
        webAlbums = gd_client.GetUserFeed()
        for webAlbum in webAlbums.entry:
            webAlbumTitle = Albums.flatten(webAlbum.title.text)
            if webAlbumTitle in self.albums:
                foundAlbum = self.albums[webAlbumTitle]
                photos = gd_client.GetFeed(webAlbum.GetPhotosUri())
                foundAlbum.webAlbum.append(WebAlbum(webAlbum, int(photos.total_results.text)))
                for photo in photos.entry:
                    photoTitle=urllib.unquote(photo.title.text)
                    if photoTitle in foundAlbum.entries: 
                        foundAlbum.entries[photoTitle].webReference = photo
                        foundAlbum.entries[photoTitle].remoteHash = photo.checksum.text
                    else:
                        print "skipping web only photo %s from [%s]"% (photoTitle, ", ".join(foundAlbum.entries))
            else:
                print "skipping web only album "+webAlbum.title.text 
            print 'Checked: %s (containing %s files)' % (webAlbum.title.text, webAlbum.numphotos.text)
    def uploadMissingAlbumsAndFiles(self,  remoteLevel, metadataLevel):
        for album in self.albums.itervalues():
            for file in album.entries.itervalues():
                if file.isLocal: 
                    if file.isWeb():
                        if file.changed():
                            print "Replacing %s as local and remote and chnage status is %s" % (file.path, file.changed())
                            gd_client.Delete(file.webReference)
                            album.upload(file, remoteLevel)
                        else:
                            print "Nothing to do with %s as local and remote and chnage status is %s" % (file.path, file.changed())
                    else:
                        album.upload(file, remoteLevel)
                else:
                    if file.isWeb():
                        print "Download to local not yet supported for %s" % file.path
                    else:
                        print "There is no way a file can be neither remote or local. Error! %s" % file.path
                            


    @staticmethod 
    def createAlbumName(name,  index):
        if index == 0:
            return name
        else:
            return "%s #%s" % (name, index)
    @staticmethod
    def flatten(name):
        return re.sub("#[0-9]*$","",name)
        

class AlbumEntry:
    def __init__(self, fileName,  albumName):
        self.paths = [fileName]
        self.rootPath= fileName
        self.albumName = albumName
        self.entries = {}
        self.webAlbum = []
        self.webAlbumIndex = 0
    def __str__(self):
        return (self.albumName+" starting at "+rootPath+" total "+str(len(self.entries))+" entries "+\
            ["exists","doesn't exist"][not self.webAlbum]+" online")
    def getAlbumName(self):
        if (len(self.albumName) > 0):
            return self.albumName
        else:
            return "Home"
    def getPathsAsString(self):
        return ",".join(self.paths)
    def upload(self, file,  remoteLevel):
        if(remoteLevel >= 1):
            while (self.webAlbumIndex<len(self.webAlbum) and self.webAlbum[self.webAlbumIndex].numberFiles >= 999):
                self.webAlbumIndex = self.webAlbumIndex + 1                        
            if self.webAlbumIndex>=len(self.webAlbum):
                subAlbum = WebAlbum(gd_client.InsertAlbum(title=Albums.createAlbumName(self.getAlbumName(), self.webAlbumIndex), access='private', summary='synced from '+self.rootPath), 0)
                self.webAlbum.append(subAlbum)
                print 'Created album %s to sync %s' % (subAlbum.album.title.text, self.rootPath)
            else:
                subAlbum = self.webAlbum[self.webAlbumIndex]
            photo = self.upload2(subAlbum, file,  remoteLevel)    
        else:
            print "Skipped upload of %s as not instructed to upload" % file.path
    def upload2(self,  subAlbum,  file,  remoteLevel):
        try:
            mimeType = mimetypes.guess_type(file.path)[0]
            if mimeType in supportedImageFormats:
                metadata = gdata.photos.PhotoEntry()
                metadata.title=atom.Title(text=urllib.quote_plus(file.name)) # have to quote as certain charecters, e.g. / seem to break it
                metadata.summary = atom.Summary(text='synced from '+file.path, summary_type='text')
                metadata.checksum= gdata.photos.Checksum(text=file.getLocalHash())
                photo = gd_client.InsertPhoto(subAlbum.album, metadata, file.path, mimeType)
                print "uploaded %s (as %s)" % (file.name, urllib.quote_plus(file.name))
                subAlbum.numberFiles = subAlbum.numberFiles + 1
                return photo
            else:
                print "Skipped %s (because can't upload file of type %s)." % (file.path, mimeType)
        except GooglePhotosException:
            print "Skipping upload of %s due to exception" % file.path 
    
# Class to store web album details

class WebAlbum:
    def __init__(self, album,  numberFiles):
        self.album = album
        self.numberFiles = numberFiles


# Class to store details of an individual file

class FileEntry:
    def __init__(self, name, path,  isWeb,  isLocal):
        self.name = name
        self.path=path
        self.isLocal=isLocal
        self.localHash=None
        self.remoteHash=None
        self.webReference=None
    def getLocalHash(self):
        if not(self.localHash):
            md5 = hashlib.md5()
            with open(self.path,'rb') as f: 
                for chunk in iter(lambda: f.read(128*md5.block_size), b''): 
                     md5.update(chunk)
            self.localHash = md5.hexdigest()
        return self.localHash
    def changed(self):
        if self.isLocal and self.isWeb():
            if self.remoteHash:
                return self.remoteHash != self.getLocalHash()
            else:
                return None
        else:
            return True
    def isWeb(self):
        return self.webReference != None
    
# Method to translate directory name to an album name   
    
def convertDirToAlbum(form,  root,  name):
    formElements = re.split("~", form)
    nameElements = re.split("/", re.sub("^/","",name[len(root):]))
    which = min(len(formElements), len(nameElements))
    work = formElements[which-1].format(*nameElements)
    return work
{'fraction_non_alphanumeric': 0.09758110596152625, 'fraction_numerical': 0.0033013776903053774, 'mean_word_length': 2.0187811422000768, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628241', 'n_tokens_mistral': 4578, 'n_tokens_neox': 4277, 'n_words': 1159}