<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import os
from frappe import _
from frappe.model.document import Document
import dropbox, json
from frappe.utils.backups import new_backup
from frappe.utils.background_jobs import enqueue
from six.moves.urllib.parse import urlparse, parse_qs
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, split_emails, get_request_site_address, cstr,
	get_files_path, get_backups_path, get_url, encode)

ignore_list = [".DS_Store"]

class DropboxSettings(Document):
	def onload(self):
		if not self.app_access_key and frappe.conf.dropbox_access_key:
			self.set_onload("dropbox_setup_via_site_config", 1)
		

@frappe.whitelist()
def take_backup():
	"Enqueue longjob for taking backup to dropbox"
	enqueue("frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", queue='long', timeout=1500)
	frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))

def take_backups_daily():
	take_backups_if("Daily")

def take_backups_weekly():
	take_backups_if("Weekly")

def take_backups_if(freq):
	if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq:
		take_backup_to_dropbox()

def take_backup_to_dropbox():
	did_not_upload, error_log = [], []
	try:
		if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")):
			did_not_upload, error_log = backup_to_dropbox()
			if did_not_upload: raise Exception

			send_email(True, "Dropbox")
	except Exception:
		file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)]
		error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback())
		frappe.errprint(error_message)
		send_email(False, "Dropbox", error_message)

def send_email(success, service_name, error_status=None):
	if success:
		subject = "Backup Upload Successful"
		message ="""<h3>Backup Uploaded Successfully</h3><p>Hi there, this is just to inform you
		that your backup was successfully uploaded to your %s account. So relax!</p>
		""" % service_name

	else:
		subject = "[Warning] Backup Upload Failed"
		message ="""<h3>Backup Upload Failed</h3><p>Oops, your automated backup to %s
		failed.</p>
		<p>Error message: <br>
		<pre><code>%s</code></pre>
		</p>
		<p>Please contact your system manager for more information.</p>
		""" % (service_name, error_status)

	if not frappe.db:
		frappe.connect()

	recipients = split_emails(frappe.db.get_value("Dropbox Settings", None, "send_notifications_to"))
	frappe.sendmail(recipients=recipients, subject=subject, message=message)

def backup_to_dropbox():
	if not frappe.db:
		frappe.connect()

	# upload database
	dropbox_settings = get_dropbox_settings()

	if not dropbox_settings['access_token']:
		access_token = generate_oauth2_access_token_from_oauth1_token(dropbox_settings)

		if not access_token.get('oauth2_token'):
			return 'Failed backup upload', 'No Access Token exists! Please generate the access token for Dropbox.'

		dropbox_settings['access_token'] = access_token['oauth2_token']
		set_dropbox_access_token(access_token['oauth2_token'])


	dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'])
	backup = new_backup(ignore_files=True)
	filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
	upload_file_to_dropbox(filename, "/database", dropbox_client)

	frappe.db.close()
	
	# upload files to files folder
	did_not_upload = []
	error_log = []

	upload_from_folder(get_files_path(), "/files", dropbox_client, did_not_upload, error_log)
	upload_from_folder(get_files_path(is_private=1), "/private/files", dropbox_client, did_not_upload, error_log)

	frappe.connect()
	return did_not_upload, list(set(error_log))

def upload_from_folder(path, dropbox_folder, dropbox_client, did_not_upload, error_log):
	if not os.path.exists(path):
		return

	try:
		response = dropbox_client.files_list_folder(dropbox_folder)
	except dropbox.exceptions.ApiError as e:
		# folder not found
		if isinstance(e.error, dropbox.files.ListFolderError):
			response = frappe._dict({"entries": []})
		else:
			raise

	for root, directory, files in os.walk(path):
		for filename in files:
			filename = cstr(filename)
			filepath = os.path.join(root, filename)

			if filename in ignore_list:
				continue

			found = False
			for file_metadata in response.entries:
				if (os.path.basename(filepath) == file_metadata.name
					and os.stat(encode(filepath)).st_size == int(file_metadata.size)):
					found = True
					break

			if not found:
				try:
					upload_file_to_dropbox(filepath, dropbox_folder, dropbox_client)
				except Exception:
					did_not_upload.append(filepath)
					error_log.append(frappe.get_traceback())

def upload_file_to_dropbox(filename, folder, dropbox_client):
	"""upload files with chunk of 15 mb to reduce session append calls"""

	create_folder_if_not_exists(folder, dropbox_client)
	chunk_size = 15 * 1024 * 1024
	file_size = os.path.getsize(encode(filename))
	mode = (dropbox.files.WriteMode.overwrite)

	f = open(encode(filename), 'rb')<commit_msg>fix dropbox<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, <NAME> and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import os
from frappe import _
from frappe.model.document import Document
import dropbox, json
from frappe.utils.backups import new_backup
from frappe.utils.background_jobs import enqueue
from six.moves.urllib.parse import urlparse, parse_qs
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, split_emails, get_request_site_address, cstr,
	get_files_path, get_backups_path, get_url, encode)
from six import text_type

ignore_list = [".DS_Store"]

class DropboxSettings(Document):
	def onload(self):
		if not self.app_access_key and frappe.conf.dropbox_access_key:
			self.set_onload("dropbox_setup_via_site_config", 1)
		

@frappe.whitelist()
def take_backup():
	"Enqueue longjob for taking backup to dropbox"
	enqueue("frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", queue='long', timeout=1500)
	frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))

def take_backups_daily():
	take_backups_if("Daily")

def take_backups_weekly():
	take_backups_if("Weekly")

def take_backups_if(freq):
	if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq:
		take_backup_to_dropbox()

def take_backup_to_dropbox():
	did_not_upload, error_log = [], []
	try:
		if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")):
			did_not_upload, error_log = backup_to_dropbox()
			if did_not_upload: raise Exception

			send_email(True, "Dropbox")
	except Exception:
		file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)]
		error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback())
		frappe.errprint(error_message)
		send_email(False, "Dropbox", error_message)

def send_email(success, service_name, error_status=None):
	if success:
		subject = "Backup Upload Successful"
		message ="""<h3>Backup Uploaded Successfully</h3><p>Hi there, this is just to inform you
		that your backup was successfully uploaded to your %s account. So relax!</p>
		""" % service_name

	else:
		subject = "[Warning] Backup Upload Failed"
		message ="""<h3>Backup Upload Failed</h3><p>Oops, your automated backup to %s
		failed.</p>
		<p>Error message: <br>
		<pre><code>%s</code></pre>
		</p>
		<p>Please contact your system manager for more information.</p>
		""" % (service_name, error_status)

	if not frappe.db:
		frappe.connect()

	recipients = split_emails(frappe.db.get_value("Dropbox Settings", None, "send_notifications_to"))
	frappe.sendmail(recipients=recipients, subject=subject, message=message)

def backup_to_dropbox():
	if not frappe.db:
		frappe.connect()

	# upload database
	dropbox_settings = get_dropbox_settings()

	if not dropbox_settings['access_token']:
		access_token = generate_oauth2_access_token_from_oauth1_token(dropbox_settings)

		if not access_token.get('oauth2_token'):
			return 'Failed backup upload', 'No Access Token exists! Please generate the access token for Dropbox.'

		dropbox_settings['access_token'] = access_token['oauth2_token']
		set_dropbox_access_token(access_token['oauth2_token'])


	dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'])
	backup = new_backup(ignore_files=True)
	filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
	upload_file_to_dropbox(filename, "/database", dropbox_client)

	frappe.db.close()
	
	# upload files to files folder
	did_not_upload = []
	error_log = []

	upload_from_folder(get_files_path(), "/files", dropbox_client, did_not_upload, error_log)
	upload_from_folder(get_files_path(is_private=1), "/private/files", dropbox_client, did_not_upload, error_log)

	frappe.connect()
	return did_not_upload, list(set(error_log))

def upload_from_folder(path, dropbox_folder, dropbox_client, did_not_upload, error_log):
	if not os.path.exists(path):
		return

	try:
		response = dropbox_client.files_list_folder(dropbox_folder)
	except dropbox.exceptions.ApiError as e:
		# folder not found
		if isinstance(e.error, dropbox.files.ListFolderError):
			response = frappe._dict({"entries": []})
		else:
			raise

	path = text_type(path)
	for root, directory, files in os.walk(path):
		for filename in files:
			filename = cstr(filename)
			filepath = os.path.join(root, filename)

			if filename in ignore_list:
				continue

			found = False
			for file_metadata in response.entries:
				if (os.path.basename(filepath) == file_metadata.name
					and os.stat(encode(filepath)).st_size == int(file_metadata.size)):
					found = True
					break

			if not found:
				try:
					upload_file_to_dropbox(filepath, dropbox_folder, dropbox_client)
				except Exception:
					did_not_upload.append(filepath)
					error_log.append(frappe.get_traceback())

def upload_file_to_dropbox(filename, folder, dropbox_client):
	"""upload files with chunk of 15 mb to reduce session append calls"""

	create_folder_if_not_exists(folder, dropbox_client)
	chunk_size = 15 * 1024 * 1024
	file_size = os.path.getsize(encode(filename))
	mode = (dropbox.files.WriteMode.overwrite)

	f = open(encode(filename), 'rb')
{'fraction_non_alphanumeric': 0.11053590259802512, 'fraction_numerical': 0.006135557472917266, 'mean_word_length': 5.665814696485623, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7150169', 'n_tokens_mistral': 3800, 'n_tokens_neox': 3560, 'n_words': 801}