<commit_before># coding=utf-8
from __future__ import absolute_import, division, print_function

__author__ = "<NAME> <<EMAIL>>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

from flask import request, jsonify, make_response, url_for

from octoprint.filemanager.destinations import FileDestinations
from octoprint.settings import settings, valid_boolean_trues
from octoprint.server import printer, fileManager, slicingManager, eventManager, NO_CONTENT, current_user
from octoprint.server.util.flask import restricted_access, get_json_command_from_request, with_revalidation_checking
from octoprint.server.api import api
from octoprint.events import Events
import octoprint.filemanager
import octoprint.filemanager.util
import octoprint.filemanager.storage
import octoprint.slicing

import psutil
import hashlib
import logging
import threading


#~~ GCODE file handling

_file_cache = dict()
_file_cache_mutex = threading.RLock()

_DATA_FORMAT_VERSION = "v2"

def _clear_file_cache():
	with _file_cache_mutex:
		_file_cache.clear()

def _create_lastmodified(path, recursive):
	if path.endswith("/files"):
		# all storages involved
		lms = [0]
		for storage in fileManager.registered_storages:
			try:
				lms.append(fileManager.last_modified(storage, recursive=recursive))
			except:
				logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(storage))
				lms.append(None)

		if filter(lambda x: x is None, lms):
			# we return None if ANY of the involved storages returned None
			return None

		# if we reach this point, we return the maximum of all dates
		return max(lms)

	elif path.endswith("/files/local"):
		# only local storage involved
		try:
			return fileManager.last_modified(FileDestinations.LOCAL, recursive=recursive)
		except:
			logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(FileDestinations.LOCAL))
			return None

	else:
		return None


def _create_etag(path, filter, recursive, lm=None):
	if lm is None:
		lm = _create_lastmodified(path, recursive)

	if lm is None:
		return None

	hash = hashlib.sha1()
	hash.update(str(lm))
	hash.update(str(filter))
	hash.update(str(recursive))

	if path.endswith("/files") or path.endswith("/files/sdcard"):
		# include sd data in etag
		hash.update(repr(sorted(printer.get_sd_files(), key=lambda x: x[0])))

	hash.update(_DATA_FORMAT_VERSION) # increment version if we change the API format

	return hash.hexdigest()


@api.route("/files", methods=["GET"])
@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path,
                                                                      request.values.get("filter", False),
                                                                      request.values.get("recursive", False),
                                                                      lm=lm),
                            lastmodified_factory=lambda: _create_lastmodified(request.path,
                                                                              request.values.get("recursive", False)),
                            unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False))
def readGcodeFiles():
	filter = request.values.get("filter", False)
	recursive = request.values.get("recursive", "false") in valid_boolean_trues
	force = request.values.get("force", "false") in valid_boolean_trues

	files = _getFileList(FileDestinations.LOCAL, filter=filter, recursive=recursive, allow_from_cache=not force)
	files.extend(_getFileList(FileDestinations.SDCARD))

	usage = psutil.disk_usage(settings().getBaseFolder("uploads", check_writable=False))
	return jsonify(files=files, free=usage.free, total=usage.total)


@api.route("/files/<string:origin>", methods=["GET"])
@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path,
                                                                      request.values.get("filter", False),
                                                                      request.values.get("recursive", False),
                                                                      lm=lm),
                            lastmodified_factory=lambda: _create_lastmodified(request.path,
                                                                              request.values.get("recursive", False)),
                            unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False))
def readGcodeFilesForOrigin(origin):
	if origin not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
		return make_response("Unknown origin: %s" % origin, 404)

	filter = request.values.get("filter", False)
	recursive = request.values.get("recursive", "false") in valid_boolean_trues
	force = request.values.get("force", "false") in valid_boolean_trues

	files = _getFileList(origin, filter=filter, recursive=recursive, allow_from_cache=not force)

	if origin == FileDestinations.LOCAL:
		usage = psutil.disk_usage(settings().getBaseFolder("uploads", check_writable=False))
		return jsonify(files=files, free=usage.free, total=usage.total)
	else:
		return jsonify(files=files)


def _getFileDetails(origin, path, recursive=True):
	parent, path = fileManager.split_path(origin, path)
	files = _getFileList(origin, path=parent, recursive=recursive)

	for f in files:
		if f["name"] == path:
			return f
	else:
		return None


def _getFileList(origin, path=None, filter=None, recursive=False, allow_from_cache=True):
	if origin == FileDestinations.SDCARD:
		sdFileList = printer.get_sd_files()

		files = []
		if sdFileList is not None:
			for sdFile, sdSize in sdFileList:
				type_path = octoprint.filemanager.get_file_type(sdFile)
				if not type_path:
					# only supported extensions
					continue
				else:
					file_type = type_path[0]
<commit_msg>Fix API error when accessing sdcard file details<commit_after># coding=utf-8
from __future__ import absolute_import, division, print_function

__author__ = "<NAME> <<EMAIL>>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

from flask import request, jsonify, make_response, url_for

from octoprint.filemanager.destinations import FileDestinations
from octoprint.settings import settings, valid_boolean_trues
from octoprint.server import printer, fileManager, slicingManager, eventManager, NO_CONTENT, current_user
from octoprint.server.util.flask import restricted_access, get_json_command_from_request, with_revalidation_checking
from octoprint.server.api import api
from octoprint.events import Events
import octoprint.filemanager
import octoprint.filemanager.util
import octoprint.filemanager.storage
import octoprint.slicing

import os
import psutil
import hashlib
import logging
import threading


#~~ GCODE file handling

_file_cache = dict()
_file_cache_mutex = threading.RLock()

_DATA_FORMAT_VERSION = "v2"

def _clear_file_cache():
	with _file_cache_mutex:
		_file_cache.clear()

def _create_lastmodified(path, recursive):
	if path.endswith("/files"):
		# all storages involved
		lms = [0]
		for storage in fileManager.registered_storages:
			try:
				lms.append(fileManager.last_modified(storage, recursive=recursive))
			except:
				logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(storage))
				lms.append(None)

		if filter(lambda x: x is None, lms):
			# we return None if ANY of the involved storages returned None
			return None

		# if we reach this point, we return the maximum of all dates
		return max(lms)

	elif path.endswith("/files/local"):
		# only local storage involved
		try:
			return fileManager.last_modified(FileDestinations.LOCAL, recursive=recursive)
		except:
			logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(FileDestinations.LOCAL))
			return None

	else:
		return None


def _create_etag(path, filter, recursive, lm=None):
	if lm is None:
		lm = _create_lastmodified(path, recursive)

	if lm is None:
		return None

	hash = hashlib.sha1()
	hash.update(str(lm))
	hash.update(str(filter))
	hash.update(str(recursive))

	if path.endswith("/files") or path.endswith("/files/sdcard"):
		# include sd data in etag
		hash.update(repr(sorted(printer.get_sd_files(), key=lambda x: x[0])))

	hash.update(_DATA_FORMAT_VERSION) # increment version if we change the API format

	return hash.hexdigest()


@api.route("/files", methods=["GET"])
@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path,
                                                                      request.values.get("filter", False),
                                                                      request.values.get("recursive", False),
                                                                      lm=lm),
                            lastmodified_factory=lambda: _create_lastmodified(request.path,
                                                                              request.values.get("recursive", False)),
                            unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False))
def readGcodeFiles():
	filter = request.values.get("filter", False)
	recursive = request.values.get("recursive", "false") in valid_boolean_trues
	force = request.values.get("force", "false") in valid_boolean_trues

	files = _getFileList(FileDestinations.LOCAL, filter=filter, recursive=recursive, allow_from_cache=not force)
	files.extend(_getFileList(FileDestinations.SDCARD))

	usage = psutil.disk_usage(settings().getBaseFolder("uploads", check_writable=False))
	return jsonify(files=files, free=usage.free, total=usage.total)


@api.route("/files/<string:origin>", methods=["GET"])
@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path,
                                                                      request.values.get("filter", False),
                                                                      request.values.get("recursive", False),
                                                                      lm=lm),
                            lastmodified_factory=lambda: _create_lastmodified(request.path,
                                                                              request.values.get("recursive", False)),
                            unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False))
def readGcodeFilesForOrigin(origin):
	if origin not in [FileDestinations.LOCAL, FileDestinations.SDCARD]:
		return make_response("Unknown origin: %s" % origin, 404)

	filter = request.values.get("filter", False)
	recursive = request.values.get("recursive", "false") in valid_boolean_trues
	force = request.values.get("force", "false") in valid_boolean_trues

	files = _getFileList(origin, filter=filter, recursive=recursive, allow_from_cache=not force)

	if origin == FileDestinations.LOCAL:
		usage = psutil.disk_usage(settings().getBaseFolder("uploads", check_writable=False))
		return jsonify(files=files, free=usage.free, total=usage.total)
	else:
		return jsonify(files=files)


def _getFileDetails(origin, path, recursive=True):
	parent, path = os.path.split(path)
	files = _getFileList(origin, path=parent, recursive=recursive)

	for f in files:
		if f["name"] == path:
			return f
	else:
		return None


def _getFileList(origin, path=None, filter=None, recursive=False, allow_from_cache=True):
	if origin == FileDestinations.SDCARD:
		sdFileList = printer.get_sd_files()

		files = []
		if sdFileList is not None:
			for sdFile, sdSize in sdFileList:
				type_path = octoprint.filemanager.get_file_type(sdFile)
				if not type_path:
					# only supported extensions
					continue
				else:
					file_type = type_path[0]

{'fraction_non_alphanumeric': 0.0985172981878089, 'fraction_numerical': 0.0023064250411861617, 'mean_word_length': 3.1981327800829877, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3725986', 'n_tokens_mistral': 3680, 'n_tokens_neox': 3414, 'n_words': 836}