<commit_before>package jdrivesync.gdrive;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.http.*;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.Lists;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.FileList;
import com.google.api.services.drive.model.ParentReference;
import jdrivesync.cli.Options;
import jdrivesync.constants.Constants;
import jdrivesync.encryption.Encryption;
import jdrivesync.exception.JDriveSyncException;
import jdrivesync.gdrive.oauth.Authorize;
import jdrivesync.logging.LoggerFactory;
import jdrivesync.model.SyncDirectory;
import jdrivesync.model.SyncFile;
import jdrivesync.model.SyncItem;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

import static jdrivesync.gdrive.RetryOperation.executeWithRetry;

public class GoogleDriveAdapter {
	public static final String MIME_TYPE_FOLDER = "application/vnd.google-apps.folder";
	public static final String MIME_TYPE_UNKNOWN = "application/octet-stream";
	private static final Logger LOGGER = LoggerFactory.getLogger();
	private final Credential credential;
	private final Options options;
	private final DriveFactory driveFactory;
	private final Encryption encryption;

	public GoogleDriveAdapter(Credential credential, Options options, DriveFactory driveFactory) {
		this.credential = credential;
		this.options = options;
		this.driveFactory = driveFactory;
		this.encryption = new Encryption(options);
	}

	public static Credential authorize() {
		Authorize authorize = new Authorize();
		try {
			return authorize.authorize();
		} catch (Exception e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.AuthorizationFailed, "Authorization failed: " + e.getMessage(), e);
		}
	}

	public File getFile(String id) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			File file = executeWithRetry(options, () -> drive.files().get(id).execute());
			if (LOGGER.isLoggable(Level.FINE)) {
				LOGGER.log(Level.FINE, "Got file : " + file.getId() + ":" + file.getTitle());
			}
			return file;
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to execute get file request: " + e.getMessage(), e);
		}
	}

	public List<File> listChildren(String parentId) {
		List<File> resultList = new LinkedList<File>();
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			Drive.Files.List request = drive.files().list();
			request.setQ("trashed = false and '" + parentId + "' in parents");
			request.setMaxResults(1000);
			LOGGER.log(Level.FINE, "Listing children of folder " + parentId + ".");
			do {
				FileList fileList = executeWithRetry(options, () -> request.execute());
				List<File> items = fileList.getItems();
				resultList.addAll(items);
				request.setPageToken(fileList.getNextPageToken());
			} while (request.getPageToken() != null && request.getPageToken().length() > 0);
			if (LOGGER.isLoggable(Level.FINE)) {
				for (File file : resultList) {
					LOGGER.log(Level.FINE, "Child of " + parentId + ": " + file.getId() + ";" + file.getTitle() + ";" + file.getMimeType());
				}
			}
			removeDuplicates(resultList);
			return resultList;
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to execute list request: " + e.getMessage(), e);
		}
	}

	private void removeDuplicates(List<File> resultList) {
		Map<String, File> fileNameMap = new HashMap<>();
		Iterator<File> iterator = resultList.iterator();
		while (iterator.hasNext()) {
			File file = iterator.next();
			String title = file.getTitle();
			File titleFound = fileNameMap.get(title);
			if (titleFound == null) {
				fileNameMap.put(title, file);
			} else {
				LOGGER.log(Level.WARNING, "Ignoring remote file '" + title + "' (id: '" + file.getId() + "') because its title/name appears more than once in this folder.");
				iterator.remove();
			}
		}
	}

	public void deleteFile(File id) {
		delete(id);
	}

	public void deleteDirectory(File id) {
		delete(id);
	}

	private void delete(File file) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			String id = file.getId();
			if (isGoogleAppsDocument(file)) {
				LOGGER.log(Level.FINE, String.format("Not deleting file '%s' because it is a Google Apps document.", id));
				return;
			}
			if (options.isNoDelete()) {
				LOGGER.log(Level.FINE, String.format("Not deleting file '%s' because option --no-delete is set.", id));
				return;
			}
			if (options.isDeleteFiles()) {
				LOGGER.log(Level.FINE, "Deleting file " + id + ".");
				if (!options.isDryRun()) {
					executeWithRetry(options, () -> drive.files().delete(id).execute());
				}
			} else {
				LOGGER.log(Level.FINE, "Trashing file " + id + ".");
				if (!options.isDryRun()) {
					executeWithRetry(options, () -> drive.files().trash(id).execute());
				}
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to delete file: " + e.getMessage(), e);
		}
	}

	public boolean isGoogleAppsDocument(File file) {
		String mimeType = file.getMimeType();
		if (mimeType != null && mimeType.startsWith("application/vnd.google-apps") && !mimeType.equals("application/vnd.google-apps.folder")) {
			LOGGER.log(Level.FINE, "Not touching file " + file.getId() + " because it is a Google Apps document.");
			return true;
		}
		return false;
	}

	public boolean isDirectory(File file) {
		if (MIME_TYPE_FOLDER.equals(file.getMimeType())) {
			return true;
		}
		return false;
	}

	public InputStream downloadFile(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			File remoteFile = syncItem.getRemoteFile().get();
			String downloadUrl = remoteFile.getDownloadUrl();
			if (downloadUrl != null) {
				HttpRequest httpRequest = drive.getRequestFactory().buildGetRequest(new GenericUrl(downloadUrl));
				LOGGER.log(Level.FINE, "Downloading file " + remoteFile.getId() + ".");
				if (!options.isDryRun()) {
					HttpResponse httpResponse = executeWithRetry(options, () -> httpRequest.execute());
					return httpResponse.getContent();
				}
			} else {
				LOGGER.log(Level.SEVERE, "No download URL for file " + remoteFile);
			}
		} catch (Exception e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to download file: " + e.getMessage(), e);
		}
		return new ByteArrayInputStream(new byte[0]);
	}

	public void updateFile(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			java.io.File localFile = syncItem.getLocalFile().get();
			File remoteFile = syncItem.getRemoteFile().get();
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			if (isGoogleAppsDocument(remoteFile)) {
				return;
			}
			LOGGER.log(Level.FINE, "Updating file " + remoteFile.getId() + " (" + syncItem.getPath() + ").");
			if (!options.isDryRun()) {
				Drive.Files.Update updateRequest = drive.files().update(remoteFile.getId(), remoteFile, new FileContent(determineMimeType(localFile), localFile));
				updateRequest.setSetModifiedDate(true);
				File updatedFile = executeWithRetry(options, () -> updateRequest.execute());
				syncItem.setRemoteFile(Optional.of(updatedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		}
	}

	public void updateMetadata(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			java.io.File localFile = syncItem.getLocalFile().get();
			File remoteFile = syncItem.getRemoteFile().get();
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			if (isGoogleAppsDocument(remoteFile)) {
				return;
			}
			LOGGER.log(Level.FINE, "Updating metadata of remote file " + remoteFile.getId() + " (" + syncItem.getPath() + ").");
			if (!options.isDryRun()) {
				Drive.Files.Update updateRequest = drive.files().update(remoteFile.getId(), remoteFile);
				updateRequest.setSetModifiedDate(true);
				File updatedFile = executeWithRetry(options, () -> updateRequest.execute());
				syncItem.setRemoteFile(Optional.of(updatedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		}
	}

	private String determineMimeType(java.io.File file) {
		String mimeType = GoogleDriveAdapter.MIME_TYPE_UNKNOWN;
		try {
			String contentType = Files.probeContentType(file.toPath());
			if (contentType != null) {
				mimeType = contentType;
			}
		} catch (IOException e) {
			LOGGER.log(Level.WARNING, "Failed to probe MIME type for file ('" + file.getAbsolutePath() + "'): " + e.getMessage() + ". Using '" + mimeType + "' instead.", e);
		}
		return mimeType;
	}

	public void store(SyncFile syncFile) {
		final String mimeType = determineMimeType(syncFile.getLocalFile().get());
		Drive drive = driveFactory.getDrive(this.credential);
		InputStream inputStream = null;
		try {
			final java.io.File localFile = syncFile.getLocalFile().get();
			inputStream = new FileInputStream(localFile);
			if (options.getEncryptFiles().matches(syncFile.getPath(), false)) {
				inputStream = encryption.encrypt(Files.readAllBytes(localFile.toPath()));
			}
			File remoteFile = new File();
			remoteFile.setTitle(localFile.getName());
			remoteFile.setMimeType(mimeType);
			remoteFile.setParents(createParentReferenceList(syncFile));
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			LOGGER.log(Level.FINE, "Inserting new file '" + syncFile.getPath() + "' (" + bytesWithUnit(attr.size()) + ").");
			if (!options.isDryRun()) {
				long startMillis = System.currentTimeMillis();
				File insertedFile;
				long chunkSizeLimit = options.getHttpChunkSizeInBytes();
				if (localFile.length() <= chunkSizeLimit) {
					LOGGER.log(Level.FINE, "File is smaller or equal than " + bytesWithUnit(chunkSizeLimit) + ": no chunked upload");
					insertedFile = executeWithRetry(options, () -> resumableUploadNoChunking(mimeType, drive, localFile, remoteFile));
				} else {
					insertedFile = executeWithRetry(options, () -> resumableUploadChunking(drive, localFile, remoteFile, chunkSizeLimit));
				}
				long duration = System.currentTimeMillis() - startMillis;
				if(LOGGER.isLoggable(Level.FINE)) {
					LOGGER.log(Level.FINE, String.format("Upload took %s ms for %s bytes: %.2f KB/s.", duration, attr.size(), (float) (attr.size() / 1024) / (float) (duration / 1000)));
				}
				syncFile.setRemoteFile(Optional.of(insertedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		} finally {
			if (inputStream != null) {
				try {
					inputStream.close();
				} catch (IOException e) {
				}
			}
		}
	}

	public boolean fileNameValid(File file) {
		String title = file.getTitle();
		if (title == null || title.contains("/") || title.contains("\\")) {
			return false;
		}
		return true;
	}

	private static class ChunkedHttpContent implements HttpContent {
		private final String mimeType;
		private final java.io.File file;
		private long currentChunkStart;
		private long currentChunkEnd;

		public ChunkedHttpContent(java.io.File file, String mimeType, long currentChunkStart, long currentChunkEnd) {
			this.file = file;
			this.mimeType = mimeType;
			this.currentChunkStart = currentChunkStart;
			this.currentChunkEnd = currentChunkEnd;<commit_msg>enhanced logging for remote modifications<commit_after>package jdrivesync.gdrive;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.http.*;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.Lists;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.FileList;
import com.google.api.services.drive.model.ParentReference;
import jdrivesync.cli.Options;
import jdrivesync.constants.Constants;
import jdrivesync.encryption.Encryption;
import jdrivesync.exception.JDriveSyncException;
import jdrivesync.gdrive.oauth.Authorize;
import jdrivesync.logging.LoggerFactory;
import jdrivesync.model.SyncDirectory;
import jdrivesync.model.SyncFile;
import jdrivesync.model.SyncItem;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

import static jdrivesync.gdrive.RetryOperation.executeWithRetry;

public class GoogleDriveAdapter {
	public static final String MIME_TYPE_FOLDER = "application/vnd.google-apps.folder";
	public static final String MIME_TYPE_UNKNOWN = "application/octet-stream";
	private static final Logger LOGGER = LoggerFactory.getLogger();
	private final Credential credential;
	private final Options options;
	private final DriveFactory driveFactory;
	private final Encryption encryption;

	public GoogleDriveAdapter(Credential credential, Options options, DriveFactory driveFactory) {
		this.credential = credential;
		this.options = options;
		this.driveFactory = driveFactory;
		this.encryption = new Encryption(options);
	}

	public static Credential authorize() {
		Authorize authorize = new Authorize();
		try {
			return authorize.authorize();
		} catch (Exception e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.AuthorizationFailed, "Authorization failed: " + e.getMessage(), e);
		}
	}

	public File getFile(String id) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			File file = executeWithRetry(options, () -> drive.files().get(id).execute());
			if (LOGGER.isLoggable(Level.FINE)) {
				LOGGER.log(Level.FINE, "Got file : " + file.getId() + ":" + file.getTitle());
			}
			return file;
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to execute get file request: " + e.getMessage(), e);
		}
	}

	public List<File> listChildren(String parentId) {
		List<File> resultList = new LinkedList<File>();
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			Drive.Files.List request = drive.files().list();
			request.setQ("trashed = false and '" + parentId + "' in parents");
			request.setMaxResults(1000);
			LOGGER.log(Level.FINE, "Listing children of folder " + parentId + ".");
			do {
				FileList fileList = executeWithRetry(options, () -> request.execute());
				List<File> items = fileList.getItems();
				resultList.addAll(items);
				request.setPageToken(fileList.getNextPageToken());
			} while (request.getPageToken() != null && request.getPageToken().length() > 0);
			if (LOGGER.isLoggable(Level.FINE)) {
				for (File file : resultList) {
					LOGGER.log(Level.FINE, "Child of " + parentId + ": " + file.getId() + ";" + file.getTitle() + ";" + file.getMimeType());
				}
			}
			removeDuplicates(resultList);
			return resultList;
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to execute list request: " + e.getMessage(), e);
		}
	}

	private void removeDuplicates(List<File> resultList) {
		Map<String, File> fileNameMap = new HashMap<>();
		Iterator<File> iterator = resultList.iterator();
		while (iterator.hasNext()) {
			File file = iterator.next();
			String title = file.getTitle();
			File titleFound = fileNameMap.get(title);
			if (titleFound == null) {
				fileNameMap.put(title, file);
			} else {
				LOGGER.log(Level.WARNING, "Ignoring remote file '" + title + "' (id: '" + file.getId() + "') because its title/name appears more than once in this folder.");
				iterator.remove();
			}
		}
	}

	public void deleteFile(File id) {
		delete(id);
	}

	public void deleteDirectory(File id) {
		delete(id);
	}

	private void delete(File file) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			String id = file.getId();
			if (isGoogleAppsDocument(file)) {
				LOGGER.log(Level.FINE, String.format("Not deleting file '%s' because it is a Google Apps document.", id));
				return;
			}
			if (options.isNoDelete()) {
				LOGGER.log(Level.FINE, String.format("Not deleting file '%s' because option --no-delete is set.", id));
				return;
			}
			if (options.isDeleteFiles()) {
				LOGGER.log(Level.FINE, "Deleting file " + id + ".");
				if (!options.isDryRun()) {
					executeWithRetry(options, () -> drive.files().delete(id).execute());
				}
			} else {
				LOGGER.log(Level.FINE, "Trashing file " + id + ".");
				if (!options.isDryRun()) {
					executeWithRetry(options, () -> drive.files().trash(id).execute());
				}
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to delete file: " + e.getMessage(), e);
		}
	}

	public boolean isGoogleAppsDocument(File file) {
		String mimeType = file.getMimeType();
		if (mimeType != null && mimeType.startsWith("application/vnd.google-apps") && !mimeType.equals("application/vnd.google-apps.folder")) {
			LOGGER.log(Level.FINE, "Not touching file " + file.getId() + " because it is a Google Apps document.");
			return true;
		}
		return false;
	}

	public boolean isDirectory(File file) {
		if (MIME_TYPE_FOLDER.equals(file.getMimeType())) {
			return true;
		}
		return false;
	}

	public InputStream downloadFile(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			File remoteFile = syncItem.getRemoteFile().get();
			String downloadUrl = remoteFile.getDownloadUrl();
			if (downloadUrl != null) {
				HttpRequest httpRequest = drive.getRequestFactory().buildGetRequest(new GenericUrl(downloadUrl));
				LOGGER.log(Level.FINE, "Downloading file " + remoteFile.getId() + ".");
				if (!options.isDryRun()) {
					HttpResponse httpResponse = executeWithRetry(options, () -> httpRequest.execute());
					return httpResponse.getContent();
				}
			} else {
				LOGGER.log(Level.SEVERE, "No download URL for file " + remoteFile);
			}
		} catch (Exception e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to download file: " + e.getMessage(), e);
		}
		return new ByteArrayInputStream(new byte[0]);
	}

	public void updateFile(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			java.io.File localFile = syncItem.getLocalFile().get();
			File remoteFile = syncItem.getRemoteFile().get();
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			if (isGoogleAppsDocument(remoteFile)) {
				return;
			}
			LOGGER.log(Level.INFO, "Updating file " + remoteFile.getId() + " (" + syncItem.getPath() + ").");
			if (!options.isDryRun()) {
				Drive.Files.Update updateRequest = drive.files().update(remoteFile.getId(), remoteFile, new FileContent(determineMimeType(localFile), localFile));
				updateRequest.setSetModifiedDate(true);
				File updatedFile = executeWithRetry(options, () -> updateRequest.execute());
				syncItem.setRemoteFile(Optional.of(updatedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		}
	}

	public void updateMetadata(SyncItem syncItem) {
		Drive drive = driveFactory.getDrive(this.credential);
		try {
			java.io.File localFile = syncItem.getLocalFile().get();
			File remoteFile = syncItem.getRemoteFile().get();
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			if (isGoogleAppsDocument(remoteFile)) {
				return;
			}
			LOGGER.log(Level.FINE, "Updating metadata of remote file " + remoteFile.getId() + " (" + syncItem.getPath() + ").");
			if (!options.isDryRun()) {
				Drive.Files.Update updateRequest = drive.files().update(remoteFile.getId(), remoteFile);
				updateRequest.setSetModifiedDate(true);
				File updatedFile = executeWithRetry(options, () -> updateRequest.execute());
				syncItem.setRemoteFile(Optional.of(updatedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		}
	}

	private String determineMimeType(java.io.File file) {
		String mimeType = GoogleDriveAdapter.MIME_TYPE_UNKNOWN;
		try {
			String contentType = Files.probeContentType(file.toPath());
			if (contentType != null) {
				mimeType = contentType;
			}
		} catch (IOException e) {
			LOGGER.log(Level.WARNING, "Failed to probe MIME type for file ('" + file.getAbsolutePath() + "'): " + e.getMessage() + ". Using '" + mimeType + "' instead.", e);
		}
		return mimeType;
	}

	public void store(SyncFile syncFile) {
		final String mimeType = determineMimeType(syncFile.getLocalFile().get());
		Drive drive = driveFactory.getDrive(this.credential);
		InputStream inputStream = null;
		try {
			final java.io.File localFile = syncFile.getLocalFile().get();
			inputStream = new FileInputStream(localFile);
			if (options.getEncryptFiles().matches(syncFile.getPath(), false)) {
				inputStream = encryption.encrypt(Files.readAllBytes(localFile.toPath()));
			}
			File remoteFile = new File();
			remoteFile.setTitle(localFile.getName());
			remoteFile.setMimeType(mimeType);
			remoteFile.setParents(createParentReferenceList(syncFile));
			BasicFileAttributes attr = Files.readAttributes(localFile.toPath(), BasicFileAttributes.class);
			remoteFile.setModifiedDate(new DateTime(attr.lastModifiedTime().toMillis()));
			LOGGER.log(Level.INFO, "Uploading new file '" + syncFile.getPath() + "' (" + bytesWithUnit(attr.size()) + ").");
			if (!options.isDryRun()) {
				long startMillis = System.currentTimeMillis();
				File insertedFile;
				long chunkSizeLimit = options.getHttpChunkSizeInBytes();
				if (localFile.length() <= chunkSizeLimit) {
					LOGGER.log(Level.FINE, "File is smaller or equal than " + bytesWithUnit(chunkSizeLimit) + ": no chunked upload");
					insertedFile = executeWithRetry(options, () -> resumableUploadNoChunking(mimeType, drive, localFile, remoteFile));
				} else {
					insertedFile = executeWithRetry(options, () -> resumableUploadChunking(drive, localFile, remoteFile, chunkSizeLimit));
				}
				long duration = System.currentTimeMillis() - startMillis;
				if(LOGGER.isLoggable(Level.FINE)) {
					LOGGER.log(Level.FINE, String.format("Upload took %s ms for %s bytes: %.2f KB/s.", duration, attr.size(), (float) (attr.size() / 1024) / (float) (duration / 1000)));
				}
				syncFile.setRemoteFile(Optional.of(insertedFile));
			}
		} catch (IOException e) {
			throw new JDriveSyncException(JDriveSyncException.Reason.IOException, "Failed to update file: " + e.getMessage(), e);
		} finally {
			if (inputStream != null) {
				try {
					inputStream.close();
				} catch (IOException ignored) {}
			}
		}
	}

	public boolean fileNameValid(File file) {
		String title = file.getTitle();
		if (title == null || title.contains("/") || title.contains("\\")) {
			return false;
		}
		return true;
	}

	private static class ChunkedHttpContent implements HttpContent {
		private final String mimeType;
		private final java.io.File file;
		private long currentChunkStart;
		private long currentChunkEnd;

		public ChunkedHttpContent(java.io.File file, String mimeType, long currentChunkStart, long currentChunkEnd) {
			this.file = file;
			this.mimeType = mimeType;
			this.currentChunkStart = currentChunkStart;
			this.currentChunkEnd = currentChunkEnd;
{'fraction_non_alphanumeric': 0.13557925950331534, 'fraction_numerical': 0.001317902887031012, 'mean_word_length': 4.473850315599639, 'pattern_counts': {'":': 6, '<': 21, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '369776', 'n_tokens_mistral': 8428, 'n_tokens_neox': 7494, 'n_words': 1571}