<commit_before>//
// gdrive.go
// Copyright(c)2014-2015 Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package gdrive provides a slightly higher-level API for Google Drive
// than is provided by the official Google Drive API Go language bindings.
// In addition to handling transient network errors, rate limit
// errors, and other http miscellania, gdrive also provides functionality
// for limiting bandwidth consumption in both uploads and downloads.
//
// gdrive was written to be independent of the skicka application; issues
// like encryption, mapping Google Drive files to Unix file semantics,
// etc., are intentionally not included here.
package gdrive

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"google.golang.org/api/drive/v2"
	"google.golang.org/api/googleapi"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
)

const timeFormat = "2006-01-02T15:04:05.000000000Z07:00"

///////////////////////////////////////////////////////////////////////////

// TODO: The File representation doesn't quite work in the case where
// multiple paths on Drive refer to the same underlying file; for example,
// if we update such a file's modification time on Drive and update the
// local metadata to reflect this (which we don't yet..), then the metadata
// won't reflect the modification time for other paths that point to the
// file.  It may be best to have File be just a Path and a pointer to a new
// FileMetadata struct.

// File represents a file or folder in Google Drive.
type File struct {
	// Path name on Drive. Does not start with a slash.
	Path string
	// Size of the file in bytes.
	FileSize int64
	// Unique id of the file (that persists over file modifications,
	// renamings, etc.) This value is generated by Google Drive when a file
	// is first created.
	Id string
	// MD5 checksum of the file's contents.
	Md5      string
	MimeType string
	// Last time that the file's contents were modified.
	ModTime time.Time
	// Array of Google Drive file ids for the folder or folders that have
	// this file in it.
	ParentIds []string
	// User-defined properties associated with the file.
	Properties []Property
}

// newFile returns a new gdrive.File corresponding to the given Google
// Drive file. The path to the file on Drive should be provided for the
// path parameter.
func newFile(path string, f *drive.File) *File {
	modTime := time.Unix(0, 0)
	if f.ModifiedDate != "" {
		modTime, _ = time.Parse(time.RFC3339Nano, f.ModifiedDate)
	}

	var properties []Property
	for _, p := range f.Properties {
		properties = append(properties, Property{Key: p.Key, Value: p.Value})
	}

	var parentIds []string
	for _, p := range f.Parents {
		parentIds = append(parentIds, p.Id)
	}

	return &File{
		Path:       path,
		FileSize:   f.FileSize,
		Id:         f.Id,
		Md5:        f.Md5Checksum,
		MimeType:   f.MimeType,
		ModTime:    modTime,
		ParentIds:  parentIds,
		Properties: properties,
	}
}

// driveFile returns a new *drive.File that corresponds to the gdrive.File.
func (f *File) driveFile() *drive.File {
	var parents []*drive.ParentReference
	for _, pid := range f.ParentIds {
		parents = append(parents, &drive.ParentReference{Id: pid})
	}

	return &drive.File{
		Title:        filepath.Base(f.Path),
		FileSize:     f.FileSize,
		Id:           f.Id,
		Md5Checksum:  f.Md5,
		MimeType:     f.MimeType,
		ModifiedDate: f.ModTime.UTC().Format(timeFormat),
		Parents:      parents,
		Properties:   convertProplist(f.Properties),
	}
}

// IsFolder returns a boolean indicating whether the given File is a
// folder.
func (f *File) IsFolder() bool {
	return f.MimeType == "application/vnd.google-apps.folder"
}

// IsGoogleAppsFile returns a boolean indicating whether the given File was created
// with Google Docs, Google Sheets, etc.
func (f *File) IsGoogleAppsFile() bool {
	return strings.HasPrefix(f.MimeType, "application/vnd.google-apps.")
}

// Property represents a user-specified property associated with a Drive
// file.
type Property struct {
	Key, Value string
}

// GetProperty returns the property of the given name associated with the
// given file, if the named property is present.  If the property isn't
// present in the fie, then an empty string and an error are returned.
func (f *File) GetProperty(name string) (string, error) {
	for _, prop := range f.Properties {
		if prop.Key == name {
			return prop.Value, nil
		}<commit_msg>Fix identification of IsGoogleAppsFile to not return true for folders<commit_after>//
// gdrive.go
// Copyright(c)2014-2015 Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package gdrive provides a slightly higher-level API for Google Drive
// than is provided by the official Google Drive API Go language bindings.
// In addition to handling transient network errors, rate limit
// errors, and other http miscellania, gdrive also provides functionality
// for limiting bandwidth consumption in both uploads and downloads.
//
// gdrive was written to be independent of the skicka application; issues
// like encryption, mapping Google Drive files to Unix file semantics,
// etc., are intentionally not included here.
package gdrive

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"google.golang.org/api/drive/v2"
	"google.golang.org/api/googleapi"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
)

const timeFormat = "2006-01-02T15:04:05.000000000Z07:00"

///////////////////////////////////////////////////////////////////////////

// TODO: The File representation doesn't quite work in the case where
// multiple paths on Drive refer to the same underlying file; for example,
// if we update such a file's modification time on Drive and update the
// local metadata to reflect this (which we don't yet..), then the metadata
// won't reflect the modification time for other paths that point to the
// file.  It may be best to have File be just a Path and a pointer to a new
// FileMetadata struct.

// File represents a file or folder in Google Drive.
type File struct {
	// Path name on Drive. Does not start with a slash.
	Path string
	// Size of the file in bytes.
	FileSize int64
	// Unique id of the file (that persists over file modifications,
	// renamings, etc.) This value is generated by Google Drive when a file
	// is first created.
	Id string
	// MD5 checksum of the file's contents.
	Md5      string
	MimeType string
	// Last time that the file's contents were modified.
	ModTime time.Time
	// Array of Google Drive file ids for the folder or folders that have
	// this file in it.
	ParentIds []string
	// User-defined properties associated with the file.
	Properties []Property
}

// newFile returns a new gdrive.File corresponding to the given Google
// Drive file. The path to the file on Drive should be provided for the
// path parameter.
func newFile(path string, f *drive.File) *File {
	modTime := time.Unix(0, 0)
	if f.ModifiedDate != "" {
		modTime, _ = time.Parse(time.RFC3339Nano, f.ModifiedDate)
	}

	var properties []Property
	for _, p := range f.Properties {
		properties = append(properties, Property{Key: p.Key, Value: p.Value})
	}

	var parentIds []string
	for _, p := range f.Parents {
		parentIds = append(parentIds, p.Id)
	}

	return &File{
		Path:       path,
		FileSize:   f.FileSize,
		Id:         f.Id,
		Md5:        f.Md5Checksum,
		MimeType:   f.MimeType,
		ModTime:    modTime,
		ParentIds:  parentIds,
		Properties: properties,
	}
}

// driveFile returns a new *drive.File that corresponds to the gdrive.File.
func (f *File) driveFile() *drive.File {
	var parents []*drive.ParentReference
	for _, pid := range f.ParentIds {
		parents = append(parents, &drive.ParentReference{Id: pid})
	}

	return &drive.File{
		Title:        filepath.Base(f.Path),
		FileSize:     f.FileSize,
		Id:           f.Id,
		Md5Checksum:  f.Md5,
		MimeType:     f.MimeType,
		ModifiedDate: f.ModTime.UTC().Format(timeFormat),
		Parents:      parents,
		Properties:   convertProplist(f.Properties),
	}
}

// IsFolder returns a boolean indicating whether the given File is a
// folder.
func (f *File) IsFolder() bool {
	return f.MimeType == "application/vnd.google-apps.folder"
}

// IsGoogleAppsFile returns a boolean indicating whether the given File was created
// with Google Docs, Google Sheets, etc.
func (f *File) IsGoogleAppsFile() bool {
	return !f.IsFolder() && strings.HasPrefix(f.MimeType, "application/vnd.google-apps.")
}

// Property represents a user-specified property associated with a Drive
// file.
type Property struct {
	Key, Value string
}

// GetProperty returns the property of the given name associated with the
// given file, if the named property is present.  If the property isn't
// present in the fie, then an empty string and an error are returned.
func (f *File) GetProperty(name string) (string, error) {
	for _, prop := range f.Properties {
		if prop.Key == name {
			return prop.Value, nil
		}
{'fraction_non_alphanumeric': 0.11213289824977751, 'fraction_numerical': 0.010877088895481064, 'mean_word_length': 3.1741642591828314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225977', 'n_tokens_mistral': 3158, 'n_tokens_neox': 2989, 'n_words': 1226}