<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goforce/api/soap"
	"github.com/goforce/log"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

type Config struct {
	Url      string      `json:"url"`
	Username string      `json:"username"`
	Password string      `json:"password"`
	Token    string      `json:"token"`
	Path     string      `json:"path"`
	Include  []string    `json:"include"`
	Exclude  []string    `json:"exclude"`
	Hours    int         `json:"hours"`
	EMail    EMailConfig `json:"email"`
	Log      string      `json:"log"`
}

type Context struct {
	Co        *soap.LoginResponse
	Describes map[string]*soap.DescribeSObjectResult
	Report    *Report
	Path      string
}

func main() {

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
		}
	}()

	if len(os.Args) < 2 || (os.Args[1] == "--config" && len(os.Args) < 3) {
		fmt.Println("Usage:")
		fmt.Println("  back-up --config <filename>      - generates sample config file")
		fmt.Println("  back-up <configfilename>         - runs back-up using config file")
		return
	}

	if os.Args[1] == "--config" {
		file, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0666)
		if err != nil {
			fmt.Println("failed to create new config file:", err)
			return
		} else {
			defer file.Close()
			file.WriteString(
				`{
"url": "https://login.salesforce.com",
"username": "<EMAIL>",
"password": "<PASSWORD>",
"token": "<PASSWORD>",
"path": "/path/to/backup/files-{YYYY}-{MM}-{DD}",
"include": [ "objects", "to", "include" ],
"comment-include-exclude": "use either include or exclude, empty or missing include means all objects.",
"hours": 24,
"comment-hours": "set 0 or delete for initial backups, for deltas set to frequency.",
"email": {
  "server": "smtp.server:port",
  "user": "user.if.needed",
  "password": "<PASSWORD>",
  "from": "<EMAIL>",
  "to": [ "<EMAIL>", "<EMAIL>" ]
},
"log":"api.soap.duration",
"comment-log": "api.soap.request api.soap.response"
}`,
			)
			return
		}
	}

	restricted := map[string]bool{
		"contentdocumentlink":      true,
		"ideacomment":              true,
		"vote":                     true,
		"support_document__kav":    true,
		"collaborationgrouprecord": true,
	}

	config := ReadConfigFile(os.Args[1])
	log.On(config.Log)

	report := NewReport(config.EMail)

	connection, err := soap.Login(config.Url, config.Username, config.Password+config.Token)
	if err != nil {
		report.Fatal(err.Error())
	}

	globalDescribe, err := soap.DescribeGlobal(connection)
	if err != nil {
		report.Fatal(err.Error())
	}

	includes := make(map[string]bool)
	for _, s := range config.Include {
		includes[strings.ToLower(s)] = true
	}

	excludes := make(map[string]bool)
	for _, s := range config.Exclude {
		excludes[strings.ToLower(s)] = true
	}

	var since *time.Time
	if config.Hours > 0 {
		t := time.Now().Add(-time.Duration(config.Hours) * time.Hour)
		since = &t
	}

	describes := make(map[string]*soap.DescribeSObjectResult)
	names := make([]string, 0, 100)
	for _, sObject := range globalDescribe.SObjects {
		if len(names) >= 100 {
			AddDescribes(connection, describes, names)
			names = names[0:0]
		}
		names = append(names, sObject.Name)
	}
	if len(names) > 0 {
		AddDescribes(connection, describes, names)
	}

	err = os.MkdirAll(config.Path, 0777)
	if err != nil {
		panic(err)
	}

	context := Context{Co: connection, Describes: describes, Report: report, Path: config.Path}

	for _, sObject := range globalDescribe.SObjects {
		lcname := strings.ToLower(sObject.Name)
		if sObject.Queryable && sObject.Createable && (len(includes) == 0 || includes[lcname]) && !excludes[lcname] && !restricted[lcname] {
			Backup(context, sObject.Name, since)
		}
	}

	report.Send()<commit_msg>removed check for Createable attribute so that backup can be used by readonly profiles<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goforce/api/soap"
	"github.com/goforce/log"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

type Config struct {
	Url      string      `json:"url"`
	Username string      `json:"username"`
	Password string      `json:"password"`
	Token    string      `json:"token"`
	Path     string      `json:"path"`
	Include  []string    `json:"include"`
	Exclude  []string    `json:"exclude"`
	Hours    int         `json:"hours"`
	EMail    EMailConfig `json:"email"`
	Log      string      `json:"log"`
}

type Context struct {
	Co        *soap.LoginResponse
	Describes map[string]*soap.DescribeSObjectResult
	Report    *Report
	Path      string
}

func main() {

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
		}
	}()

	if len(os.Args) < 2 || (os.Args[1] == "--config" && len(os.Args) < 3) {
		fmt.Println("Usage:")
		fmt.Println("  back-up --config <filename>      - generates sample config file")
		fmt.Println("  back-up <configfilename>         - runs back-up using config file")
		return
	}

	if os.Args[1] == "--config" {
		file, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0666)
		if err != nil {
			fmt.Println("failed to create new config file:", err)
			return
		} else {
			defer file.Close()
			file.WriteString(
				`{
"url": "https://login.salesforce.com",
"username": "<EMAIL>",
"password": "<PASSWORD>",
"token": "<PASSWORD>",
"path": "/path/to/backup/files-{YYYY}-{MM}-{DD}",
"include": [ "objects", "to", "include" ],
"comment-include-exclude": "use either include or exclude, empty or missing include means all objects.",
"hours": 24,
"comment-hours": "set 0 or delete for initial backups, for deltas set to frequency.",
"email": {
  "server": "smtp.server:port",
  "user": "user.if.needed",
  "password": "<PASSWORD>",
  "from": "<EMAIL>",
  "to": [ "<EMAIL>", "<EMAIL>" ]
},
"log":"api.soap.duration",
"comment-log": "api.soap.request api.soap.response"
}`,
			)
			return
		}
	}

	restricted := map[string]bool{
		"contentdocumentlink":      true,
		"ideacomment":              true,
		"vote":                     true,
		"support_document__kav":    true,
		"collaborationgrouprecord": true,
	}

	config := ReadConfigFile(os.Args[1])
	log.On(config.Log)

	report := NewReport(config.EMail)

	connection, err := soap.Login(config.Url, config.Username, config.Password+config.Token)
	if err != nil {
		report.Fatal(err.Error())
	}

	globalDescribe, err := soap.DescribeGlobal(connection)
	if err != nil {
		report.Fatal(err.Error())
	}

	includes := make(map[string]bool)
	for _, s := range config.Include {
		includes[strings.ToLower(s)] = true
	}

	excludes := make(map[string]bool)
	for _, s := range config.Exclude {
		excludes[strings.ToLower(s)] = true
	}

	var since *time.Time
	if config.Hours > 0 {
		t := time.Now().Add(-time.Duration(config.Hours) * time.Hour)
		since = &t
	}

	describes := make(map[string]*soap.DescribeSObjectResult)
	names := make([]string, 0, 100)
	for _, sObject := range globalDescribe.SObjects {
		if len(names) >= 100 {
			AddDescribes(connection, describes, names)
			names = names[0:0]
		}
		names = append(names, sObject.Name)
	}
	if len(names) > 0 {
		AddDescribes(connection, describes, names)
	}

	err = os.MkdirAll(config.Path, 0777)
	if err != nil {
		panic(err)
	}

	context := Context{Co: connection, Describes: describes, Report: report, Path: config.Path}

	for _, sObject := range globalDescribe.SObjects {
		lcname := strings.ToLower(sObject.Name)
		if sObject.Queryable && (len(includes) == 0 || includes[lcname]) && !excludes[lcname] && !restricted[lcname] {
			Backup(context, sObject.Name, since)
		}
	}

	report.Send()
{'fraction_non_alphanumeric': 0.17101602861306134, 'fraction_numerical': 0.008213008345476221, 'mean_word_length': 3.5454545454545454, 'pattern_counts': {'":': 44, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5744815', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2681, 'n_words': 601}