<commit_before>package main

import (
	"bytes"
	"container/list"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"strconv"
)

const (
	ProductionClientId = "<KEY>"
	PrereleaseClientId = "<KEY>"
	Mobile1ClientId    = "<KEY>"
	RedirectUri        = "https://force-cli.herokuapp.com/auth/callback"
	RedirectUriStaging = "https://force-clistaging.herokuapp.com/auth/callback"
)

var CustomEndpoint = ``

const (
	EndpointProduction = iota
	EndpointTest       = iota
	EndpointPrerelease = iota
	EndpointMobile1    = iota
	EndpointCustom     = iota
)

/*const (
	apiVersion       = "v34.0"
	apiVersionNumber = "34.0"
)*/

type Force struct {
	Credentials ForceCredentials
	Metadata    *ForceMetadata
	Partner     *ForcePartner
}

type ForceCredentials struct {
	AccessToken   string
	Id            string
	UserId        string
	InstanceUrl   string
	IssuedAt      string
	Scope         string
	IsCustomEP    bool
	Namespace     string
	ApiVersion    string
	ForceEndpoint ForceEndpoint
}

type LoginFault struct {
	ExceptionCode    string `xml:"exceptionCode"`
	ExceptionMessage string `xml:"exceptionMessage"`
}

type SoapFault struct {
	FaultCode   string     `xml:"Body>Fault>faultcode"`
	FaultString string     `xml:"Body>Fault>faultstring"`
	Detail      LoginFault `xml:"Body>Fault>detail>LoginFault"`
}

type GenericForceError struct {
	Error_Description string
	Error             string
}

type ForceError struct {
	Message   string
	ErrorCode string
}

type FieldName struct {
	FieldName string
	IsObject bool
}

type SelectStruct struct {
	ObjectName string
	FieldNames []FieldName
}

type ForceEndpoint int

type ForceSobject map[string]interface{}

type ForceCreateRecordResult struct {
	Errors  []string
	Id      string
	Success bool
}

type ForceLimits map[string]ForceLimit

type ForceLimit struct {
	Name      string
	Remaining int64
	Max       int64
}

type ForcePasswordStatusResult struct {
	IsExpired bool
}

type ForcePasswordResetResult struct {
	NewPassword string
}

type ForceRecord map[string]interface{}

type ForceQueryResult struct {
	Done           bool
	Records        []ForceRecord
	TotalSize      int
	NextRecordsUrl string
}

type ForceSobjectsResult struct {
	Encoding     string
	MaxBatchSize int
	Sobjects     []ForceSobject
}

type Result struct {
	Id      string
	Success bool
	Created bool
	Message string
}

type BatchResult struct {
	Results []Result
}

type BatchInfo struct {
	Id                     string `xml:"id"`
	JobId                  string `xml:"jobId"`
	State                  string `xml:"state"`
	CreatedDate            string `xml:"createdDate"`
	SystemModstamp         string `xml:"systemModstamp"`
	NumberRecordsProcessed int    `xml:"numberRecordsProcessed"`
}

type JobInfo struct {
	Id                      string `xml:"id"`
	Operation               string `xml:"operation"`
	Object                  string `xml:"object"`
	CreatedById             string `xml:"createdById"`
	CreatedDate             string `xml:"createdDate"`
	SystemModStamp          string `xml:"systemModstamp"`
	State                   string `xml:"state"`
	ContentType             string `xml:"contentType"`
	ConcurrencyMode         string `xml:"concurrencyMode"`
	NumberBatchesQueued     int    `xml:"numberBatchesQueued"`
	NumberBatchesInProgress int    `xml:"numberBatchesInProgress"`
	NumberBatchesCompleted  int    `xml:"numberBatchesCompleted"`
	NumberBatchesFailed     int    `xml:"numberBatchesFailed"`
	NumberBatchesTotal      int    `xml:"numberBatchesTotal"`
	NumberRecordsProcessed  int    `xml:"numberRecordsProcessed"`
	NumberRetries           int    `xml:"numberRetries"`
	ApiVersion              string `xml:"apiVersion"`
	NumberRecordsFailed     int    `xml:"numberRecordsFailed"`
	TotalProcessingTime     int    `xml:"totalProcessingTime"`
	ApiActiveProcessingTime int    `xml:"apiActiveProcessingTime"`
	ApexProcessingTime      int    `xml:"apexProcessingTime"`
}

type AuraDefinitionBundleResult struct {
	Done           bool
	Records        []ForceRecord
	TotalSize      int
	QueryLocator   string
	Size           int
	EntityTypeName string
	NextRecordsUrl string
}

type AuraDefinitionBundle struct {
	Id               string
	IsDeleted        bool
	DeveloperName    string
	Language         string
	MasterLabel      string
	NamespacePrefix  string
	CreatedDate      string
	CreatedById      string
	LastModifiedDate string
	LastModifiedById string
	SystemModstamp   string
	ApiVersion       int
	Description      string
}

type AuraDefinition struct {
	Id                     string
	IsDeleted              bool
	CreatedDate            string
	CreatedById            string
	LastModifiedDate       string
	LastModifiedById       string
	SystemModstamp         string
	AuraDefinitionBundleId string
	DefType                string
	Format                 string
	Source                 string
}

type ComponentFile struct {
	FileName    string
	ComponentId string
}

type BundleManifest struct {
	Name  string
	Id    string
	Files []ComponentFile
}

func NewForce(creds ForceCredentials) (force *Force) {
	force = new(Force)
	force.Credentials = creds
	force.Metadata = NewForceMetadata(force)
	force.Partner = NewForcePartner(force)
	return
}

func ForceSoapLogin(endpoint ForceEndpoint, username string, password string) (creds ForceCredentials, err error) {
	var surl string
	version := strings.Split(apiVersion, "v")[1]
	switch endpoint {
	case EndpointProduction:
		surl = fmt.Sprintf("https://login.salesforce.com/services/Soap/u/%s", version)
	case EndpointTest:
		surl = fmt.Sprintf("https://test.salesforce.com/services/Soap/u/%s", version)
	case EndpointPrerelease:
		surl = fmt.Sprintf("https://prerelna1.pre.salesforce.com/services/Soap/u/%s", version)
	case EndpointMobile1:
		surl = fmt.Sprintf("https://mobile1.t.salesforce.com/services/Soap/u/%s", version)
	case EndpointCustom:
		surl = fmt.Sprintf("%s/services/Soap/u/%s", CustomEndpoint, version)
	default:
		ErrorAndExit("Unable to login with SOAP. Unknown endpoint type")
	}

	soap := NewSoap(surl, "", "")
	response, err := soap.ExecuteLogin(username, password)
	var result struct {
		SessionId    string `xml:"Body>loginResponse>result>sessionId"`
		Id           string `xml:"Body>loginResponse>result>userId"`
		Instance_url string `xml:"Body>loginResponse>result>serverUrl"`
	}
	var fault SoapFault
	if err = xml.Unmarshal(response, &fault); fault.Detail.ExceptionMessage != "" {
		ErrorAndExit(fault.Detail.ExceptionCode + ": " + fault.Detail.ExceptionMessage)
	}
	if err = xml.Unmarshal(response, &result); err != nil {
		return
	}
	orgid := strings.Split(result.SessionId, "!")[0]
	u, err := url.Parse(result.Instance_url)
	if err != nil {
		return
	}
	instanceUrl := u.Scheme + "://" + u.Host
	identity := u.Scheme + "://" + u.Host + "/id/" + orgid + "/" + result.Id
	creds = ForceCredentials{result.SessionId, identity, result.Id, instanceUrl, "", "", endpoint == EndpointCustom, "", apiVersionNumber, endpoint}

	f := NewForce(creds)
	url := "https://force-cli"
	if version == "dev" {
		url = fmt.Sprintf("%sstaging.herokuapp.com/auth/soaplogin/?id=%s&access_token=%s&instance_url=%s", url, creds.Id, creds.AccessToken, creds.InstanceUrl)
	} else {
		url = fmt.Sprintf("https://force-cli.herokuapp.com/auth/soaplogin/?id=%s&access_token=%s&instance_url=%s", creds.Id, creds.AccessToken, creds.InstanceUrl)
	}

	body, err := f.httpGet(url)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	fmt.Println("Save login was ", string(body))
	return
}

func ForceLogin(endpoint ForceEndpoint) (creds ForceCredentials, err error) {
	ch := make(chan ForceCredentials)
	port, err := startLocalHttpServer(ch)
	var url string

	Redir := RedirectUri
	if Version == "dev" {
		Redir = RedirectUriStaging
	}

	switch endpoint {
	case EndpointProduction:
		url = fmt.Sprintf("https://login.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", ProductionClientId, Redir, port)
	case EndpointTest:
		url = fmt.Sprintf("https://test.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", ProductionClientId, Redir, port)
	case EndpointPrerelease:
		url = fmt.Sprintf("https://prerellogin.pre.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", PrereleaseClientId, Redir, port)
	case EndpointMobile1:
		url = fmt.Sprintf("https://EndpointMobile1.t.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", Mobile1ClientId, Redir, port)
	case EndpointCustom:
		url = fmt.Sprintf("%s/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", CustomEndpoint, ProductionClientId, Redir, port)
	default:
		ErrorAndExit("Unable to login with OAuth. Unknown endpoint type")
	}

	err = Open(url)
	creds = <-ch
	creds.ForceEndpoint = endpoint
	return
}

func (f *Force) GetCodeCoverage(classId string, className string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/query/?q=Select+Id+From+ApexClass+Where+Name+=+'%s'", f.Credentials.InstanceUrl, apiVersion, className)

	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	var result ForceQueryResult
	json.Unmarshal(body, &result)

	classId = result.Records[0]["Id"].(string)
	url = fmt.Sprintf("%s/services/data/%s/tooling/query/?q=Select+Coverage,+NumLinesCovered,+NumLinesUncovered,+ApexTestClassId,+ApexClassorTriggerId+From+ApexCodeCoverage+Where+ApexClassorTriggerId='%s'", f.Credentials.InstanceUrl, apiVersion, classId)

	body, err = f.httpGet(url)
	if err != nil {
		return
	}

	//var result ForceSobjectsResult
	json.Unmarshal(body, &result)
	fmt.Printf("\n%d lines covered\n%d lines not covered\n", int(result.Records[0]["NumLinesCovered"].(float64)), int(result.Records[0]["NumLinesUncovered"].(float64)))
	return
}

func (f *Force) DeleteDataPipeline(id string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline/%s", f.Credentials.InstanceUrl, apiVersion, id)
	_, err = f.httpDelete(url)
	return
}

func (f *Force) UpdateDataPipeline(id string, masterLabel string, scriptContent string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline/%s", f.Credentials.InstanceUrl, apiVersion, id)
	attrs := make(map[string]string)
	attrs["MasterLabel"] = masterLabel
	attrs["ScriptContent"] = scriptContent

	_, err = f.httpPatch(url, attrs)
	return
}

func (f *Force) CreateDataPipeline(name string, masterLabel string, apiVersionNumber string, scriptContent string, scriptType string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline", f.Credentials.InstanceUrl, apiVersion)

	attrs := make(map[string]string)
	attrs["DeveloperName"] = name
	attrs["ScriptType"] = scriptType
	attrs["MasterLabel"] = masterLabel
	attrs["ApiVersion"] = apiVersionNumber
	attrs["ScriptContent"] = scriptContent

	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return

}

func (f *Force) CreateDataPipelineJob(id string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipelineJob", f.Credentials.InstanceUrl, apiVersion)

	attrs := make(map[string]string)
	attrs["DataPipelineId"] = id

	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return

}

func (f *Force) GetDataPipeline(name string) (results ForceQueryResult, err error) {

	query := fmt.Sprintf("SELECT Id, MasterLabel, DeveloperName, ScriptContent, ScriptType FROM DataPipeline Where DeveloperName = '%s'", name)
	results, err = f.QueryDataPipeline(query)

	return

}

func (f *Force) QueryDataPipeline(soql string) (results ForceQueryResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(soql))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &results)

	return

}

func (f *Force) QueryDataPipelineJob(soql string) (results ForceQueryResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(soql))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &results)

	return

}

func (f *Force) GetAuraBundles() (bundles AuraDefinitionBundleResult, definitions AuraDefinitionBundleResult, err error) {
	bundles, err = f.GetAuraBundlesList()
	definitions, err = f.GetAuraBundleDefinitions()
	return
}

func (f *Force) GetAuraBundleDefinitions() (definitions AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape("SELECT Id, Source, AuraDefinitionBundleId, DefType, Format FROM AuraDefinition"))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &definitions)

	if !definitions.Done {
		f.GetMoreAuraBundleDefinitions(&definitions)
	}

	return
}

func (f *Force) GetMoreAuraBundleDefinitions(definitions *AuraDefinitionBundleResult) (err error) {

	isDone := definitions.Done
	nextRecordsUrl := definitions.NextRecordsUrl

	for !isDone {

		moreDefs := new(AuraDefinitionBundleResult)
		aurl := fmt.Sprintf("%s%s", f.Credentials.InstanceUrl, nextRecordsUrl)

		body, err := f.httpGet(aurl)
		if err != nil {
			return err
		}
		json.Unmarshal(body, &moreDefs)

		definitions.Done = moreDefs.Done
		definitions.Records = append(definitions.Records, moreDefs.Records...)

		isDone = moreDefs.Done

		if !isDone {
			nextRecordsUrl = moreDefs.NextRecordsUrl
		}
	}

	return
}

func (f *Force) GetAuraBundlesList() (bundles AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape("SELECT Id, DeveloperName, NamespacePrefix, ApiVersion, Description FROM AuraDefinitionBundle"))
	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &bundles)

	return
}

func (f *Force) GetAuraBundle(bundleName string) (bundles AuraDefinitionBundleResult, definitions AuraDefinitionBundleResult, err error) {
	bundles, err = f.GetAuraBundleByName(bundleName)
	if len(bundles.Records) == 0 {
		ErrorAndExit(fmt.Sprintf("There is no Aura bundle named %q", bundleName))
	}
	bundle := bundles.Records[0]
	definitions, err = f.GetAuraBundleDefinition(bundle["Id"].(string))
	return
}

func (f *Force) GetAuraBundleByName(bundleName string) (bundles AuraDefinitionBundleResult, err error) {
	criteria := fmt.Sprintf(" Where DeveloperName = '%s'", bundleName)

	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(fmt.Sprintf("SELECT Id, DeveloperName, NamespacePrefix, ApiVersion, Description FROM AuraDefinitionBundle%s", criteria)))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &bundles)

	return
}

func (f *Force) GetAuraBundleDefinition(id string) (definitions AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(fmt.Sprintf("SELECT Id, Source, AuraDefinitionBundleId, DefType, Format FROM AuraDefinition WHERE AuraDefinitionBundleId = '%s'", id)))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &definitions)

	return
}

func (f *Force) CreateAuraBundle(bundleName string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/AuraDefinitionBundle", f.Credentials.InstanceUrl, apiVersion)
	attrs := make(map[string]string)
	attrs["DeveloperName"] = bundleName
	attrs["Description"] = "An Aura Bundle"
	attrs["MasterLabel"] = bundleName
	attrs["ApiVersion"] = apiVersionNumber
	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return
}

func (f *Force) CreateAuraComponent(attrs map[string]string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/AuraDefinition", f.Credentials.InstanceUrl, apiVersion)
	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return
}

func (f *Force) ListSobjects() (sobjects []ForceSobject, err error) {
	url := fmt.Sprintf("%s/services/data/%s/sobjects", f.Credentials.InstanceUrl, apiVersion)
	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	var result ForceSobjectsResult
	json.Unmarshal(body, &result)
	sobjects = result.Sobjects
	return
}

func (f *Force) GetSobject(name string) (sobject ForceSobject, err error) {
	url := fmt.Sprintf("%s/services/data/%s/sobjects/%s/describe", f.Credentials.InstanceUrl, apiVersion, name)
	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	json.Unmarshal(body, &sobject)
	return
}

func (f *Force) Query(query string, isTooling bool) (result ForceQueryResult, fieldList *list.List, err error) {
	vurl := ""
	if isTooling == true {
		vurl = fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion, url.QueryEscape(query))
	} else {
		vurl = fmt.Sprintf("%s/services/data/%s/query?q=%s", f.Credentials.InstanceUrl, apiVersion, url.QueryEscape(query))
	}
	body, err := f.httpGet(vurl)
	if err != nil {
		return
	}
	//fmt.Println(string(body))
	//dec := json.NewDecoder(strings.NewReader(string(body)))
	//err = dec.Decode(&result)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	//fieldList = f.DecodeMe(string(body))
	qr := f.DecodeMe2(string(body))
	fmt.Sprintf("%v", qr)
	json.Unmarshal(body, &result)
	if result.Done == false {
		var nextResult ForceQueryResult
		nextResult.NextRecordsUrl = result.NextRecordsUrl
		for nextResult.Done == false {
			nextUrl := fmt.Sprintf("%s%s", f.Credentials.InstanceUrl, nextResult.NextRecordsUrl)
			nextBody, nextErr := f.httpGet(nextUrl)
			if nextErr != nil {
				return
			}
			nextResult.Records = []ForceRecord{}
			json.Unmarshal(nextBody, &nextResult)

			result.Records = append(result.Records, nextResult.Records...)
		}
	}
	return
}

func (f *Force) DecodeMe2(jsonStream string) (result ForceQueryResult) {
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	dec.UseNumber()
	recordsFound := false

	for {
		t, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			ErrorAndExit(err.Error())
		}

		var tokenType = fmt.Sprintf("%T", t)
		var token = fmt.Sprintf("%v", t)
		//fmt.Printf("%s: %s\n", tokenType, token)
		if tokenType == "json.Delim" {

		} else {
			if !recordsFound {
				switch token {
				case "totalSize":
					t, _ = dec.Token()
					v, _ := strconv.Atoi(t.(json.Number).String())
					result.TotalSize = v
				case "done":
					t, _ = dec.Token()
					v, _ := t.(bool)
					result.Done = v
				case "nextRecordsUrl":
					t, _ = dec.Token()
					result.NextRecordsUrl = fmt.Sprintf("%v", t)
				case "records":
					recordsFound = true
				}

			}
		}
		switch tokenType {
			case "{":
				if !recordsFound {
					// This should be the start of the entire json
				} else {
					// Need to set a flag that in the next loop we are adding fields
					// to a child object
				}
			default:
				
		}
	}
	return
}

func (f *Force) DecodeMe(jsonStream string) (result *list.List) {
	type val interface{}
	type keyval struct {
		Key string
		Val val
	}
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	dec.UseNumber()
	//var records = list.New()
	//var currentContainer = new(val)
	var recordsFound = false
	//var stack = make([]string, 0, 0)
	result = list.New()
	SObjecttype := ""
	var isAttributes = false
	for {
		t, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			ErrorAndExit(err.Error())
		}

		var tokenType = fmt.Sprintf("%T", t)
		var token = fmt.Sprintf("%v", t)
		//fmt.Printf("tokenType: %s, token: %s\n", tokenType, token)
		if token == "type" {
			t, err = dec.Token()
			token = fmt.Sprintf("%v", t)
			SObjecttype = token
			//fmt.Printf("case token=type %s\n", SObjecttype)
			spec := f.GetObjectSpec(SObjecttype, result)
			spec.ObjectName = token
		} else if token == "totalSize" {
			t, err = dec.Token()
		} else if token == "done" {
			t, err = dec.Token()
		} else if token == "records" {
			//fmt.Println("Records have been found...")
			recordsFound = true
		} else if token == "attributes" {
			isAttributes = true
		} else if tokenType != "json.Delim" && recordsFound && token != "attributes" && token != "url" {
				//fmt.Printf("\ncase fieldName: %v.%v\n", SObjecttype, token)
				spec := f.GetObjectSpec(SObjecttype, result)
				//fmt.Printf("%v:", token)
				t, err = dec.Token()
				tt := fmt.Sprintf("%T", t)
				//if /*t != nil &&*/ tt != "json.Delim" {
					//fmt.Printf("%T\n", t)
					f.PushFieldName(token, spec, (tt == "json.Delim" || t == nil))
					/*if tt == "json.Number" {
						fmt.Printf("Value: %#v\n",  JSONNumberToString(t, ','))
					} else {
						fmt.Printf("Value: %s\n", t)
					}*/
				//}
				//fmt.Printf("\n")
		} else {
			if err != nil {
				ErrorAndExit(err.Error())
			}
			if token == "url" {
				t, err = dec.Token()
			} else if token == "[" {
				//fmt.Println("Starting Array...")
			} else if token == "]" {
				//fmt.Println("Ending Array...")
			} else if token == "}" {
				if isAttributes {
					isAttributes = false
				} else {
					prev := f.GetPrevObjectSpec(SObjecttype, result)
					if prev != nil {
						SObjecttype = prev.ObjectName
						//fmt.Printf("Prev Obj: %s\n\n", prev.ObjectName)
					} else {
						//fmt.Printf("NO PREV OBJ\n\n")
					}
					//fmt.Println("Ending Object...")
				}
			} else if token == "{" {
				//spec := f.GetObjectSpec(SObjecttype, result)
				//result.PushFront(spec)
				//fmt.Println("Starting Object...")
			}
		}
		f.DumpListStack(result)
	}
	f.DumpListStack(result)
	return
}

func (f *Force) DumpListStack(l *list.List) {
	fmt.Printf("\nDecode Results:\n")
	for e := l.Front(); e != nil; e = e.Next() {
		spec := e.Value.(*SelectStruct)
		fmt.Println(spec.ObjectName)
		for _, v := range spec.FieldNames {
			fmt.Printf("\t%v", v.FieldName)
			if v.IsObject { fmt.Printf(" (Object)\n") } else { fmt.Printf("\n") }
		}
	}
	fmt.Printf("\n\n")
}

func (f *Force) PushFieldName(fieldName string, spec *SelectStruct, IsObject bool) {
	//fmt.Println("Pushing fieldname: ", fieldName)
	for _, v := range spec.FieldNames {
		if v.FieldName == fieldName {
			return
		}
	}
	spec.FieldNames = append(spec.FieldNames, FieldName{fieldName, IsObject})
	return
}

func (f *Force) GetPrevObjectSpec(objectName string, l *list.List) (foundItem *SelectStruct) {
	for e := l.Front(); e != nil; e = e.Next() {
		if e.Value.(*SelectStruct).ObjectName == objectName {
			p := e.Prev()
			if p != nil {
				foundItem = e.Prev().Value.(*SelectStruct)
				return
			}
		}
	}
	return
}
<commit_msg>go fmt<commit_after>package main

import (
	"bytes"
	"container/list"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
)

const (
	ProductionClientId = "<KEY>"
	PrereleaseClientId = "<KEY>"
	Mobile1ClientId    = "<KEY>"
	RedirectUri        = "https://force-cli.herokuapp.com/auth/callback"
	RedirectUriStaging = "https://force-clistaging.herokuapp.com/auth/callback"
)

var CustomEndpoint = ``

const (
	EndpointProduction = iota
	EndpointTest       = iota
	EndpointPrerelease = iota
	EndpointMobile1    = iota
	EndpointCustom     = iota
)

/*const (
	apiVersion       = "v34.0"
	apiVersionNumber = "34.0"
)*/

type Force struct {
	Credentials ForceCredentials
	Metadata    *ForceMetadata
	Partner     *ForcePartner
}

type ForceCredentials struct {
	AccessToken   string
	Id            string
	UserId        string
	InstanceUrl   string
	IssuedAt      string
	Scope         string
	IsCustomEP    bool
	Namespace     string
	ApiVersion    string
	ForceEndpoint ForceEndpoint
}

type LoginFault struct {
	ExceptionCode    string `xml:"exceptionCode"`
	ExceptionMessage string `xml:"exceptionMessage"`
}

type SoapFault struct {
	FaultCode   string     `xml:"Body>Fault>faultcode"`
	FaultString string     `xml:"Body>Fault>faultstring"`
	Detail      LoginFault `xml:"Body>Fault>detail>LoginFault"`
}

type GenericForceError struct {
	Error_Description string
	Error             string
}

type ForceError struct {
	Message   string
	ErrorCode string
}

type FieldName struct {
	FieldName string
	IsObject  bool
}

type SelectStruct struct {
	ObjectName string
	FieldNames []FieldName
}

type ForceEndpoint int

type ForceSobject map[string]interface{}

type ForceCreateRecordResult struct {
	Errors  []string
	Id      string
	Success bool
}

type ForceLimits map[string]ForceLimit

type ForceLimit struct {
	Name      string
	Remaining int64
	Max       int64
}

type ForcePasswordStatusResult struct {
	IsExpired bool
}

type ForcePasswordResetResult struct {
	NewPassword string
}

type ForceRecord map[string]interface{}

type ForceQueryResult struct {
	Done           bool
	Records        []ForceRecord
	TotalSize      int
	NextRecordsUrl string
}

type ForceSobjectsResult struct {
	Encoding     string
	MaxBatchSize int
	Sobjects     []ForceSobject
}

type Result struct {
	Id      string
	Success bool
	Created bool
	Message string
}

type BatchResult struct {
	Results []Result
}

type BatchInfo struct {
	Id                     string `xml:"id"`
	JobId                  string `xml:"jobId"`
	State                  string `xml:"state"`
	CreatedDate            string `xml:"createdDate"`
	SystemModstamp         string `xml:"systemModstamp"`
	NumberRecordsProcessed int    `xml:"numberRecordsProcessed"`
}

type JobInfo struct {
	Id                      string `xml:"id"`
	Operation               string `xml:"operation"`
	Object                  string `xml:"object"`
	CreatedById             string `xml:"createdById"`
	CreatedDate             string `xml:"createdDate"`
	SystemModStamp          string `xml:"systemModstamp"`
	State                   string `xml:"state"`
	ContentType             string `xml:"contentType"`
	ConcurrencyMode         string `xml:"concurrencyMode"`
	NumberBatchesQueued     int    `xml:"numberBatchesQueued"`
	NumberBatchesInProgress int    `xml:"numberBatchesInProgress"`
	NumberBatchesCompleted  int    `xml:"numberBatchesCompleted"`
	NumberBatchesFailed     int    `xml:"numberBatchesFailed"`
	NumberBatchesTotal      int    `xml:"numberBatchesTotal"`
	NumberRecordsProcessed  int    `xml:"numberRecordsProcessed"`
	NumberRetries           int    `xml:"numberRetries"`
	ApiVersion              string `xml:"apiVersion"`
	NumberRecordsFailed     int    `xml:"numberRecordsFailed"`
	TotalProcessingTime     int    `xml:"totalProcessingTime"`
	ApiActiveProcessingTime int    `xml:"apiActiveProcessingTime"`
	ApexProcessingTime      int    `xml:"apexProcessingTime"`
}

type AuraDefinitionBundleResult struct {
	Done           bool
	Records        []ForceRecord
	TotalSize      int
	QueryLocator   string
	Size           int
	EntityTypeName string
	NextRecordsUrl string
}

type AuraDefinitionBundle struct {
	Id               string
	IsDeleted        bool
	DeveloperName    string
	Language         string
	MasterLabel      string
	NamespacePrefix  string
	CreatedDate      string
	CreatedById      string
	LastModifiedDate string
	LastModifiedById string
	SystemModstamp   string
	ApiVersion       int
	Description      string
}

type AuraDefinition struct {
	Id                     string
	IsDeleted              bool
	CreatedDate            string
	CreatedById            string
	LastModifiedDate       string
	LastModifiedById       string
	SystemModstamp         string
	AuraDefinitionBundleId string
	DefType                string
	Format                 string
	Source                 string
}

type ComponentFile struct {
	FileName    string
	ComponentId string
}

type BundleManifest struct {
	Name  string
	Id    string
	Files []ComponentFile
}

func NewForce(creds ForceCredentials) (force *Force) {
	force = new(Force)
	force.Credentials = creds
	force.Metadata = NewForceMetadata(force)
	force.Partner = NewForcePartner(force)
	return
}

func ForceSoapLogin(endpoint ForceEndpoint, username string, password string) (creds ForceCredentials, err error) {
	var surl string
	version := strings.Split(apiVersion, "v")[1]
	switch endpoint {
	case EndpointProduction:
		surl = fmt.Sprintf("https://login.salesforce.com/services/Soap/u/%s", version)
	case EndpointTest:
		surl = fmt.Sprintf("https://test.salesforce.com/services/Soap/u/%s", version)
	case EndpointPrerelease:
		surl = fmt.Sprintf("https://prerelna1.pre.salesforce.com/services/Soap/u/%s", version)
	case EndpointMobile1:
		surl = fmt.Sprintf("https://mobile1.t.salesforce.com/services/Soap/u/%s", version)
	case EndpointCustom:
		surl = fmt.Sprintf("%s/services/Soap/u/%s", CustomEndpoint, version)
	default:
		ErrorAndExit("Unable to login with SOAP. Unknown endpoint type")
	}

	soap := NewSoap(surl, "", "")
	response, err := soap.ExecuteLogin(username, password)
	var result struct {
		SessionId    string `xml:"Body>loginResponse>result>sessionId"`
		Id           string `xml:"Body>loginResponse>result>userId"`
		Instance_url string `xml:"Body>loginResponse>result>serverUrl"`
	}
	var fault SoapFault
	if err = xml.Unmarshal(response, &fault); fault.Detail.ExceptionMessage != "" {
		ErrorAndExit(fault.Detail.ExceptionCode + ": " + fault.Detail.ExceptionMessage)
	}
	if err = xml.Unmarshal(response, &result); err != nil {
		return
	}
	orgid := strings.Split(result.SessionId, "!")[0]
	u, err := url.Parse(result.Instance_url)
	if err != nil {
		return
	}
	instanceUrl := u.Scheme + "://" + u.Host
	identity := u.Scheme + "://" + u.Host + "/id/" + orgid + "/" + result.Id
	creds = ForceCredentials{result.SessionId, identity, result.Id, instanceUrl, "", "", endpoint == EndpointCustom, "", apiVersionNumber, endpoint}

	f := NewForce(creds)
	url := "https://force-cli"
	if version == "dev" {
		url = fmt.Sprintf("%sstaging.herokuapp.com/auth/soaplogin/?id=%s&access_token=%s&instance_url=%s", url, creds.Id, creds.AccessToken, creds.InstanceUrl)
	} else {
		url = fmt.Sprintf("https://force-cli.herokuapp.com/auth/soaplogin/?id=%s&access_token=%s&instance_url=%s", creds.Id, creds.AccessToken, creds.InstanceUrl)
	}

	body, err := f.httpGet(url)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	fmt.Println("Save login was ", string(body))
	return
}

func ForceLogin(endpoint ForceEndpoint) (creds ForceCredentials, err error) {
	ch := make(chan ForceCredentials)
	port, err := startLocalHttpServer(ch)
	var url string

	Redir := RedirectUri
	if Version == "dev" {
		Redir = RedirectUriStaging
	}

	switch endpoint {
	case EndpointProduction:
		url = fmt.Sprintf("https://login.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", ProductionClientId, Redir, port)
	case EndpointTest:
		url = fmt.Sprintf("https://test.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", ProductionClientId, Redir, port)
	case EndpointPrerelease:
		url = fmt.Sprintf("https://prerellogin.pre.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", PrereleaseClientId, Redir, port)
	case EndpointMobile1:
		url = fmt.Sprintf("https://EndpointMobile1.t.salesforce.com/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", Mobile1ClientId, Redir, port)
	case EndpointCustom:
		url = fmt.Sprintf("%s/services/oauth2/authorize?response_type=token&client_id=%s&redirect_uri=%s&state=%d&prompt=login", CustomEndpoint, ProductionClientId, Redir, port)
	default:
		ErrorAndExit("Unable to login with OAuth. Unknown endpoint type")
	}

	err = Open(url)
	creds = <-ch
	creds.ForceEndpoint = endpoint
	return
}

func (f *Force) GetCodeCoverage(classId string, className string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/query/?q=Select+Id+From+ApexClass+Where+Name+=+'%s'", f.Credentials.InstanceUrl, apiVersion, className)

	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	var result ForceQueryResult
	json.Unmarshal(body, &result)

	classId = result.Records[0]["Id"].(string)
	url = fmt.Sprintf("%s/services/data/%s/tooling/query/?q=Select+Coverage,+NumLinesCovered,+NumLinesUncovered,+ApexTestClassId,+ApexClassorTriggerId+From+ApexCodeCoverage+Where+ApexClassorTriggerId='%s'", f.Credentials.InstanceUrl, apiVersion, classId)

	body, err = f.httpGet(url)
	if err != nil {
		return
	}

	//var result ForceSobjectsResult
	json.Unmarshal(body, &result)
	fmt.Printf("\n%d lines covered\n%d lines not covered\n", int(result.Records[0]["NumLinesCovered"].(float64)), int(result.Records[0]["NumLinesUncovered"].(float64)))
	return
}

func (f *Force) DeleteDataPipeline(id string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline/%s", f.Credentials.InstanceUrl, apiVersion, id)
	_, err = f.httpDelete(url)
	return
}

func (f *Force) UpdateDataPipeline(id string, masterLabel string, scriptContent string) (err error) {
	url := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline/%s", f.Credentials.InstanceUrl, apiVersion, id)
	attrs := make(map[string]string)
	attrs["MasterLabel"] = masterLabel
	attrs["ScriptContent"] = scriptContent

	_, err = f.httpPatch(url, attrs)
	return
}

func (f *Force) CreateDataPipeline(name string, masterLabel string, apiVersionNumber string, scriptContent string, scriptType string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipeline", f.Credentials.InstanceUrl, apiVersion)

	attrs := make(map[string]string)
	attrs["DeveloperName"] = name
	attrs["ScriptType"] = scriptType
	attrs["MasterLabel"] = masterLabel
	attrs["ApiVersion"] = apiVersionNumber
	attrs["ScriptContent"] = scriptContent

	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return

}

func (f *Force) CreateDataPipelineJob(id string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/DataPipelineJob", f.Credentials.InstanceUrl, apiVersion)

	attrs := make(map[string]string)
	attrs["DataPipelineId"] = id

	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return

}

func (f *Force) GetDataPipeline(name string) (results ForceQueryResult, err error) {

	query := fmt.Sprintf("SELECT Id, MasterLabel, DeveloperName, ScriptContent, ScriptType FROM DataPipeline Where DeveloperName = '%s'", name)
	results, err = f.QueryDataPipeline(query)

	return

}

func (f *Force) QueryDataPipeline(soql string) (results ForceQueryResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(soql))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &results)

	return

}

func (f *Force) QueryDataPipelineJob(soql string) (results ForceQueryResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(soql))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &results)

	return

}

func (f *Force) GetAuraBundles() (bundles AuraDefinitionBundleResult, definitions AuraDefinitionBundleResult, err error) {
	bundles, err = f.GetAuraBundlesList()
	definitions, err = f.GetAuraBundleDefinitions()
	return
}

func (f *Force) GetAuraBundleDefinitions() (definitions AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape("SELECT Id, Source, AuraDefinitionBundleId, DefType, Format FROM AuraDefinition"))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &definitions)

	if !definitions.Done {
		f.GetMoreAuraBundleDefinitions(&definitions)
	}

	return
}

func (f *Force) GetMoreAuraBundleDefinitions(definitions *AuraDefinitionBundleResult) (err error) {

	isDone := definitions.Done
	nextRecordsUrl := definitions.NextRecordsUrl

	for !isDone {

		moreDefs := new(AuraDefinitionBundleResult)
		aurl := fmt.Sprintf("%s%s", f.Credentials.InstanceUrl, nextRecordsUrl)

		body, err := f.httpGet(aurl)
		if err != nil {
			return err
		}
		json.Unmarshal(body, &moreDefs)

		definitions.Done = moreDefs.Done
		definitions.Records = append(definitions.Records, moreDefs.Records...)

		isDone = moreDefs.Done

		if !isDone {
			nextRecordsUrl = moreDefs.NextRecordsUrl
		}
	}

	return
}

func (f *Force) GetAuraBundlesList() (bundles AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape("SELECT Id, DeveloperName, NamespacePrefix, ApiVersion, Description FROM AuraDefinitionBundle"))
	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &bundles)

	return
}

func (f *Force) GetAuraBundle(bundleName string) (bundles AuraDefinitionBundleResult, definitions AuraDefinitionBundleResult, err error) {
	bundles, err = f.GetAuraBundleByName(bundleName)
	if len(bundles.Records) == 0 {
		ErrorAndExit(fmt.Sprintf("There is no Aura bundle named %q", bundleName))
	}
	bundle := bundles.Records[0]
	definitions, err = f.GetAuraBundleDefinition(bundle["Id"].(string))
	return
}

func (f *Force) GetAuraBundleByName(bundleName string) (bundles AuraDefinitionBundleResult, err error) {
	criteria := fmt.Sprintf(" Where DeveloperName = '%s'", bundleName)

	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(fmt.Sprintf("SELECT Id, DeveloperName, NamespacePrefix, ApiVersion, Description FROM AuraDefinitionBundle%s", criteria)))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &bundles)

	return
}

func (f *Force) GetAuraBundleDefinition(id string) (definitions AuraDefinitionBundleResult, err error) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion,
		url.QueryEscape(fmt.Sprintf("SELECT Id, Source, AuraDefinitionBundleId, DefType, Format FROM AuraDefinition WHERE AuraDefinitionBundleId = '%s'", id)))

	body, err := f.httpGet(aurl)
	if err != nil {
		return
	}
	json.Unmarshal(body, &definitions)

	return
}

func (f *Force) CreateAuraBundle(bundleName string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/AuraDefinitionBundle", f.Credentials.InstanceUrl, apiVersion)
	attrs := make(map[string]string)
	attrs["DeveloperName"] = bundleName
	attrs["Description"] = "An Aura Bundle"
	attrs["MasterLabel"] = bundleName
	attrs["ApiVersion"] = apiVersionNumber
	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return
}

func (f *Force) CreateAuraComponent(attrs map[string]string) (result ForceCreateRecordResult, err error, emessages []ForceError) {
	aurl := fmt.Sprintf("%s/services/data/%s/tooling/sobjects/AuraDefinition", f.Credentials.InstanceUrl, apiVersion)
	body, err, emessages := f.httpPost(aurl, attrs)
	if err != nil {
		return
	}
	json.Unmarshal(body, &result)

	return
}

func (f *Force) ListSobjects() (sobjects []ForceSobject, err error) {
	url := fmt.Sprintf("%s/services/data/%s/sobjects", f.Credentials.InstanceUrl, apiVersion)
	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	var result ForceSobjectsResult
	json.Unmarshal(body, &result)
	sobjects = result.Sobjects
	return
}

func (f *Force) GetSobject(name string) (sobject ForceSobject, err error) {
	url := fmt.Sprintf("%s/services/data/%s/sobjects/%s/describe", f.Credentials.InstanceUrl, apiVersion, name)
	body, err := f.httpGet(url)
	if err != nil {
		return
	}
	json.Unmarshal(body, &sobject)
	return
}

func (f *Force) Query(query string, isTooling bool) (result ForceQueryResult, fieldList *list.List, err error) {
	vurl := ""
	if isTooling == true {
		vurl = fmt.Sprintf("%s/services/data/%s/tooling/query?q=%s", f.Credentials.InstanceUrl, apiVersion, url.QueryEscape(query))
	} else {
		vurl = fmt.Sprintf("%s/services/data/%s/query?q=%s", f.Credentials.InstanceUrl, apiVersion, url.QueryEscape(query))
	}
	body, err := f.httpGet(vurl)
	if err != nil {
		return
	}
	//fmt.Println(string(body))
	//dec := json.NewDecoder(strings.NewReader(string(body)))
	//err = dec.Decode(&result)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	//fieldList = f.DecodeMe(string(body))
	qr := f.DecodeMe2(string(body))
	fmt.Sprintf("%v", qr)
	json.Unmarshal(body, &result)
	if result.Done == false {
		var nextResult ForceQueryResult
		nextResult.NextRecordsUrl = result.NextRecordsUrl
		for nextResult.Done == false {
			nextUrl := fmt.Sprintf("%s%s", f.Credentials.InstanceUrl, nextResult.NextRecordsUrl)
			nextBody, nextErr := f.httpGet(nextUrl)
			if nextErr != nil {
				return
			}
			nextResult.Records = []ForceRecord{}
			json.Unmarshal(nextBody, &nextResult)

			result.Records = append(result.Records, nextResult.Records...)
		}
	}
	return
}

func (f *Force) DecodeMe2(jsonStream string) (result ForceQueryResult) {
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	dec.UseNumber()
	recordsFound := false

	for {
		t, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			ErrorAndExit(err.Error())
		}

		var tokenType = fmt.Sprintf("%T", t)
		var token = fmt.Sprintf("%v", t)
		//fmt.Printf("%s: %s\n", tokenType, token)
		if tokenType == "json.Delim" {

		} else {
			if !recordsFound {
				switch token {
				case "totalSize":
					t, _ = dec.Token()
					v, _ := strconv.Atoi(t.(json.Number).String())
					result.TotalSize = v
				case "done":
					t, _ = dec.Token()
					v, _ := t.(bool)
					result.Done = v
				case "nextRecordsUrl":
					t, _ = dec.Token()
					result.NextRecordsUrl = fmt.Sprintf("%v", t)
				case "records":
					recordsFound = true
				}

			}
		}
		switch tokenType {
		case "{":
			if !recordsFound {
				// This should be the start of the entire json
			} else {
				// Need to set a flag that in the next loop we are adding fields
				// to a child object
			}
		default:

		}
	}
	return
}

func (f *Force) DecodeMe(jsonStream string) (result *list.List) {
	type val interface{}
	type keyval struct {
		Key string
		Val val
	}
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	dec.UseNumber()
	//var records = list.New()
	//var currentContainer = new(val)
	var recordsFound = false
	//var stack = make([]string, 0, 0)
	result = list.New()
	SObjecttype := ""
	var isAttributes = false
	for {
		t, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			ErrorAndExit(err.Error())
		}

		var tokenType = fmt.Sprintf("%T", t)
		var token = fmt.Sprintf("%v", t)
		//fmt.Printf("tokenType: %s, token: %s\n", tokenType, token)
		if token == "type" {
			t, err = dec.Token()
			token = fmt.Sprintf("%v", t)
			SObjecttype = token
			//fmt.Printf("case token=type %s\n", SObjecttype)
			spec := f.GetObjectSpec(SObjecttype, result)
			spec.ObjectName = token
		} else if token == "totalSize" {
			t, err = dec.Token()
		} else if token == "done" {
			t, err = dec.Token()
		} else if token == "records" {
			//fmt.Println("Records have been found...")
			recordsFound = true
		} else if token == "attributes" {
			isAttributes = true
		} else if tokenType != "json.Delim" && recordsFound && token != "attributes" && token != "url" {
			//fmt.Printf("\ncase fieldName: %v.%v\n", SObjecttype, token)
			spec := f.GetObjectSpec(SObjecttype, result)
			//fmt.Printf("%v:", token)
			t, err = dec.Token()
			tt := fmt.Sprintf("%T", t)
			//if /*t != nil &&*/ tt != "json.Delim" {
			//fmt.Printf("%T\n", t)
			f.PushFieldName(token, spec, (tt == "json.Delim" || t == nil))
			/*if tt == "json.Number" {
				fmt.Printf("Value: %#v\n",  JSONNumberToString(t, ','))
			} else {
				fmt.Printf("Value: %s\n", t)
			}*/
			//}
			//fmt.Printf("\n")
		} else {
			if err != nil {
				ErrorAndExit(err.Error())
			}
			if token == "url" {
				t, err = dec.Token()
			} else if token == "[" {
				//fmt.Println("Starting Array...")
			} else if token == "]" {
				//fmt.Println("Ending Array...")
			} else if token == "}" {
				if isAttributes {
					isAttributes = false
				} else {
					prev := f.GetPrevObjectSpec(SObjecttype, result)
					if prev != nil {
						SObjecttype = prev.ObjectName
						//fmt.Printf("Prev Obj: %s\n\n", prev.ObjectName)
					} else {
						//fmt.Printf("NO PREV OBJ\n\n")
					}
					//fmt.Println("Ending Object...")
				}
			} else if token == "{" {
				//spec := f.GetObjectSpec(SObjecttype, result)
				//result.PushFront(spec)
				//fmt.Println("Starting Object...")
			}
		}
		f.DumpListStack(result)
	}
	f.DumpListStack(result)
	return
}

func (f *Force) DumpListStack(l *list.List) {
	fmt.Printf("\nDecode Results:\n")
	for e := l.Front(); e != nil; e = e.Next() {
		spec := e.Value.(*SelectStruct)
		fmt.Println(spec.ObjectName)
		for _, v := range spec.FieldNames {
			fmt.Printf("\t%v", v.FieldName)
			if v.IsObject {
				fmt.Printf(" (Object)\n")
			} else {
				fmt.Printf("\n")
			}
		}
	}
	fmt.Printf("\n\n")
}

func (f *Force) PushFieldName(fieldName string, spec *SelectStruct, IsObject bool) {
	//fmt.Println("Pushing fieldname: ", fieldName)
	for _, v := range spec.FieldNames {
		if v.FieldName == fieldName {
			return
		}
	}
	spec.FieldNames = append(spec.FieldNames, FieldName{fieldName, IsObject})
	return
}

func (f *Force) GetPrevObjectSpec(objectName string, l *list.List) (foundItem *SelectStruct) {
	for e := l.Front(); e != nil; e = e.Next() {
		if e.Value.(*SelectStruct).ObjectName == objectName {
			p := e.Prev()
			if p != nil {
				foundItem = e.Prev().Value.(*SelectStruct)
				return
			}
		}
	}
	return
}

{'fraction_non_alphanumeric': 0.1347058569640468, 'fraction_numerical': 0.001640122577582114, 'mean_word_length': 3.968264179264501, 'pattern_counts': {'":': 16, '<': 11, '<?xml version=': 0, '>': 41, 'https://': 24, 'lorem ipsum': 0, 'www.': 0, 'xml': 76}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751440', 'n_tokens_mistral': 16361, 'n_tokens_neox': 15407, 'n_words': 3403}