<commit_before>// Package conf parses start up args and config file
package conf

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/MG-RAST/golib/goconfig/config"
	"gopkg.in/yaml.v2"
)

// types Config
type TypeConfig struct {
	ID          string `bson:"id" json:"id" yaml:"ID" `                           // e.g. default or Image or Backup
	Description string `bson:"description" json:"description" yaml:"Description"` // e.g. some description
	Priority    int    `bson:"priority" json:"priority" yaml:"Priority"`          // e.g. location priority for pushing files upstream to remote locations, 0 is lowest, 100 highest
}

// Location set of storage locations
type LocationConfig struct {
	ID          string `bson:"ID" json:"ID" yaml:"ID" `                           // e.g. ANLs3 or local for local store
	Type        string `bson:"type" json:"type" yaml:"Type" `                     // e.g. S3
	URL         string `bson:"url" json:"url" yaml:"URL"`                         // e.g. http://s3api.invalid.org/download&id=
	Token       string `bson:"token" json:"-" yaml:"Token" `                      // e.g.  Key or password
	Description string `bson:"Description" json:"Description" yaml:"Description"` // e.g. ANL official S3 service
	Prefix      string `bson:"prefix" json:"-" yaml:"Prefix"`                     // e.g. any prefix needed
	AuthKey     string `bson:"AuthKey" json:"-" yaml:"AuthKey"`                   // e.g. AWS auth-key
	Persistent  bool   `bson:"persistent" json:"persistent" yaml:"Persistent"`    // e.g. is this a valid long term storage location
	Priority    int    `bson:"priority" json:"priority" yaml:"Priority"`          // e.g. f priority for pushing files upstream to this location, 0 is lowest, 100 highest
	MinPriority int    `bson:"minpriority" json:"minpriority" yaml:"MinPriority"` // e.g. minimum node priority level for this location (e.g. some stores will only handle non temporary files or high value files)
	Tier        int    `bson:"tier" json:"tier" yaml:"Tier"`                      // e.g. class or tier 0= cache, 3=ssd based backend, 5=disk based backend, 10=tape archive
	Cost        int    `bson:"cost" json:"cost" yaml:"Cost"`                      // e.g.  cost per GB for this store, default=0
	SecretKey   string `bson:"SecretKey" json:"-" yaml:"SecretKey" `              // e.g.g AWS secret-key
	Bucket      string `bson:"bucket" json:"bucket" yaml:"Bucket" `               // for AWS and GCloud

	S3Location     `bson:",inline" json:",inline" yaml:",inline"` // extensions specific to S3
	AzureLocation  `bson:",inline" json:",inline" yaml:",inline"` // extensions specific to Microsoft Azure
	GCloudLocation `bson:",inline" json:",inline" yaml:",inline"` // extension sspecific to IBM TSM
	IRodsLocation  `bson:",inline" json:",inline" yaml:",inline"` // extension sspecific to IRods
}

// S3Location S3 specific fields
type S3Location struct {
	Region string `bson:"region" json:"region" yaml:"Region" `
}

// AzureLocation Microsoft Azure specific fields
type AzureLocation struct {
	Account   string `bson:"Account" json:"-" yaml:"Account" `             // e.g.g Account name<commit_msg>Added Glacier specific config<commit_after>// Package conf parses start up args and config file
package conf

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/MG-RAST/golib/goconfig/config"
	"gopkg.in/yaml.v2"
)

// types Config
type TypeConfig struct {
	ID          string `bson:"id" json:"id" yaml:"ID" `                           // e.g. default or Image or Backup
	Description string `bson:"description" json:"description" yaml:"Description"` // e.g. some description
	Priority    int    `bson:"priority" json:"priority" yaml:"Priority"`          // e.g. location priority for pushing files upstream to remote locations, 0 is lowest, 100 highest
}

// Location set of storage locations
type LocationConfig struct {
	ID          string `bson:"ID" json:"ID" yaml:"ID" `                           // e.g. ANLs3 or local for local store
	Type        string `bson:"type" json:"type" yaml:"Type" `                     // e.g. S3
	URL         string `bson:"url" json:"url" yaml:"URL"`                         // e.g. http://s3api.invalid.org/download&id=
	Token       string `bson:"token" json:"-" yaml:"Token" `                      // e.g.  Key or password
	Description string `bson:"Description" json:"Description" yaml:"Description"` // e.g. ANL official S3 service
	Prefix      string `bson:"prefix" json:"-" yaml:"Prefix"`                     // e.g. any prefix needed
	AuthKey     string `bson:"AuthKey" json:"-" yaml:"AuthKey"`                   // e.g. AWS auth-key
	Persistent  bool   `bson:"persistent" json:"persistent" yaml:"Persistent"`    // e.g. is this a valid long term storage location
	Priority    int    `bson:"priority" json:"priority" yaml:"Priority"`          // e.g. f priority for pushing files upstream to this location, 0 is lowest, 100 highest
	MinPriority int    `bson:"minpriority" json:"minpriority" yaml:"MinPriority"` // e.g. minimum node priority level for this location (e.g. some stores will only handle non temporary files or high value files)
	Tier        int    `bson:"tier" json:"tier" yaml:"Tier"`                      // e.g. class or tier 0= cache, 3=ssd based backend, 5=disk based backend, 10=tape archive
	Cost        int    `bson:"cost" json:"cost" yaml:"Cost"`                      // e.g.  cost per GB for this store, default=0
	SecretKey   string `bson:"SecretKey" json:"-" yaml:"SecretKey" `              // e.g.g AWS secret-key
	Bucket      string `bson:"bucket" json:"bucket" yaml:"Bucket" `               // for AWS and GCloud

	S3Location      `bson:",inline" json:",inline" yaml:",inline"` // extensions specific to S3
	AzureLocation   `bson:",inline" json:",inline" yaml:",inline"` // extensions specific to Microsoft Azure
	GCloudLocation  `bson:",inline" json:",inline" yaml:",inline"` // extension sspecific to IBM TSM
	IRodsLocation   `bson:",inline" json:",inline" yaml:",inline"` // extension sspecific to IRods
	GlacierLocation `bson:",inline" json:",inline" yaml:",inline"` // extension sspecific to Glacier
}

// S3Location S3 specific fields
type S3Location struct {
	Region string `bson:"region" json:"region" yaml:"Region" `
}

// GlacierLocation specific fields
type GlacierLocation struct {
	Region string `bson:"region" json:"region" yaml:"Region" `
	Vault  string `bson:"vault" json:"vault" yaml:"Vault" `
}

// AzureLocation Microsoft Azure specific fields
type AzureLocation struct {
	Account   string `bson:"Account" json:"-" yaml:"Account" `             // e.g.g Account name
{'fraction_non_alphanumeric': 0.13381016241990762, 'fraction_numerical': 0.007152436298614216, 'mean_word_length': 2.7856739988719683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860623', 'n_tokens_mistral': 2070, 'n_tokens_neox': 1952, 'n_words': 642}