<commit_before>// pgconfig.go provides methods for configuring a node corresponding to if it is
// running a 'master' or 'slave' instance of postgres.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

// configureHBAConf attempts to open the 'pg_hba.conf' file. Once open it will scan
// the file line by line looking for replication settings, and overwrite only those
// settings with the settings required for redundancy on Pagoda Box
func configureHBAConf() error {

	// get the role of the other node in the cluster that is running an instance
	// of postgresql
	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Warn("[pg_config.configureHBAConf] Unable to find another!\n%s\n", err)
	}

	// open the pg_hba.conf
	file := conf.DataDir + "pg_hba.conf"
	f, err := os.Open(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to open '%s'!\n%s\n", file, err)
		return err
	}

	defer f.Close()

	scanner := bufio.NewScanner(f)
	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(local|host)\s*(replication)`)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any line that deals with redundancy.
	for scanner.Scan() {

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// dont care about submatches, just if the string matches, 'skipping' any lines
		// that are custom configurations
		if reFindConfigOption.FindString(scanner.Text()) == "" {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// if the other node is present, write redundancy into the 'entry' (otherwise
	// just leave it out)
	if other != nil {
		entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

host    replication     postgres        %s/32            trust`, other.Ip)
	}

	// write the 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// configurePGConf attempts to open the 'postgresql.conf' file. Once open it will
// scan the file line by line looking for replication settings, and overwrite only
// those settings with the settings required for redundancy on Pagoda Box
func configurePGConf(master bool) error {

	// open the postgresql.conf
	file := conf.DataDir + "postgresql.conf"
	f, err := os.Open(file)
	if err != nil {
		return err
	}

	defer f.Close()

	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(listen_addresses|port|wal_level|archive_mode|archive_command|max_wal_senders|wal_keep_segments|hot_standby|synchronous_standby_names)\s*=\s*`)
	scanner := bufio.NewScanner(f)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any lines that need to be manually configured
	for scanner.Scan() {

		submatch := reFindConfigOption.FindStringSubmatch(scanner.Text())

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// build the 'entry' from all lines that don't match the custom configurations
		if submatch == nil {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// write manual configurations into an 'entry'
	entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

listen_addresses = '0.0.0.0'      # what IP address(es) to listen on;
                                  # comma-separated list of addresses;
                                  # defaults to 'localhost'; use '*' for all
                                  # (change requires restart)
port = %d                     # (change requires restart)
wal_level = hot_standby           # minimal, archive, or hot_standby
                                  # (change requires restart)
archive_mode = on                 # allows archiving to be done
                                  # (change requires restart)
archive_command = 'exit 0'        # command to use to archive a logfile segment
                                  # placeholders: \%p = path of file to archive
                                  #               \%f = file name only
                                  # e.g. 'test ! -f /mnt/server/archivedir/\%f && cp \%p /mnt/server/archivedir/\%f'
max_wal_senders = 10              # max number of walsender processes
                                  # (change requires restart)
wal_keep_segments = 5000          # in logfile segments, 16MB each; 0 disables
hot_standby = on                  # "on" allows queries during recovery
                                  # (change requires restart)
`, conf.PGPort)

	// if this node is currenty 'master' then write one additional configuration
	// into the 'entry'
	if master {
		entry += `
# added only the the node running postgres as 'master'
synchronous_standby_names = slave # standby servers that provide sync rep
                                  # comma-separated list of application_name
                                  # from standby(s); '*' = all
`
	}

	// write 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
	}

	return nil<commit_msg>adding a pgconfig opts thing<commit_after>// pgconfig.go provides methods for configuring a node corresponding to if it is
// running a 'master' or 'slave' instance of postgres.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

//
type pgConfig struct {
	listenAddr string
	master string
}

// configureHBAConf attempts to open the 'pg_hba.conf' file. Once open it will scan
// the file line by line looking for replication settings, and overwrite only those
// settings with the settings required for redundancy on Pagoda Box
func configureHBAConf() error {

	// get the role of the other node in the cluster that is running an instance
	// of postgresql
	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Warn("[pg_config.configureHBAConf] Unable to find another!\n%s\n", err)
	}

	// open the pg_hba.conf
	file := conf.DataDir + "pg_hba.conf"
	f, err := os.Open(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to open '%s'!\n%s\n", file, err)
		return err
	}

	defer f.Close()

	scanner := bufio.NewScanner(f)
	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(local|host)\s*(replication)`)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any line that deals with redundancy.
	for scanner.Scan() {

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// dont care about submatches, just if the string matches, 'skipping' any lines
		// that are custom configurations
		if reFindConfigOption.FindString(scanner.Text()) == "" {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// if the other node is present, write redundancy into the 'entry' (otherwise
	// just leave it out)
	if other != nil {
		entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

host    replication     postgres        %s/32            trust`, other.Ip)
	}

	// write the 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// configurePGConf attempts to open the 'postgresql.conf' file. Once open it will
// scan the file line by line looking for replication settings, and overwrite only
// those settings with the settings required for redundancy on Pagoda Box
func configurePGConf(opts pgConfig) error {

	// open the postgresql.conf
	file := conf.DataDir + "postgresql.conf"
	f, err := os.Open(file)
	if err != nil {
		return err
	}

	defer f.Close()

	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(listen_addresses|port|wal_level|archive_mode|archive_command|max_wal_senders|wal_keep_segments|hot_standby|synchronous_standby_names)\s*=\s*`)
	scanner := bufio.NewScanner(f)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any lines that need to be manually configured
	for scanner.Scan() {

		submatch := reFindConfigOption.FindStringSubmatch(scanner.Text())

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// build the 'entry' from all lines that don't match the custom configurations
		if submatch == nil {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// write manual configurations into an 'entry'
	entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

listen_addresses = '%s'      # what IP address(es) to listen on;
                                  # comma-separated list of addresses;
                                  # defaults to 'localhost'; use '*' for all
                                  # (change requires restart)
port = %d                     # (change requires restart)
wal_level = hot_standby           # minimal, archive, or hot_standby
                                  # (change requires restart)
archive_mode = on                 # allows archiving to be done
                                  # (change requires restart)
archive_command = 'exit 0'        # command to use to archive a logfile segment
                                  # placeholders: \%p = path of file to archive
                                  #               \%f = file name only
                                  # e.g. 'test ! -f /mnt/server/archivedir/\%f && cp \%p /mnt/server/archivedir/\%f'
max_wal_senders = 10              # max number of walsender processes
                                  # (change requires restart)
wal_keep_segments = 5000          # in logfile segments, 16MB each; 0 disables
hot_standby = on                  # "on" allows queries during recovery
                                  # (change requires restart)
`, opts.listenAddr, conf.PGPort)

	// if this node is currenty 'master' then write one additional configuration
	// into the 'entry'
	if opts.master {
		entry += `
# added only the the node running postgres as 'master'
synchronous_standby_names = slave # standby servers that provide sync rep
                                  # comma-separated list of application_name
                                  # from standby(s); '*' = all
`
	}

	// write 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
	}

	return nil
{'fraction_non_alphanumeric': 0.1430794367073835, 'fraction_numerical': 0.0034233252937057495, 'mean_word_length': 2.2541772151898734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241880', 'n_tokens_mistral': 3752, 'n_tokens_neox': 3501, 'n_words': 1313}