<commit_before>package config

import (
	"io/ioutil"

	log "github.com/sirupsen/logrus"
)

var initContainerTemplate = `
- name: init-pytorch
  image: alpine:3.10
  imagePullPolicy: IfNotPresent
  command: ['sh', '-c', 'until nslookup {{.MasterAddr}}; do echo waiting for master; sleep 2; done;']`

func init() {
	bytes, err := ioutil.ReadFile("/etc/config/initContainer.yaml")
	if err != nil {
		log.Info("Using default init container template")
	} else {
		log.Info("Using init container template from /etc/config/initContainer.yaml")
		initContainerTemplate = string(bytes)
	}
}
<commit_msg>fix: Add resource limits for init container (#253)<commit_after>package config

import (
	"io/ioutil"

	log "github.com/sirupsen/logrus"
)

var initContainerTemplate = `
- name: init-pytorch
  image: alpine:3.10
  imagePullPolicy: IfNotPresent
  resources:
    limits:
      cpu: 100m
      memory: 10Mi
    requests:
      cpu: 10m
      memory: 1Mi
  command: ['sh', '-c', 'until nslookup {{.MasterAddr}}; do echo waiting for master; sleep 2; done;']`

func init() {
	bytes, err := ioutil.ReadFile("/etc/config/initContainer.yaml")
	if err != nil {
		log.Info("Using default init container template")
	} else {
		log.Info("Using init container template from /etc/config/initContainer.yaml")
		initContainerTemplate = string(bytes)
	}
}

{'fraction_non_alphanumeric': 0.13288288288288289, 'fraction_numerical': 0.014264264264264264, 'mean_word_length': 4.030188679245283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5496131', 'n_tokens_mistral': 476, 'n_tokens_neox': 449, 'n_words': 118}