<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"

	"github.com/hashicorp/terraform/terraform"
)

func fetchState(dir string) (*terraform.State, error) {
	file := "terraform.tfstate"
	if flag.Arg(0) != "" {
		file = flag.Arg(0)
	}
	f, err := os.Open(file)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to open '%s'", file))
	}
	defer f.Close()

	state, err := terraform.ReadState(f)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to read state from '%s'", file))
	}
	return state, nil
}
<commit_msg>Clean up state file handling<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/davecgh/go-spew/spew"
	"github.com/hashicorp/terraform/terraform"
)

func fetchState(dir string) (*terraform.State, error) {
	src, err := openState(dir)
	if err != nil {
		return nil, err
	}
	defer src.Close()

	state, err := terraform.ReadState(src)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to read state from src"))
	}
	return state, nil
}

func openState(dir string) (io.ReadCloser, error) {
	file := filepath.Join(dir, "terraform.tfstate")
	spew.Dump(file)
	f, err := os.Open(file)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to open '%s'", file))
	}
	return f, nil
}

{'fraction_non_alphanumeric': 0.1642023346303502, 'fraction_numerical': 0.0015564202334630351, 'mean_word_length': 4.003891050583658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5051769', 'n_tokens_mistral': 512, 'n_tokens_neox': 514, 'n_words': 124}