<commit_before>package main

import (
	"encoding/csv"
	"os"
	"strconv"
	"strings"
)

type ZipcodeDetails struct {
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	City      string  `json:"city"`
	State     string  `json:"state"`
}

type ZipcodeDatabase struct {
	Zipcodes map[string]*ZipcodeDetails
}

func NewZipcodeDatabase() *ZipcodeDatabase {
	db := &ZipcodeDatabase{}
	db.Zipcodes = make(map[string]*ZipcodeDetails)
	return db
}

func (db *ZipcodeDatabase) Insert(zipcode string, details *ZipcodeDetails) {
	db.Zipcodes[zipcode] = details
}

func (db *ZipcodeDatabase) Find(zipcode string) *ZipcodeDetails {
	return db.Zipcodes[zipcode]
}

func (db *ZipcodeDatabase) LoadFromCSV(filename string) error {
	zips, err := getZipsFromFile(filename)
	if err != nil {
		return err
	}

	for _, row := range zips {
		details, err := getDetailsFromRow(row)
		if err != nil {
			return err
		}
		db.Insert(row[0], details)
	}

	return nil
}

func getZipsFromFile(filename string) ([][]string, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	csvReader := csv.NewReader(reader)
	zips, err := csvReader.ReadAll()
	if err != nil {
		return nil, err
	}

	return zips, nil
}

func getDetailsFromRow(row []string) (*ZipcodeDetails, error) {
	lat, err := strconv.ParseFloat(strings.TrimSpace(row[2]), 64)
	if err != nil {
		return nil, err
	}
	lon, err := strconv.ParseFloat(strings.TrimSpace(row[3]), 64)
	if err != nil {
		return nil, err
	}

	zip := &ZipcodeDetails{
		State:     row[1],
		Latitude:  lat,
		Longitude: lon,
		City:      row[4],
	}

	return zip, nil
}
<commit_msg>Deduplicate string -> float64 conversion<commit_after>package main

import (
	"encoding/csv"
	"os"
	"strconv"
	"strings"
)

type ZipcodeDetails struct {
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	City      string  `json:"city"`
	State     string  `json:"state"`
}

type ZipcodeDatabase struct {
	Zipcodes map[string]*ZipcodeDetails
}

func NewZipcodeDatabase() *ZipcodeDatabase {
	db := &ZipcodeDatabase{}
	db.Zipcodes = make(map[string]*ZipcodeDetails)
	return db
}

func (db *ZipcodeDatabase) Insert(zipcode string, details *ZipcodeDetails) {
	db.Zipcodes[zipcode] = details
}

func (db *ZipcodeDatabase) Find(zipcode string) *ZipcodeDetails {
	return db.Zipcodes[zipcode]
}

func (db *ZipcodeDatabase) LoadFromCSV(filename string) error {
	zips, err := getZipsFromFile(filename)
	if err != nil {
		return err
	}
	for _, row := range zips {
		db.Insert(row[0], getDetailsFromRow(row))
	}
	return nil
}

func getZipsFromFile(filename string) ([][]string, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	csvReader := csv.NewReader(reader)
	zips, err := csvReader.ReadAll()
	if err != nil {
		return nil, err
	}
	return zips, nil
}

func getFloat(stringVal string) float64 {
	floatVal, err := strconv.ParseFloat(strings.TrimSpace(stringVal), 64)
	if err != nil {
		return 0
	} else {
		return floatVal
	}
}

func getDetailsFromRow(row []string) *ZipcodeDetails {
	return &ZipcodeDetails{
		State:     row[1],
		Latitude:  getFloat(row[2]),
		Longitude: getFloat(row[3]),
		City:      row[4],
	}
}

{'fraction_non_alphanumeric': 0.13187152563310686, 'fraction_numerical': 0.008956145768993206, 'mean_word_length': 4.3448844884488445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7264628', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1163, 'n_words': 289}