<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/Shopify/sarama"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"

	"github.com/tomheon/kafkafs/kafkafs"
)

func unmountOnInt(c chan os.Signal, server *fuse.Server, client *sarama.Client) {
	s := <-c
	fmt.Println("Got signal:", s)
	server.Unmount()
	client.Close()
}

func main() {
	addrs := []string{"localhost:9092"}
	client, err := sarama.NewClient("kafkafs", addrs, &sarama.ClientConfig{MetadataRetries: 10, WaitForElection: 250 * time.Millisecond})
	if err != nil {
		log.Fatalf("Error from client %s", err)
	}

	kClient := kafkafs.NewKafkaClient(client, 1024*1024)

	nfs := pathfs.NewPathNodeFs(kafkafs.NewKafkaRoFs(kClient), nil)

	server, _, err := nodefs.MountFileSystem("/home/edmund/hi", nfs, nil)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	go unmountOnInt(c, server, client)
	server.Serve()
}
<commit_msg>Add command line flags.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"

	"github.com/tomheon/kafkafs/kafkafs"
)

var kafkaAddrs = flag.String("kafkaAddrs", "localhost:9092",
	"Kafka server addresses host1:port1[;host2:port2...]")
var metadataRetries = flag.Int("metadataRetries", 10,
	"Max times to attempt metadata refresh from Kafka before failing")
var waitForElectionMs = flag.Int("waitForElectionMs", 250,
	"Max milliseconds to wait for Kafka leader election before failing")
var maxMsgBytes = flag.Int("maxMsgBytes", 1024*1024*10,
	"Max bytes to pull for a single message")

func unmountOnInt(c chan os.Signal, server *fuse.Server, client *sarama.Client) {
	s := <-c
	fmt.Println("Got signal:", s)
	server.Unmount()
	client.Close()
}

func main() {
	flag.Parse()

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: kafkafs [options] mountpoint\n")
		flag.PrintDefaults()
		os.Exit(2)
	}

	if flag.NArg() != 1 {
		flag.Usage()
	}

	mountpoint := flag.Arg(0)

	addrs := strings.Split(*kafkaAddrs, ";")
	client, err := sarama.NewClient("kafkafs", addrs,
		&sarama.ClientConfig{MetadataRetries: *metadataRetries,
			WaitForElection: time.Duration(*waitForElectionMs) * time.Millisecond})
	if err != nil {
		log.Fatalf("Error from client %s", err)
	}

	kClient := kafkafs.NewKafkaClient(client, int32(*maxMsgBytes))

	nfs := pathfs.NewPathNodeFs(kafkafs.NewKafkaRoFs(kClient), nil)

	server, _, err := nodefs.MountFileSystem(mountpoint, nfs, nil)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	go unmountOnInt(c, server, client)
	server.Serve()
}

{'fraction_non_alphanumeric': 0.15918958031837915, 'fraction_numerical': 0.017004341534008684, 'mean_word_length': 4.821052631578947, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2686002', 'n_tokens_mistral': 1183, 'n_tokens_neox': 1103, 'n_words': 211}