<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

// Convert MS Word DOC
func ConvertDoc(r io.Reader) (string, map[string]string) {
	f, err := ioutil.TempFile("/tmp", "sajari-convert-")
	if err != nil {
		log.Println("TempFile:", err)
	}
	defer os.Remove(f.Name())
	io.Copy(f, r)

	// Meta data
	mc := make(chan map[string]string, 1)
	go func() {
		meta := make(map[string]string)
		metaStr, err := exec.Command("wvSummary", f.Name()).Output()
		if err != nil {
			log.Println("wvSummary:", err)
		}

		// Parse meta output
		info := make(map[string]string)
		for _, line := range strings.Split(string(metaStr), "\n") {
			if parts := strings.SplitN(line, "=", 2); len(parts) > 1 {
				info[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
			}
		}

		// Convert parsed meta
		if tmp, ok := info["Last Modified"]; ok {
			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
				meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if tmp, ok := info["Created"]; ok {
			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
				meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}<commit_msg>Update ConvertDoc to use LocalFile.<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

// Convert MS Word DOC
func ConvertDoc(r io.Reader) (string, map[string]string) {
	f, err := NewLocalFile(r, "/tmp", "sajari-convert-")
	if err != nil {
		log.Println("error creating local file:", err)
		return "", nil
	}
	defer f.Done()

	// Meta data
	mc := make(chan map[string]string, 1)
	go func() {
		meta := make(map[string]string)
		metaStr, err := exec.Command("wvSummary", f.Name()).Output()
		if err != nil {
			log.Println("wvSummary:", err)
		}

		// Parse meta output
		info := make(map[string]string)
		for _, line := range strings.Split(string(metaStr), "\n") {
			if parts := strings.SplitN(line, "=", 2); len(parts) > 1 {
				info[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
			}
		}

		// Convert parsed meta
		if tmp, ok := info["Last Modified"]; ok {
			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
				meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if tmp, ok := info["Created"]; ok {
			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
				meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
{'fraction_non_alphanumeric': 0.20057899090157155, 'fraction_numerical': 0.010752688172043012, 'mean_word_length': 3.438532110091743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3963651', 'n_tokens_mistral': 1033, 'n_tokens_neox': 928, 'n_words': 218}