<commit_before>package ocrworker

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/couchbaselabs/logg"
)

type ConvertPdf struct {
}

func (c ConvertPdf) preprocess(ocrRequest *OcrRequest) error {

	// write bytes to a temp file

	tmpFileNameInput, err := createTempFileName()
	tmpFileNameInput = fmt.Sprintf("%s.pdf", tmpFileNameInput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameInput)

	tmpFileNameOutput, err := createTempFileName()
	tmpFileNameOutput = fmt.Sprintf("%s.png", tmpFileNameOutput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameOutput)

	err = saveBytesToFileName(ocrRequest.ImgBytes, tmpFileNameInput)
	if err != nil {
		return err
	}

	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Convert PDF on %s -> %s",
		tmpFileNameInput,
		tmpFileNameOutput,
	)

	tmpDir, err_tmpdir := createTempDir()
	if err_tmpdir != nil {
		logg.LogFatal("Error running command: %s.", err_tmpdir)
	}
	defer os.RemoveAll(tmpDir)

	tmpDirFiles := fmt.Sprintf("%s/%s_%s.pdf", tmpDir, ocrRequest.Name, "%03d")
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir is: %s",
		tmpDir,
	)
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir files is: %s",
		tmpDirFiles,
	)

	out_pdftk, err_pdftk := exec.Command(
		"pdftk",
		tmpFileNameInput,
		"burst",
		"dont_ask",
		"output",
		tmpDirFiles,
	).CombinedOutput()
	if err_pdftk != nil {
		logg.LogFatal("Error running command: %s.  out: %s", err_pdftk, out_pdftk)
	}
	logg.LogTo("PREPROCESSOR_WORKER", "output: %v", string(out_pdftk))

	err_walk := filepath.Walk(tmpDir + "/", func(path string, f os.FileInfo, err error) error {
		if err != nil {
			logg.LogFatal("Error running command: %s.", err)
		}
		logg.LogTo("PREPROCESSOR_WORKER", "Path is: %s. Name is: %s.", path, f.Name())
		matched, _ := regexp.MatchString("^.*?_[0-9]{3}\\.pdf", f.Name())
		if matched {
			reg := regexp.MustCompile("(^.*_|\\.pdf)")
			i, _ := strconv.Atoi(reg.ReplaceAllString(f.Name(), ""))
			out_imagemagick, err_imagemagick := exec.Command(
				"convert",
				"-density",
				"300",
				"-depth",
				"8",
				"-alpha",
				"Off",
				path,
				tmpFileNameOutput,
			).CombinedOutput()
			logg.LogTo("PREPROCESSOR_WORKER", "output: %v, error: %v. ", string(out_imagemagick), err_imagemagick)
			// read bytes from output file into ocrRequest.ImgBytes
			resultBytes, err := ioutil.ReadFile(tmpFileNameOutput)
			if err != nil {
				return err
			}

			ocrRequest.ImgFiles[i] = resultBytes
		}
		return nil
	})
	if err_walk != nil {
		logg.LogFatal("Error running command: %s.", err_walk)
	}

	logg.LogTo("PREPROCESSOR_WORKER", "length: %v", len(ocrRequest.ImgFiles))

	return nil
}<commit_msg>Safer way to work with slice<commit_after>package ocrworker

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"

	"github.com/couchbaselabs/logg"
)

type ConvertPdf struct {
}

func (c ConvertPdf) preprocess(ocrRequest *OcrRequest) error {

	// write bytes to a temp file

	tmpFileNameInput, err := createTempFileName()
	tmpFileNameInput = fmt.Sprintf("%s.pdf", tmpFileNameInput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameInput)

	tmpFileNameOutput, err := createTempFileName()
	tmpFileNameOutput = fmt.Sprintf("%s.png", tmpFileNameOutput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameOutput)

	err = saveBytesToFileName(ocrRequest.ImgBytes, tmpFileNameInput)
	if err != nil {
		return err
	}

	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Convert PDF on %s -> %s",
		tmpFileNameInput,
		tmpFileNameOutput,
	)

	tmpDir, err_tmpdir := createTempDir()
	if err_tmpdir != nil {
		logg.LogFatal("Error running command: %s.", err_tmpdir)
	}
	defer os.RemoveAll(tmpDir)

	tmpDirFiles := fmt.Sprintf("%s/%s_%s.pdf", tmpDir, ocrRequest.Name, "%03d")
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir is: %s",
		tmpDir,
	)
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir files is: %s",
		tmpDirFiles,
	)

	out_pdftk, err_pdftk := exec.Command(
		"pdftk",
		tmpFileNameInput,
		"burst",
		"dont_ask",
		"output",
		tmpDirFiles,
	).CombinedOutput()
	if err_pdftk != nil {
		logg.LogFatal("Error running command: %s.  out: %s", err_pdftk, out_pdftk)
	}
	logg.LogTo("PREPROCESSOR_WORKER", "output: %v", string(out_pdftk))

	err_walk := filepath.Walk(tmpDir + "/", func(path string, f os.FileInfo, err error) error {
		if err != nil {
			logg.LogFatal("Error running command: %s.", err)
		}
		logg.LogTo("PREPROCESSOR_WORKER", "Path is: %s. Name is: %s.", path, f.Name())
		matched, _ := regexp.MatchString("^.*?_[0-9]{3}\\.pdf", f.Name())
		if matched {
			out_imagemagick, err_imagemagick := exec.Command(
				"convert",
				"-density",
				"300",
				"-depth",
				"8",
				"-alpha",
				"Off",
				path,
				tmpFileNameOutput,
			).CombinedOutput()
			logg.LogTo("PREPROCESSOR_WORKER", "output: %v, error: %v. ", string(out_imagemagick), err_imagemagick)
			// read bytes from output file into ocrRequest.ImgBytes
			resultBytes, err := ioutil.ReadFile(tmpFileNameOutput)
			if err != nil {
				return err
			}

			ocrRequest.ImgFiles = append(ocrRequest.ImgFiles, resultBytes)
		}
		return nil
	})
	if err_walk != nil {
		logg.LogFatal("Error running command: %s.", err_walk)
	}

	logg.LogTo("PREPROCESSOR_WORKER", "length: %v", len(ocrRequest.ImgFiles))

	return nil
}
{'fraction_non_alphanumeric': 0.15371557902678235, 'fraction_numerical': 0.0033949453036589967, 'mean_word_length': 4.002830188679245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4574846', 'n_tokens_mistral': 2249, 'n_tokens_neox': 2040, 'n_words': 431}