<commit_before>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

type processPipelineStep func(workingDirectoryPath string, inputFilePath string, args *ProcessArgs) (outputFilePath string, err error)

var defaultPipeline = []processPipelineStep{
	downloadRemote,
	preProcessImage,
	processImage,
}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	var filePath string

	for _, step := range defaultPipeline {
		filePath, err = step(tempDir, filePath, args)
		if err != nil {
			return
		}
	}

	// serve response
	http.ServeFile(w, r, filePath)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, _ string, args *ProcessArgs) (string, error) {
	url := args.Url
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func preProcessImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	if isAnimatedGif(inFile) {
		args.Format = "gif" // Total hack cos format is incorrectly .png on example
		return coalesceAnimatedGif(tempDir, inFile)
	} else {
		return inFile, nil
	}
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	var outErr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &outErr, &outErr
	err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr.Bytes()),
		})
	}

	return outFileWithFormat, err
}

func isAnimatedGif(inFile string) bool {
	// identify -format %n updates-product-click.gif # => 105
	cmd := exec.Command("identify", "-format", "%n", inFile)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := runWithTimeout(cmd, 10*time.Second)
	if err != nil {
		output := string(stderr.Bytes())
		grohl.Log(grohl.Data{
			"identify-error": err,
			"raw-stderr":     output,
		})
	} else {
		output := string(stdout.Bytes())
		numFrames, err := strconv.Atoi(output)
		if err != nil {
			grohl.Log(grohl.Data{
				"identify-error": "non numeric identify output",
				"raw-stdout":     output,
			})
		} else {
			grohl.Log(grohl.Data{
				"processor":  "imagick",
				"num-frames": numFrames,
			})
			return numFrames > 1
		}
	}
	// if anything fucks out assume not animated
	return false
}

func coalesceAnimatedGif(tempDir string, inFile string) (string, error) {
	outFile := filepath.Join(tempDir, "temp")

	// convert do.gif -coalesce temporary.gif
	cmd := exec.Command("convert", inFile, "-coalesce", outFile)
	_ = runWithTimeout(cmd, 60*time.Second)

	return outFile, nil
}
<commit_msg>Make error logging more consistent.<commit_after>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

type processPipelineStep func(workingDirectoryPath string, inputFilePath string, args *ProcessArgs) (outputFilePath string, err error)

var defaultPipeline = []processPipelineStep{
	downloadRemote,
	preProcessImage,
	processImage,
}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	var filePath string

	for _, step := range defaultPipeline {
		filePath, err = step(tempDir, filePath, args)
		if err != nil {
			return
		}
	}

	// serve response
	http.ServeFile(w, r, filePath)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, _ string, args *ProcessArgs) (string, error) {
	url := args.Url
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func preProcessImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	if isAnimatedGif(inFile) {
		args.Format = "gif" // Total hack cos format is incorrectly .png on example
		return coalesceAnimatedGif(tempDir, inFile)
	} else {
		return inFile, nil
	}
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	var outErr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &outErr, &outErr
	err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"step":      "convert",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr.Bytes()),
		})
	}

	return outFileWithFormat, err
}

func isAnimatedGif(inFile string) bool {
	// identify -format %n updates-product-click.gif # => 105
	cmd := exec.Command("identify", "-format", "%n", inFile)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := runWithTimeout(cmd, 10*time.Second)
	if err != nil {
		output := string(stderr.Bytes())
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"step":      "identify",
			"failure":   err,
			"output":    output,
		})
	} else {
		output := string(stdout.Bytes())
		numFrames, err := strconv.Atoi(output)
		if err != nil {
			grohl.Log(grohl.Data{
				"processor": "imagick",
				"step":      "identify",
				"failure":   err,
				"output":    output,
				"message":   "non numeric identify output",
			})
		} else {
			grohl.Log(grohl.Data{
				"processor":  "imagick",
				"step":       "identify",
				"num-frames": numFrames,
			})
			return numFrames > 1
		}
	}
	// if anything fucks out assume not animated
	return false
}

func coalesceAnimatedGif(tempDir string, inFile string) (string, error) {
	outFile := filepath.Join(tempDir, "temp")

	// convert do.gif -coalesce temporary.gif
	cmd := exec.Command("convert", inFile, "-coalesce", outFile)
	_ = runWithTimeout(cmd, 60*time.Second)

	return outFile, nil
}

{'fraction_non_alphanumeric': 0.14142100258117105, 'fraction_numerical': 0.0027170221437304715, 'mean_word_length': 3.8307086614173227, 'pattern_counts': {'":': 37, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2446055', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2607, 'n_words': 664}