<commit_before>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if src.Scheme == "git+ssh" || src.Scheme == "ssh+git" {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	stdout, err := cmd.CombinedOutput()
	if err != nil {
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
	}
	return ioutil.NopCloser(buf), nil<commit_msg>Better err handling<commit_after>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if src.Scheme == "git+ssh" || src.Scheme == "ssh+git" {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	stdout, err := cmd.CombinedOutput()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
	}
	return ioutil.NopCloser(buf), nil
{'fraction_non_alphanumeric': 0.15713848975848366, 'fraction_numerical': 0.003057169061449098, 'mean_word_length': 3.825221238938053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5955195', 'n_tokens_mistral': 2664, 'n_tokens_neox': 2552, 'n_words': 557}