<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/omniscale/magnacarto/mml"

	"code.google.com/p/go.net/websocket"

	"github.com/omniscale/magnacarto"
	"github.com/omniscale/magnacarto/builder"
	mapnikBuilder "github.com/omniscale/magnacarto/builder/mapnik"
	"github.com/omniscale/magnacarto/builder/mapserver"
	"github.com/omniscale/magnacarto/config"
	"github.com/omniscale/magnacarto/maps"
	mssPkg "github.com/omniscale/magnacarto/mss"
	"github.com/omniscale/magnacarto/render"

	"github.com/omniscale/go-mapnik"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

type magnaserv struct {
	config       *config.Magnacarto
	builderCache *builder.Cache
	mapnikMaker  builder.MapMaker
}

func (s *magnaserv) styleParams(r *http.Request) (mml string, mss []string) {
	baseDir := s.config.StylesDir
	base := r.FormValue("base")
	if base != "" {
		baseDir = filepath.Join(baseDir, base)
	}

	mml = r.FormValue("mml")
	if mml != "" {
		mml = filepath.Join(baseDir, mml)
	}

	mssList := r.FormValue("mss")
	if mssList != "" {
		for _, f := range strings.Split(mssList, ",") {
			mss = append(mss, filepath.Join(baseDir, f))
		}
	}

	return mml, mss
}

func (s *magnaserv) render(w http.ResponseWriter, r *http.Request) {
	mapReq, err := maps.ParseMapRequest(r)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var maker builder.MapMaker
	renderer := mapReq.Query.Get("RENDERER")
	if renderer == "mapserver" {
		maker = mapserver.Maker
	} else {
		maker = s.mapnikMaker
	}

	styleFile := mapReq.Query.Get("FILE")
	if styleFile == "" {
		mml, mss := s.styleParams(r)
		if mml == "" {
			log.Println("missing mml param in request")
			http.Error(w, "missing mml param", http.StatusBadRequest)
			return
		}

		styleFile, err = s.builderCache.StyleFile(maker, mml, mss)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	var b []byte
	if renderer == "mapserver" {
		mapReq.Format = mapReq.Query.Get("FORMAT") // use requested format, not internal mapnik format
		b, err = render.MapServer(s.config.MapServer.DevBin, styleFile, renderReq(mapReq))
	} else {
		b, err = render.Mapnik(styleFile, renderReq(mapReq))

	}
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	w.Header().Add("Content-Type", "image/png")
	w.Header().Add("Content-Length", strconv.FormatUint(uint64(len(b)), 10))

	io.Copy(w, bytes.NewBuffer(b))
}

func (s *magnaserv) projects(w http.ResponseWriter, r *http.Request) {
	projects, err := findProjects(s.config.StylesDir)
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	w.Header().Add("Content-Type", "application/json")

	enc := json.NewEncoder(w)
	err = enc.Encode(struct {
		Projects []project `json:"projects"`
	}{Projects: projects})
	if err != nil {
		s.internalError(w, r, err)
		return
	}
}

func (s *magnaserv) mml(w http.ResponseWriter, r *http.Request) {
	baseName := mux.Vars(r)["base"]
	mmlName := mux.Vars(r)["mml"]

	fileName, err := filepath.Abs(filepath.Join(s.config.StylesDir, baseName, mmlName+".mml"))
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	if r.Method == "POST" {
		if err := writeCheckedMML(r.Body, fileName); err != nil {
			s.internalError(w, r, err)
			return
		}
		return
	}
	http.ServeFile(w, r, fileName)
}

func (s *magnaserv) mcp(w http.ResponseWriter, r *http.Request) {
	baseName := mux.Vars(r)["base"]
	mcpName := mux.Vars(r)["mcp"]

	fileName, err := filepath.Abs(filepath.Join(s.config.StylesDir, baseName, mcpName+".mcp"))
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	if r.Method == "POST" {
		if err := writeCheckedJSON(r.Body, fileName); err != nil {
			s.internalError(w, r, err)
			return
		}
		return
	}
	http.ServeFile(w, r, fileName)
}

// writeCheckedMML writes MML from io.ReadCloser to fileName.
// Checks if r is a valid MML before (over)writing file.
func writeCheckedMML(r io.ReadCloser, fileName string) error {
	return writeCheckedFile(r, fileName, func(r io.Reader) error {
		_, err := mml.Parse(r)
		return err
	})
	return nil
}

// writeCheckedMML writes JSON from io.ReadCloser to fileName.
// Checks if r is a valid JSON before (over)writing file.
func writeCheckedJSON(r io.ReadCloser, fileName string) error {
	return writeCheckedFile(r, fileName, func(r io.Reader) error {
		d := json.NewDecoder(r)
		tmp := map[string]interface{}{}
		return d.Decode(&tmp)
	})
	return nil
}<commit_msg>return empty JSON if .mcp does not exists but .mml<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/omniscale/magnacarto/mml"

	"code.google.com/p/go.net/websocket"

	"github.com/omniscale/magnacarto"
	"github.com/omniscale/magnacarto/builder"
	mapnikBuilder "github.com/omniscale/magnacarto/builder/mapnik"
	"github.com/omniscale/magnacarto/builder/mapserver"
	"github.com/omniscale/magnacarto/config"
	"github.com/omniscale/magnacarto/maps"
	mssPkg "github.com/omniscale/magnacarto/mss"
	"github.com/omniscale/magnacarto/render"

	"github.com/omniscale/go-mapnik"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

type magnaserv struct {
	config       *config.Magnacarto
	builderCache *builder.Cache
	mapnikMaker  builder.MapMaker
}

func (s *magnaserv) styleParams(r *http.Request) (mml string, mss []string) {
	baseDir := s.config.StylesDir
	base := r.FormValue("base")
	if base != "" {
		baseDir = filepath.Join(baseDir, base)
	}

	mml = r.FormValue("mml")
	if mml != "" {
		mml = filepath.Join(baseDir, mml)
	}

	mssList := r.FormValue("mss")
	if mssList != "" {
		for _, f := range strings.Split(mssList, ",") {
			mss = append(mss, filepath.Join(baseDir, f))
		}
	}

	return mml, mss
}

func (s *magnaserv) render(w http.ResponseWriter, r *http.Request) {
	mapReq, err := maps.ParseMapRequest(r)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var maker builder.MapMaker
	renderer := mapReq.Query.Get("RENDERER")
	if renderer == "mapserver" {
		maker = mapserver.Maker
	} else {
		maker = s.mapnikMaker
	}

	styleFile := mapReq.Query.Get("FILE")
	if styleFile == "" {
		mml, mss := s.styleParams(r)
		if mml == "" {
			log.Println("missing mml param in request")
			http.Error(w, "missing mml param", http.StatusBadRequest)
			return
		}

		styleFile, err = s.builderCache.StyleFile(maker, mml, mss)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	var b []byte
	if renderer == "mapserver" {
		mapReq.Format = mapReq.Query.Get("FORMAT") // use requested format, not internal mapnik format
		b, err = render.MapServer(s.config.MapServer.DevBin, styleFile, renderReq(mapReq))
	} else {
		b, err = render.Mapnik(styleFile, renderReq(mapReq))

	}
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	w.Header().Add("Content-Type", "image/png")
	w.Header().Add("Content-Length", strconv.FormatUint(uint64(len(b)), 10))

	io.Copy(w, bytes.NewBuffer(b))
}

func (s *magnaserv) projects(w http.ResponseWriter, r *http.Request) {
	projects, err := findProjects(s.config.StylesDir)
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	w.Header().Add("Content-Type", "application/json")

	enc := json.NewEncoder(w)
	err = enc.Encode(struct {
		Projects []project `json:"projects"`
	}{Projects: projects})
	if err != nil {
		s.internalError(w, r, err)
		return
	}
}

func (s *magnaserv) mml(w http.ResponseWriter, r *http.Request) {
	baseName := mux.Vars(r)["base"]
	mmlName := mux.Vars(r)["mml"]

	fileName, err := filepath.Abs(filepath.Join(s.config.StylesDir, baseName, mmlName+".mml"))
	if err != nil {
		s.internalError(w, r, err)
		return
	}

	if r.Method == "POST" {
		if err := writeCheckedMML(r.Body, fileName); err != nil {
			s.internalError(w, r, err)
			return
		}
		return
	}
	http.ServeFile(w, r, fileName)
}

func (s *magnaserv) mcp(w http.ResponseWriter, r *http.Request) {
	baseName := mux.Vars(r)["base"]
	mcpName := mux.Vars(r)["mcp"]

	projDir, err := filepath.Abs(filepath.Join(s.config.StylesDir, baseName))
	if err != nil {
		s.internalError(w, r, err)
		return
	}
	mcpFile := filepath.Join(projDir, mcpName+".mcp")

	if r.Method == "POST" {
		if err := writeCheckedJSON(r.Body, mcpFile); err != nil {
			s.internalError(w, r, err)
			return
		}
		return
	}

	// return mcp if exists
	if _, err := os.Stat(mcpFile); err == nil {
		http.ServeFile(w, r, mcpFile)
	} else {
		mmlFile := filepath.Join(projDir, mcpName+".mml")
		// return empty JSON if mml exists
		if _, err := os.Stat(mmlFile); err == nil {
			w.Header().Add("Content-type", "application/json")
			w.Write([]byte("{}\n"))
		} else { // otherwise 404
			http.NotFound(w, r)
		}
	}
}

// writeCheckedMML writes MML from io.ReadCloser to fileName.
// Checks if r is a valid MML before (over)writing file.
func writeCheckedMML(r io.ReadCloser, fileName string) error {
	return writeCheckedFile(r, fileName, func(r io.Reader) error {
		_, err := mml.Parse(r)
		return err
	})
	return nil
}

// writeCheckedMML writes JSON from io.ReadCloser to fileName.
// Checks if r is a valid JSON before (over)writing file.
func writeCheckedJSON(r io.ReadCloser, fileName string) error {
	return writeCheckedFile(r, fileName, func(r io.Reader) error {
		d := json.NewDecoder(r)
		tmp := map[string]interface{}{}
		return d.Decode(&tmp)
	})
	return nil
}
{'fraction_non_alphanumeric': 0.15839120970249818, 'fraction_numerical': 0.0011402508551881414, 'mean_word_length': 3.9553158705701077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3484817', 'n_tokens_mistral': 3994, 'n_tokens_neox': 3827, 'n_words': 787}