<commit_before>package main

import (
	".."
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
)

func main() {
	tmp, err := gitmedia.TempFile()
	if err != nil {
		fmt.Println("Error trying to create temp file")
		panic(err)
	}

	sha1Hash := sha1.New()
	md5Hash := md5.New()
	writer := io.MultiWriter(sha1Hash, md5Hash, tmp)

	written, _ := io.Copy(writer, os.Stdin)
	sha := hex.EncodeToString(sha1Hash.Sum(nil))

	meta := &gitmedia.LargeAsset{
		MediaType: "application/vnd.github.large-asset",
		Size:      written,
		MD5:       hex.EncodeToString(md5Hash.Sum(nil)),
		SHA1:      sha,
	}

	mediafile := gitmedia.LocalMediaPath(sha)
	metafile := mediafile + ".json"

	if err := os.Rename(tmp.Name(), mediafile); err != nil {
		fmt.Printf("Unable to move %s to %s\n", tmp.Name(), mediafile)
		panic(err)
	}
	fmt.Println(tmp.Name())

	file, err := os.Create(metafile)
	if err != nil {
		fmt.Printf("Unable to create meta data file: %s\n", metafile)
		panic(err)
	}

	defer file.Close()
	defer func() { os.Remove(tmp.Name()) }()

	enc := json.NewEncoder(io.MultiWriter(os.Stdout, file))
	enc.Encode(meta)
}<commit_msg>ンンー ンンンン ンーンン<commit_after>package main

import (
	".."
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
)

func main() {
	tmp, err := gitmedia.TempFile()
	if err != nil {
		fmt.Println("Error trying to create temp file")
		panic(err)
	}

	sha1Hash := sha1.New()
	md5Hash := md5.New()
	writer := io.MultiWriter(sha1Hash, md5Hash, tmp)

	written, _ := io.Copy(writer, os.Stdin)
	sha := hex.EncodeToString(sha1Hash.Sum(nil))

	meta := &gitmedia.LargeAsset{
		MediaType: "application/vnd.github.large-asset",
		Size:      written,
		MD5:       hex.EncodeToString(md5Hash.Sum(nil)),
		SHA1:      sha,
	}

	mediafile := gitmedia.LocalMediaPath(sha)
	metafile := mediafile + ".json"

	if err := os.Rename(tmp.Name(), mediafile); err != nil {
		fmt.Printf("Unable to move %s to %s\n", tmp.Name(), mediafile)
		panic(err)
	}

	file, err := os.Create(metafile)
	if err != nil {
		fmt.Printf("Unable to create meta data file: %s\n", metafile)
		panic(err)
	}

	defer file.Close()
	defer func() { os.Remove(tmp.Name()) }()

	enc := json.NewEncoder(io.MultiWriter(os.Stdout, file))
	enc.Encode(meta)
}
{'fraction_non_alphanumeric': 0.17556241729157476, 'fraction_numerical': 0.0105866784296427, 'mean_word_length': 3.7348643006263047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964641', 'n_tokens_mistral': 934, 'n_tokens_neox': 898, 'n_words': 187}