<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	api           = "http://api.tumblr.com/v2"
	apiBlog       = api + "/blog/%s.tumblr.com"
	apiPhotoPosts = apiBlog + "/posts?api_key=%s&filter=raw&offset=%d"
)

type fakeCloser struct {
	io.Reader
}

func (f fakeCloser) Close() (err error) {
	return
}

func Tumblr(u url.URL, c *http.Client, _, key string) (
	files []File, err error) {
	for i := int64(0); i < 150; i += 20 {
		reqUrl := apiPhotoPosts
		r, err := c.Get(fmt.Sprintf(reqUrl, u.Path[1:], key, i))
		if err != nil {
			return files, err
		}

		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return files, err
		}
		println(string(r.Request.URL.String()))
		// println(string(body))
		var cr completeResponse
		err = json.Unmarshal(body, &cr)
		if err != nil {
			if cr.Meta.Msg != "" {
				err = errors.New(u.String() + ": " + cr.Meta.Msg)
			}
			return files, err
		}

		for _, rawPost := range cr.Response.Posts {
			var p post
			err = json.Unmarshal(rawPost, &p)
			if err != nil {
				return nil, err
			}

			fileName := strconv.FormatInt(p.Id, 10)
			mtime := time.Unix(p.Timestamp, 0)

			//store metadata
			files = append(files, File{
				Path:  fmt.Sprintf(".%s.json", fileName),
				Mtime: mtime,
				FileFunc: func() (r io.ReadCloser, err error) {
					b, err := json.MarshalIndent(p, "", "\t")
					if err != nil {
						return
					}
					// println(string(b))
					return fakeCloser{bytes.NewReader(b)}, err
				},
			})

			switch p.PostType {
			case "quote", "link", "answer", "audio", "chat":
				//not implemented
				continue

			case "video":
				// println("video source: ", p.Source_url)
				// println("post url: ", p.Post_url)
				u, _ := url.Parse(p.Source_url)
				f, err := YoutubeDl(*u, c, "", "")
				if err != nil || len(f) != 1 {
					continue
				}
				files = append(files, f[0])

			case "text":
				var p textPost
				err = json.Unmarshal(rawPost, &p)
				if err != nil {
					return nil, err
				}
				files = append(files, File{
					Path: fmt.Sprintf(
						"%d.md", i),
					Mtime: mtime,
					FileFunc: func() (r io.ReadCloser, err error) {
						return fakeCloser{strings.NewReader(p.Body)}, nil
					},
				})<commit_msg>implement more tumblr post types<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	api           = "http://api.tumblr.com/v2"
	apiBlog       = api + "/blog/%s.tumblr.com"
	apiPhotoPosts = apiBlog + "/posts?api_key=%s&filter=raw&offset=%d"
)

type fakeCloser struct {
	io.Reader
}

func (f fakeCloser) Close() (err error) {
	return
}

func Tumblr(u url.URL, c *http.Client, _, key string) (
	files []File, err error) {
	for i := int64(0); i < 60; i += 20 {
		reqUrl := apiPhotoPosts
		r, err := c.Get(fmt.Sprintf(reqUrl, u.Path[1:], key, i))
		if err != nil {
			return files, err
		}

		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return files, err
		}
		// println(string(r.Request.URL.String()))
		// println(string(body))
		var cr completeResponse
		err = json.Unmarshal(body, &cr)
		if err != nil {
			if cr.Meta.Msg != "" {
				err = errors.New(u.String() + ": " + cr.Meta.Msg)
			}
			return files, err
		}

		for _, rawPost := range cr.Response.Posts {
			var p post
			err = json.Unmarshal(rawPost, &p)
			if err != nil {
				return nil, err
			}

			fileName := strconv.FormatInt(p.Id, 10)
			mtime := time.Unix(p.Timestamp, 0)

			//store metadata
			files = append(files, File{
				Path:  fmt.Sprintf(".%s.json", fileName),
				Mtime: mtime,
				FileFunc: func() (r io.ReadCloser, err error) {
					b, err := json.MarshalIndent(p, "", "\t")
					if err != nil {
						return
					}
					// println(string(b))
					return fakeCloser{bytes.NewReader(b)}, err
				},
			})

			switch p.PostType {
			case "answer", "audio", "chat":
				//not implemented
				continue
			case "link":
				var p linkPost
				err = json.Unmarshal(rawPost, &p)
				if err != nil {
					return nil, err
				}
				files = append(files, File{
					Path: fmt.Sprintf(
						"%d_link.txt", p.Id),
					Mtime: mtime,
					FileFunc: func() (r io.ReadCloser, err error) {
						return fakeCloser{strings.NewReader(p.Url)}, nil
					},
				})
			case "quote":
				var p quotePost
				err = json.Unmarshal(rawPost, &p)
				if err != nil {
					return nil, err
				}
				files = append(files, File{
					Path: fmt.Sprintf(
						"%d_quote.txt", p.Id),
					Mtime: mtime,
					FileFunc: func() (r io.ReadCloser, err error) {
						return fakeCloser{strings.NewReader(p.Text)}, nil
					},
				})

			case "text":
				var p textPost
				err = json.Unmarshal(rawPost, &p)
				if err != nil {
					return nil, err
				}
				files = append(files, File{
					Path: fmt.Sprintf(
						"%d.md", p.Id),
					Mtime: mtime,
					FileFunc: func() (r io.ReadCloser, err error) {
						return fakeCloser{strings.NewReader(p.Body)}, nil
					},
				})

			case "video":
				// println("video source: ", p.Source_url)
				// println("post url: ", p.Post_url)
				u, _ := url.Parse(p.Source_url)
				f, err := YoutubeDl(*u, c, "", "")
				if err != nil || len(f) != 1 {
					continue
				}
				files = append(files, f[0])
{'fraction_non_alphanumeric': 0.16820493501601055, 'fraction_numerical': 0.005462422301751742, 'mean_word_length': 2.472858077174624, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3765527', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2002, 'n_words': 486}