<commit_before>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "strings"
    "time"

    _ "github.com/mattn/go-sqlite3"
    _ "github.com/ziutek/mymysql/godrv"
)

// evo_categories
type Category struct {
    id        int64
    parent_id int64
    name      string
    url       string
}

// evo_items__item
type Post struct {
    id        int64
    id_sqlite int64
    date      time.Time
    body      string
    title     string
    url       string
}

// evo_comments
type Comment struct {
    postId        int64
    postId_sqlite int64
    authorId      sql.NullInt64
    author        sql.NullString
    authorEmail   sql.NullString
    authorUrl     sql.NullString
    authorIp      sql.NullString
    date          time.Time
    content       string
}

// evo_users
type User struct {
    id        int64
    id_sqlite int64
    login     string
    firstname string
    lastname  string
    nickname  string
    email     string
    url       string
    ip        string
}

type DbConfig map[string]string

func xferPosts(sconn, mconn *sql.DB) (posts []*Post, err error) {
    rows, err := mconn.Query(`select post_ID, post_datecreated, post_content,
                                     post_title, post_urltitle
                              from evo_items__item
                              where post_creator_user_ID=?`, 15)
    //posts := make([]*Post, 0, 10)
    for rows.Next() {
        var p Post
        err = rows.Scan(&p.id, &p.date, &p.body, &p.title, &p.url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        posts = append(posts, &p)
    }
    fmt.Printf("#posts: %d\n", len(posts))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, p := range posts {
        stmt, err := xaction.Prepare(`insert into post
                                      (author_id, title, date, url, body)
                                      values(?, ?, ?, ?, ?)`)
        if err != nil {
            fmt.Println(err)
            return posts, err
        }
        defer stmt.Close()
        newBody := fixupBody(p.body)
        result, err := stmt.Exec(1, p.title, p.date.Unix(), p.url, newBody)
        p.id_sqlite, _ = result.LastInsertId()
        //fmt.Printf("%+v\n", p)
        //fmt.Printf("%q | %q\n", p.title, p.url)
    }
    xaction.Commit()
    return posts, err
}

func fixupBody(obody string) (nbody string) {
    nbody = fixupPre(obody)
    nbody = fixupTt(nbody)
    nbody = fixupOl(nbody)
    nbody = fixupImgLinks(nbody)
    return
}

func fixupImgLinks(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    for _, line := range ilines {
        olines = append(olines, strings.Replace(line, "http://blog.stent.lt/media/blogs/rtfb", "", -1))
    }
    nbody = strings.Join(olines, "\n")
    return<commit_msg>dbtool: use smaller headings<commit_after>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "strings"
    "time"

    _ "github.com/mattn/go-sqlite3"
    _ "github.com/ziutek/mymysql/godrv"
)

// evo_categories
type Category struct {
    id        int64
    parent_id int64
    name      string
    url       string
}

// evo_items__item
type Post struct {
    id        int64
    id_sqlite int64
    date      time.Time
    body      string
    title     string
    url       string
}

// evo_comments
type Comment struct {
    postId        int64
    postId_sqlite int64
    authorId      sql.NullInt64
    author        sql.NullString
    authorEmail   sql.NullString
    authorUrl     sql.NullString
    authorIp      sql.NullString
    date          time.Time
    content       string
}

// evo_users
type User struct {
    id        int64
    id_sqlite int64
    login     string
    firstname string
    lastname  string
    nickname  string
    email     string
    url       string
    ip        string
}

type DbConfig map[string]string

func xferPosts(sconn, mconn *sql.DB) (posts []*Post, err error) {
    rows, err := mconn.Query(`select post_ID, post_datecreated, post_content,
                                     post_title, post_urltitle
                              from evo_items__item
                              where post_creator_user_ID=?`, 15)
    //posts := make([]*Post, 0, 10)
    for rows.Next() {
        var p Post
        err = rows.Scan(&p.id, &p.date, &p.body, &p.title, &p.url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        posts = append(posts, &p)
    }
    fmt.Printf("#posts: %d\n", len(posts))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, p := range posts {
        stmt, err := xaction.Prepare(`insert into post
                                      (author_id, title, date, url, body)
                                      values(?, ?, ?, ?, ?)`)
        if err != nil {
            fmt.Println(err)
            return posts, err
        }
        defer stmt.Close()
        newBody := fixupBody(p.body)
        result, err := stmt.Exec(1, p.title, p.date.Unix(), p.url, newBody)
        p.id_sqlite, _ = result.LastInsertId()
        //fmt.Printf("%+v\n", p)
        //fmt.Printf("%q | %q\n", p.title, p.url)
    }
    xaction.Commit()
    return posts, err
}

func fixupBody(obody string) (nbody string) {
    nbody = fixupPre(obody)
    nbody = fixupTt(nbody)
    nbody = fixupOl(nbody)
    nbody = fixupImgLinks(nbody)
    return
}

func fixupImgLinks(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    for _, line := range ilines {
        newline := strings.Replace(line, "http://blog.stent.lt/media/blogs/rtfb", "", -1)
        newline = strings.Replace(newline, "h3", "h4", -1)
        olines = append(olines, newline)
    }
    nbody = strings.Join(olines, "\n")
    return
{'fraction_non_alphanumeric': 0.11618603100516753, 'fraction_numerical': 0.009501583597266211, 'mean_word_length': 1.738475581926061, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5132481', 'n_tokens_mistral': 2042, 'n_tokens_neox': 2001, 'n_words': 461}