<commit_before>package core

import (
	"database/sql"
	"errors"
	"log"
	"sort"
	"strconv"
	"sync"

	"github.com/wawandco/transporter/Godeps/_workspace/src/github.com/apaganobeleno/pq"
	"github.com/wawandco/transporter/Godeps/_workspace/src/github.com/go-sql-driver/mysql"

	"github.com/wawandco/transporter/Godeps/_workspace/src/gopkg.in/yaml.v1"
	"github.com/wawandco/transporter/managers"
)

var mu sync.Mutex

func init() {
	mu.Lock()
	defer mu.Unlock()

	if !driverRegistered("postgres") {
		sql.Register("postgres", &pq.Driver{})
	}

	if !driverRegistered("mysql") {
		sql.Register("mysql", &mysql.MySQLDriver{})
	}
}

var migrations []Migration
var manager managers.DatabaseManager
var databaseManagers = map[string]managers.DatabaseManager{
	"postgres": &managers.PostgreSQLManager{},
	"mysql":    &managers.MySQLManager{},
}

const (
	// MigrationsTable is the Db table where we will store migrations
	MigrationsTable = "transporter_migrations"
)

// Add function adds a migration to the migrations array
// So Transporter verifies if the migration is already completed or runs
// the desired migration as needed.
//
// Register checks if migration attempted to be registered id is unique.
func Add(m Migration) {
	migrations = append(migrations, m)
}

// MigrationsTableExists returns true if the table for the migrations already exists.
func MigrationsTableExists(db *sql.DB) bool {
	query := manager.AllMigrationsQuery(MigrationsTable)
	_, err := db.Query(query)
	return err == nil
}

//RunAllMigrationsUp runs pending migrations and stores migration on the migrations table.
func RunAllMigrationsUp(db *sql.DB) {

	//1. Migrations table exists? -> Create if needed
	if !MigrationsTableExists(db) {
		CreateMigrationsTable(db)
	}

	sort.Sort(ByIdentifier(migrations))

	version := DatabaseVersion(db)
	if len(migrations) > 0 && version == migrations[len(migrations)-1].GetID() {
		log.Println("| No migrations to run, DB is on latest version. (" + version + ")")
		return
	}

	var err error
	for _, migration := range migrations {
		// Check if migration is on the database already
		if migration.Pending(db) {
			err = RunMigrationUp(db, &migration)

			if err != nil {
				log.Println("| Could not complete your migration (" + migration.GetID() + "), please check your SQL.")
				log.Println("| " + err.Error())
				break
			}
		}
	}

	version = DatabaseVersion(db)
	if err == nil && version != "" {
		log.Println("| Done, new database version is " + version)
	}
}

// RunMigrationUp runs a single migration up and if success it saves the
// Migration identifier on the migrations table.
func RunMigrationUp(db *sql.DB, m *Migration) error {
	tx, err := dbTransaction(db)
	if err != nil {
		return errors.New("Could not open a db transaction.")
	}

	if m.Up != nil {
		m.Up(tx)
		migerr := tx.err
		err = tx.Commit()

		if err != nil || migerr != nil {
			return migerr
		}

		query := manager.AddMigrationQuery(MigrationsTable, m.GetID())
		_, err = db.Exec(query)
		return nil
	}

	return errors.New("Migration doesnt have Up function defined.")
}

// RunMigrationDown Runs one migration down and if successful it removes the migration from
// the migrations table.
func RunMigrationDown(db *sql.DB, m *Migration) error {
	tx, err := dbTransaction(db)
	if err != nil {
		return errors.New("Could not begin a transaction.")
	}

	if m.Down != nil {
		m.Down(tx)
		migerr := tx.err
		err = tx.Commit()

		if err != nil || migerr != nil {
			return migerr
		}
		query := manager.DeleteMigrationQuery(MigrationsTable, m.GetID())
		_, err = db.Exec(query)
		return nil
	}

	return errors.New("Migration (" + m.GetID() + ") doesn't have Down function defined.")
}

//RunOneMigrationDown Run down last migration that have completed.
func RunOneMigrationDown(db *sql.DB) {
	if !MigrationsTableExists(db) {
		CreateMigrationsTable(db)
	}

	sort.Sort(ByIdentifier(migrations))
	identifier := DatabaseVersion(db)
	id, err := strconv.Atoi(identifier)

	if err != nil {
		log.Println("Sorry, there is no migration to run back")
	}

	for _, mig := range migrations {
		if mig.Identifier-int64(id) == 0 {
			log.Println("| Running " + mig.GetID() + " back")
			err := RunMigrationDown(db, &mig)
			if err == nil {
				version := DatabaseVersion(db)
				if version != "" {
					log.Println("| Done, new database version is " + version)
				} else {
					log.Println("| Done, All existing migrations down.")
				}
			} else {
				log.Println("| Could not rollback your migration (" + mig.GetID() + "), please check your SQL.")
				log.Println("| " + err.Error())
			}
			break
		}
	}

}

//DatabaseVersion returns the latest database version.
func DatabaseVersion(db *sql.DB) string {
	query := manager.LastMigrationQuery(MigrationsTable)
	rows, _ := db.Query(query)
	var identifier string
	for rows.Next() {
		rows.Scan(&identifier)
		break
	}

	return identifier<commit_msg>[bugfix] fallback for database version<commit_after>package core

import (
	"database/sql"
	"errors"
	"log"
	"math"
	"sort"
	"strconv"
	"sync"

	"github.com/wawandco/transporter/Godeps/_workspace/src/github.com/apaganobeleno/pq"
	"github.com/wawandco/transporter/Godeps/_workspace/src/github.com/go-sql-driver/mysql"

	"github.com/wawandco/transporter/Godeps/_workspace/src/gopkg.in/yaml.v1"
	"github.com/wawandco/transporter/managers"
)

var mu sync.Mutex

func init() {
	mu.Lock()
	defer mu.Unlock()

	if !driverRegistered("postgres") {
		sql.Register("postgres", &pq.Driver{})
	}

	if !driverRegistered("mysql") {
		sql.Register("mysql", &mysql.MySQLDriver{})
	}
}

var migrations []Migration
var manager managers.DatabaseManager
var databaseManagers = map[string]managers.DatabaseManager{
	"postgres": &managers.PostgreSQLManager{},
	"mysql":    &managers.MySQLManager{},
}

const (
	// MigrationsTable is the Db table where we will store migrations
	MigrationsTable = "transporter_migrations"
)

// Add function adds a migration to the migrations array
// So Transporter verifies if the migration is already completed or runs
// the desired migration as needed.
//
// Register checks if migration attempted to be registered id is unique.
func Add(m Migration) {
	migrations = append(migrations, m)
}

// MigrationsTableExists returns true if the table for the migrations already exists.
func MigrationsTableExists(db *sql.DB) bool {
	query := manager.AllMigrationsQuery(MigrationsTable)
	_, err := db.Query(query)
	return err == nil
}

//RunAllMigrationsUp runs pending migrations and stores migration on the migrations table.
func RunAllMigrationsUp(db *sql.DB) {

	//1. Migrations table exists? -> Create if needed
	if !MigrationsTableExists(db) {
		CreateMigrationsTable(db)
	}

	sort.Sort(ByIdentifier(migrations))

	version := DatabaseVersion(db)
	if len(migrations) > 0 && version == migrations[len(migrations)-1].GetID() {
		log.Println("| No migrations to run, DB is on latest version. (" + version + ")")
		return
	}

	var err error
	for _, migration := range migrations {
		// Check if migration is on the database already
		if migration.Pending(db) {
			err = RunMigrationUp(db, &migration)

			if err != nil {
				log.Println("| Could not complete your migration (" + migration.GetID() + "), please check your SQL.")
				log.Println("| " + err.Error())
				break
			}
		}
	}

	version = DatabaseVersion(db)
	if err == nil && version != "" {
		log.Println("| Done, new database version is " + version)
	}
}

// RunMigrationUp runs a single migration up and if success it saves the
// Migration identifier on the migrations table.
func RunMigrationUp(db *sql.DB, m *Migration) error {
	tx, err := dbTransaction(db)
	if err != nil {
		return errors.New("Could not open a db transaction.")
	}

	if m.Up != nil {
		m.Up(tx)
		migerr := tx.err
		err = tx.Commit()

		if err != nil || migerr != nil {
			return migerr
		}

		query := manager.AddMigrationQuery(MigrationsTable, m.GetID())
		_, err = db.Exec(query)
		return nil
	}

	return errors.New("Migration doesnt have Up function defined.")
}

// RunMigrationDown Runs one migration down and if successful it removes the migration from
// the migrations table.
func RunMigrationDown(db *sql.DB, m *Migration) error {
	tx, err := dbTransaction(db)
	if err != nil {
		return errors.New("Could not begin a transaction.")
	}

	if m.Down != nil {
		m.Down(tx)
		migerr := tx.err
		err = tx.Commit()

		if err != nil || migerr != nil {
			return migerr
		}
		query := manager.DeleteMigrationQuery(MigrationsTable, m.GetID())
		_, err = db.Exec(query)
		return nil
	}

	return errors.New("Migration (" + m.GetID() + ") doesn't have Down function defined.")
}

//RunOneMigrationDown Run down last migration that have completed.
func RunOneMigrationDown(db *sql.DB) {
	if !MigrationsTableExists(db) {
		CreateMigrationsTable(db)
	}

	sort.Sort(ByIdentifier(migrations))
	identifier := DatabaseVersion(db)
	id, err := strconv.Atoi(identifier)

	if err != nil {
		log.Println("Sorry, there is no migration to run back")
	}

	for _, mig := range migrations {
		if mig.Identifier-int64(id) == 0 {
			log.Println("| Running " + mig.GetID() + " back")
			err := RunMigrationDown(db, &mig)
			if err == nil {
				version := DatabaseVersion(db)
				if version != "" {
					log.Println("| Done, new database version is " + version)
				} else {
					log.Println("| Done, All existing migrations down.")
				}
			} else {
				log.Println("| Could not rollback your migration (" + mig.GetID() + "), please check your SQL.")
				log.Println("| " + err.Error())
			}
			break
		}
	}

}

//DatabaseVersion returns the latest database version.
func DatabaseVersion(db *sql.DB) string {
	query := manager.LastMigrationQuery(MigrationsTable)
	rows, err := db.Query(query)
	if err != nil {
		return strconv.FormatInt(math.MaxInt64, 10)
	}

	var identifier string
	for rows.Next() {
		rows.Scan(&identifier)
		break
	}

	return identifier
{'fraction_non_alphanumeric': 0.12845247766043866, 'fraction_numerical': 0.0018277822908204712, 'mean_word_length': 3.8350515463917527, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '757294', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3263, 'n_words': 905}