<commit_before>package main

import (
	"os"
	"path/filepath"

	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/goutil/ast"
	"github.com/cosiner/gohper/sortedmap"
	"github.com/cosiner/gohper/strings2"
)

type Table struct {
	Name   string
	Fields sortedmap.Map
}

type Visitor map[string]*Table

// add an model and it's field to parse result
func (v Visitor) add(model, table, field, col string) {
	if table == "" {
		table = strings2.ToSnake(model)
	}

	if col == "" {
		col = strings2.ToSnake(field)
	}

	t, has := v[model]
	if !has {
		t = &Table{Name: table}
		v[model] = t
	}

	t.Fields.Set(field, col)
}

// parse ast tree to find exported struct and it's fields
func (v Visitor) parseFiles(files ...string) error {
	for _, file := range files {
		err := v.parseFile(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (v Visitor) parseDir(dir string) error {
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		return v.parseFile(path)
	})
}

func (v Visitor) parseFile(file string) error {
	return ast.Parser{
		Struct: func(a *ast.Attrs) (err error) {
			if !goutil.IsExported(a.TypeName) {
				err = ast.TYPE_END
			} else if table := a.S.Tag.Get("table"); table == "-" {
				err = ast.TYPE_END
			} else if col := a.S.Tag.Get("column"); col != "-" {
				v.add(a.TypeName, table, a.S.Field, col)
			}

			return
		},
	}.ParseFile(file)
}

// buildModelFields build model map from parse result
func (v Visitor) buildModelFields() map[*Model][]*Field {
	names := make(map[*Model][]*Field, len(v))

	for model, table := range v {
		m := NewModel(model, table.Name)
		fields := table.Fields
		for field, index := range fields.Indexes {
			names[m] = append(names[m], NewField(m, field, fields.Elements[index].(string)))
		}
	}

	return names
}<commit_msg>cmd/gomodel: fix bug for fields order<commit_after>package main

import (
	"os"
	"path/filepath"

	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/goutil/ast"
	"github.com/cosiner/gohper/sortedmap"
	"github.com/cosiner/gohper/strings2"
)

type Table struct {
	Name   string
	Fields sortedmap.Map
}

type Visitor map[string]*Table

// add an model and it's field to parse result
func (v Visitor) add(model, table, field, col string) {
	if table == "" {
		table = strings2.ToSnake(model)
	}

	if col == "" {
		col = strings2.ToSnake(field)
	}

	t, has := v[model]
	if !has {
		t = &Table{Name: table}
		v[model] = t
	}

	t.Fields.Set(field, col)
}

// parse ast tree to find exported struct and it's fields
func (v Visitor) parseFiles(files ...string) error {
	for _, file := range files {
		err := v.parseFile(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (v Visitor) parseDir(dir string) error {
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		return v.parseFile(path)
	})
}

func (v Visitor) parseFile(file string) error {
	return ast.Parser{
		Struct: func(a *ast.Attrs) (err error) {
			if !goutil.IsExported(a.TypeName) {
				err = ast.TYPE_END
			} else if table := a.S.Tag.Get("table"); table == "-" {
				err = ast.TYPE_END
			} else if col := a.S.Tag.Get("column"); col != "-" {
				v.add(a.TypeName, table, a.S.Field, col)
			}

			return
		},
	}.ParseFile(file)
}

// buildModelFields build model map from parse result
func (v Visitor) buildModelFields() map[*Model][]*Field {
	names := make(map[*Model][]*Field, len(v))

	for model, table := range v {
		m := NewModel(model, table.Name)
		fields := table.Fields
		for _, field := range fields.Elements {
			names[m] = append(names[m], NewField(m, field.Key, field.Value.(string)))
		}
	}

	return names
}
{'fraction_non_alphanumeric': 0.15249343832020998, 'fraction_numerical': 0.0015748031496062992, 'mean_word_length': 3.3110859728506785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2093495', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1437, 'n_words': 370}