<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strconv"
)

var (
	pretty = flag.Bool("p", false, "indent (pretty print) output")
)

var allowedImports = map[string]struct{}{
	"fmt": struct{}{},
	"log": struct{}{},
}

type block struct {
	Statements []statement `json:"statements"`
}

type dataType struct {
	Name string `json:"name"`
}

type parameter struct {
	Name     string   `json:"name"`
	DataType dataType `json:"data-type"`
}

type statement struct {
	Line       int         `json:"line"`
	Type       string      `json:"type"`
	Name       string      `json:"name"`
	ReturnType dataType    `json:"return-type"`
	Parameters []parameter `json:"parameters"`
	Block      block       `json:"block"`
}

type superAST struct {
	fset      *token.FileSet
	rootBlock block
}

func (a *superAST) Visit(node ast.Node) ast.Visitor {
	if node == nil {
		return nil
	}
	pos := a.fset.Position(node.Pos())
	log.Printf("%#v", pos)
	log.Printf("%T", node)
	switch x := node.(type) {
	case *ast.BasicLit:
	case *ast.BlockStmt:
	case *ast.CallExpr:
	case *ast.ExprStmt:
	case *ast.FieldList:
	case *ast.File:
		pname := x.Name.Name
		if pname != "main" {
			log.Fatalf(`Package name is not "main": "%s"`, pname)
		}
		imports := x.Imports
		for _, imp := range imports {
			path, err := strconv.Unquote(imp.Path.Value)
			if err != nil {
				log.Fatalf("Error when unquoting import: %s", err)
			}
			if _, e := allowedImports[path]; !e {
				log.Fatalf(`Import path not allowed: "%s"`, path)
			}
		}
	case *ast.FuncDecl:
		name := x.Name.Name
		var params, results []*ast.Field
		if x.Type.Params != nil {
			params = x.Type.Params.List
		}
		if x.Type.Results != nil {
			results = x.Type.Results.List
		}
		log.Printf("func %s %v %v", name, params, results)
		function := statement{
			Line: pos.Line,
			Type: "function-call",
			Name: name,
			ReturnType: dataType{
				Name: "int",
			},
			Parameters: nil,
		}
		a.rootBlock.Statements = append(a.rootBlock.Statements, function)
	case *ast.FuncType:
	case *ast.GenDecl:
	case *ast.Ident:
	case *ast.ImportSpec:
	case *ast.SelectorExpr:
	default:
		log.Printf("Uncatched ast.Node type: %T\n", node)
	}
	return a
}

func main() {
	flag.Parse()
	fset := token.NewFileSet()
	src := `
package main

import "fmt"

func main() {<commit_msg>Use non-nil slices to use [] instead of null<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strconv"
)

var (
	pretty = flag.Bool("p", false, "indent (pretty print) output")
)

var allowedImports = map[string]struct{}{
	"fmt": struct{}{},
	"log": struct{}{},
}

type block struct {
	Statements []statement `json:"statements"`
}

type dataType struct {
	Name string `json:"name"`
}

type parameter struct {
	Name     string   `json:"name"`
	DataType dataType `json:"data-type"`
}

type statement struct {
	Line       int         `json:"line"`
	Type       string      `json:"type"`
	Name       string      `json:"name"`
	ReturnType dataType    `json:"return-type"`
	Parameters []parameter `json:"parameters"`
	Block      block       `json:"block"`
}

type superAST struct {
	fset      *token.FileSet
	rootBlock block
}

func (a *superAST) Visit(node ast.Node) ast.Visitor {
	if node == nil {
		return nil
	}
	pos := a.fset.Position(node.Pos())
	log.Printf("%#v", pos)
	log.Printf("%T", node)
	switch x := node.(type) {
	case *ast.BasicLit:
	case *ast.BlockStmt:
	case *ast.CallExpr:
	case *ast.ExprStmt:
	case *ast.FieldList:
	case *ast.File:
		pname := x.Name.Name
		if pname != "main" {
			log.Fatalf(`Package name is not "main": "%s"`, pname)
		}
		imports := x.Imports
		for _, imp := range imports {
			path, err := strconv.Unquote(imp.Path.Value)
			if err != nil {
				log.Fatalf("Error when unquoting import: %s", err)
			}
			if _, e := allowedImports[path]; !e {
				log.Fatalf(`Import path not allowed: "%s"`, path)
			}
		}
	case *ast.FuncDecl:
		name := x.Name.Name
		var params, results []*ast.Field
		if x.Type.Params != nil {
			params = x.Type.Params.List
		}
		if x.Type.Results != nil {
			results = x.Type.Results.List
		}
		log.Printf("func %s %v %v", name, params, results)
		function := statement{
			Line: pos.Line,
			Type: "function-call",
			Name: name,
			ReturnType: dataType{
				Name: "int",
			},
			Parameters: make([]parameter, 0),
			Block: block{
				Statements: make([]statement, 0),
			},
		}
		a.rootBlock.Statements = append(a.rootBlock.Statements, function)
	case *ast.FuncType:
	case *ast.GenDecl:
	case *ast.Ident:
	case *ast.ImportSpec:
	case *ast.SelectorExpr:
	default:
		log.Printf("Uncatched ast.Node type: %T\n", node)
	}
	return a
}

func main() {
	flag.Parse()
	fset := token.NewFileSet()
	src := `
package main

import "fmt"

func main() {
{'fraction_non_alphanumeric': 0.16338786498239802, 'fraction_numerical': 0.0004141644232760406, 'mean_word_length': 3.2480211081794197, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2661268', 'n_tokens_mistral': 1915, 'n_tokens_neox': 1804, 'n_words': 455}