<commit_before>package commands

import (
    "fmt"
    "os"
    "path/filepath"

    "github.com/codegangsta/cli"
    "github.com/AlasdairF/File"

    "github.com/nbremond/double-kill/models"
    "github.com/nbremond/double-kill/helpers"
)


var CmdSearch = cli.Command{
    Name:  "search",
    Usage: "search path",
    Description: `Index files in database`,
    Before: runSearch,
    Action: func(ctx *cli.Context) {},
    Flags:  []cli.Flag{},
}


func runSearch(c *cli.Context) error {
    var err error

    if err = models.InitDB(); err != nil {
        return err
    }
    basePath := filepath.Clean(c.Args()[0])+"/"

    fmt.Println("Removing deleted subfiles of «"+basePath +"»…")
    dbFiles := models.GetSubfiles(basePath)
    //fmt.Println(dbFiles)
    for pos := range dbFiles {
        forFile := dbFiles[pos]
        if ! file.Exists(filepath.Join(forFile.Dir, forFile.Filename)){
            fmt.Println("Removing «"+forFile.Filename+"» from database.")
            forFile.Delete()
        }
    }
    fmt.Println("Indexing new files…")
    filepath.Walk(c.Args()[0],indexFile)
    return nil
}

func indexFile(path string, info os.FileInfo, err error) error {
    if info.IsDir() {
    }else{// path isn' t a dir. We must check if it's a regular file.
        dir, filename := filepath.Split(path)
        upToDate := true
        isNew := false
        dbFile := models.GetOrCreateFile(dir, filename)
        if dbFile.Id == 0 {
            upToDate = false
            isNew = true
        }
        if dbFile.ModTime != info.ModTime(){
            dbFile.ModTime = info.ModTime()
            upToDate = false
        }
        if dbFile.Size != info.Size(){
            dbFile.Size = info.Size()
            upToDate = false
        }
        if ! upToDate {
            dbFile.TinyHash = ""
            dbFile.Hash = ""
            if isNew {
                dbFile.TinyHash = helpers.ComputeTinyHash(path)
            }
        }
        dbFile.Save()
        //fmt.Println("done"+dbFile.Filename)
    }
    return nil
}
<commit_msg>ignore errors when indexing files<commit_after>package commands

import (
    "fmt"
    "os"
    "path/filepath"
    "log"

    "github.com/codegangsta/cli"
    "github.com/AlasdairF/File"

    "github.com/nbremond/double-kill/models"
    "github.com/nbremond/double-kill/helpers"
)


var CmdSearch = cli.Command{
    Name:  "search",
    Usage: "search path",
    Description: `Index files in database`,
    Before: runSearch,
    Action: func(ctx *cli.Context) {},
    Flags:  []cli.Flag{},
}


func runSearch(c *cli.Context) error {
    var err error

    if err = models.InitDB(); err != nil {
        return err
    }
    basePath := filepath.Clean(c.Args()[0])+"/"

    fmt.Println("Removing deleted subfiles of «"+basePath +"»…")
    dbFiles := models.GetSubfiles(basePath)
    //fmt.Println(dbFiles)
    for pos := range dbFiles {
        forFile := dbFiles[pos]
        if ! file.Exists(filepath.Join(forFile.Dir, forFile.Filename)){
            fmt.Println("Removing «"+forFile.Filename+"» from database.")
            forFile.Delete()
        }
    }
    fmt.Println("Indexing new files…")
    filepath.Walk(c.Args()[0],indexFile)
    return nil
}

func indexFile(path string, info os.FileInfo, err error) error {
    if err != nil {
        log.Println(err)
        return err
    }
    if info.IsDir() {
    }else{// path isn' t a dir. We must check if it's a regular file.
        dir, filename := filepath.Split(path)
        upToDate := true
        isNew := false
        dbFile := models.GetOrCreateFile(dir, filename)
        if dbFile.Id == 0 {
            upToDate = false
            isNew = true
        }
        if dbFile.ModTime != info.ModTime(){
            dbFile.ModTime = info.ModTime()
            upToDate = false
        }
        if dbFile.Size != info.Size(){
            dbFile.Size = info.Size()
            upToDate = false
        }
        if ! upToDate {
            dbFile.TinyHash = ""
            dbFile.Hash = ""
            if isNew {
                dbFile.TinyHash = helpers.ComputeTinyHash(path)
            }
        }
        dbFile.Save()
        //fmt.Println("done"+dbFile.Filename)
    }
    return nil
}

{'fraction_non_alphanumeric': 0.13098057792515397, 'fraction_numerical': 0.0014211274277593558, 'mean_word_length': 1.9614305750350631, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3991947', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1356, 'n_words': 303}