<commit_before>package main

import "fmt"

import "path/filepath"
import "log"

import (
	"os"
	"os/exec"
)
import "strings"

type marker struct {
	name string
	path string
	link *volume
}

type volume struct {
	dir  string
	mark *marker
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []volume
}

func mustcmd(acmd string) string {
	fmt.Println(acmd)
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -1 /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		dirlink, err := cmd(fmt.Sprintf("sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'", dir, dirlink, err)
		if err != nil {
			fmt.Printf("Invalid dir detected: '%s'\n", dir)
			mustcmd(fmt.Sprintf("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	readVolumes()
	os.Exit(0)
	out := mustcmd("docker ps -qa --no-trunc")
	containers := strings.Split(out, "\n")
	containers = containers[:len(containers)-1]
	for _, id := range containers {
		fmt.Printf("container id: %s\n", id)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ .Name }}' %s", id))
		name := strings.TrimSpace(out)[1:]
		fmt.Printf("  name: '%s'\n", name)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' %s", name))
		volumes := strings.Split(out, "##~#")
		volumes = volumes[:len(volumes)-1]
		fmt.Printf("    volumes: '%v': %d\n", volumes, len(volumes))
		for i, volume := range volumes {
			vols := strings.Split(volume, ",")
			path := vols[0]
			vfs := vols[1]
			fmt.Printf("      volume %d: '%v'=>'%v'\n", i, path, vfs)
			if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
				vfs2, _ := cmd(fmt.Sprintf("sudo readlink %s 2> /dev/null", vfs))
				vfs2 = strings.TrimSpace(vfs2)
				fmt.Printf("      vfs2 '%s' from sudo readlink %s\n", vfs2, vfs)
				link := "." + name + "###" + strings.Replace(path, "/", ",#,", -1)<commit_msg>gcl.go: uses os.Stat() for reading vfs folders<commit_after>package main

import "fmt"

import "path/filepath"
import "log"

import (
	"os"
	"os/exec"
)
import "strings"

type marker struct {
	name string
	path string
	link *volume
}

type volume struct {
	dir  string
	mark *marker
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []volume
}

func mustcmd(acmd string) string {
	fmt.Println(acmd)
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -1 /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/", dir)
		dirlink, err := cmd(fdir)
		fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'", dir, dirlink, err)
		if err != nil {
			finfo, err := os.Stat(fdir)
			if err != nil {
				fmt.Printf("Invalid dir detected: '%s' (%s)\n", dir, err)
			}
			if finfo.IsDir() {
				fmt.Printf("dir detected: '%s'\n", dir)
			} else {
				fmt.Printf("Invalid dir (file) detected: '%s'\n", dir)
			}
			// mustcmd(fmt.Sprintf("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	readVolumes()
	os.Exit(0)
	out := mustcmd("docker ps -qa --no-trunc")
	containers := strings.Split(out, "\n")
	containers = containers[:len(containers)-1]
	for _, id := range containers {
		fmt.Printf("container id: %s\n", id)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ .Name }}' %s", id))
		name := strings.TrimSpace(out)[1:]
		fmt.Printf("  name: '%s'\n", name)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' %s", name))
		volumes := strings.Split(out, "##~#")
		volumes = volumes[:len(volumes)-1]
		fmt.Printf("    volumes: '%v': %d\n", volumes, len(volumes))
		for i, volume := range volumes {
			vols := strings.Split(volume, ",")
			path := vols[0]
			vfs := vols[1]
			fmt.Printf("      volume %d: '%v'=>'%v'\n", i, path, vfs)
			if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
				vfs2, _ := cmd(fmt.Sprintf("sudo readlink %s 2> /dev/null", vfs))
				vfs2 = strings.TrimSpace(vfs2)
				fmt.Printf("      vfs2 '%s' from sudo readlink %s\n", vfs2, vfs)
				link := "." + name + "###" + strings.Replace(path, "/", ",#,", -1)
{'fraction_non_alphanumeric': 0.2117019173749753, 'fraction_numerical': 0.006720695789681755, 'mean_word_length': 3.458149779735683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2411994', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2022, 'n_words': 480}