<commit_before>package main

import (
  "bufio"
  "fmt"
  "io"
  "os"
  "log"
  "time"
  "flag"
  "strings"
  "os/exec"
)

func ReadChanges(file *os.File) chan string {

    changes := make(chan string)

    file.Seek(0, os.SEEK_END)
    bf := bufio.NewReader(file)

    go func() {
        for {
            line, _, err := bf.ReadLine()
            if len(line) != 0 {
                changes <- string(line)
            } else if err == io.EOF {
                time.Sleep(1 * time.Millisecond)
            }
        }
    }()
    return changes
}

func NotifyDucc(file_name string, repository_name string) {

    file, err := os.OpenFile(file_name, os.O_RDONLY, 0755)
    if err != nil {
        log.Fatalf("OpenFile: %s", err)
        }

    changes := ReadChanges(file)

    for {
        msg := <-changes

        if msg == "xx|file rotation|xx" {
                NotifyDucc(file_name, repository_name)
        }


        msg_split := strings.Split(msg, "|")
        image := msg_split[len(msg_split)-1]
        image = strings.ReplaceAll(image, "https://", "")

        fmt.Printf("DUCC ingestion for %s started...\n", image)

        _, err := exec.Command("cvmfs_ducc", "convert-single-image", "-p", image, repository_name, "--skip-thin-image", "--skip-podman").Output()
        if err != nil {
                log.Fatal(err)
        }

        fmt.Printf("[done]\n")
    }
}

func main() {

    file_name := flag.String("notifications_file", "notifications.txt", "Notification file")
    repository_name := flag.String("repository_name", "test-unpacked.cern.ch", "Repository")
    flag.Parse()

    name := *file_name
    repo := *repository_name

    NotifyDucc(name, repo)
}
<commit_msg>Wait longer for new lines and close old file at rotation<commit_after>package main

import (
  "bufio"
  "fmt"
  "io"
  "os"
  "log"
  "time"
  "flag"
  "strings"
  "os/exec"
)

func ReadChanges(file *os.File) chan string {

    changes := make(chan string)

    file.Seek(0, os.SEEK_END)
    bf := bufio.NewReader(file)

    go func() {
        for {
            line, _, err := bf.ReadLine()
            if len(line) != 0 {
                changes <- string(line)
            } else if err == io.EOF {
                time.Sleep(1 * time.Second)
            }
        }
    }()
    return changes
}

func NotifyDucc(file_name string, repository_name string) {

    file, err := os.OpenFile(file_name, os.O_RDONLY, 0755)
    if err != nil {
        log.Fatalf("OpenFile: %s", err)
        }

    changes := ReadChanges(file)

    for {
        msg := <-changes

        if msg == "xx|file rotation|xx" {
		file.Close()
                NotifyDucc(file_name, repository_name)
        }

        msg_split := strings.Split(msg, "|")
        image := msg_split[len(msg_split)-1]
        image = strings.ReplaceAll(image, "https://", "")

        fmt.Printf("DUCC ingestion for %s started...\n", image)

        _, err := exec.Command("cvmfs_ducc", "convert-single-image", "-p", image, repository_name, "--skip-thin-image", "--skip-podman").Output()
        if err != nil {
                log.Fatal(err)
        }

        fmt.Printf("[done]\n")
    }
}

func main() {

    file_name := flag.String("notifications_file", "notifications.txt", "Notification file")
    repository_name := flag.String("repository_name", "test-unpacked.cern.ch", "Repository")
    flag.Parse()

    fname := *file_name
    rname := *repository_name

    NotifyDucc(fname, rname)
}

{'fraction_non_alphanumeric': 0.15240023134759978, 'fraction_numerical': 0.004626951995373048, 'mean_word_length': 2.12748643761302, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4019772', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1133, 'n_words': 239}