<commit_before>/** opa-dynamic
  * Monitor a directory conataining a project, recompiles it and relaunch it when needed
  */

import stdlib.io.file
import stdlib.core.concurrency
import stdlib.system

do Scheduler.sleep(1, -> main(CommandLine.filter(CommandLineConf)) ) // Workaround

@expand
debug = #<Ifstatic:DEBUG> s -> println("[DEBUG] {s}") #<Else> _ -> void #<End>

type config.notifier = {notify_send} / {terminal_notifier} / {none}

/** Configuration for one project directory */
type config = {
  command : list(string) // list of command , inferred if empty
  directory : string     // project directory
  avoid_dir : list(string)
  avoid_ext : list(string)
  avoid_prefix : list(string)
  launch : bool
  opa_opt : string
  notifier : config.notifier
}


// Add browser support, launch and reload
DConfig = {{
  /** Control grouping delay of sequential events */
  grouping_delay = Duration.ms(300)
  config = {
    command = [] // empty => inferred
    directory="."
    avoid_dir=[".git", "_build", "_tracks","node_modules"]
    avoid_ext=[".opx", "_depends"/*opa*/, ".log", ".bak", ".old", "~"/*emacs*/, ".exe"]
    avoid_prefix =[ ".#"/*emacs*/, "#" ]
    launch = true
    opa_opt = ""
    notifier = {notify_send}
  }
}}

CommandLineConf = {{
  title = "OpaDyn"
  init = DConfig.config
  anonymous = []
  parsers = [
    CommandLine.string(["--src-dir"],"Directory to auto-build","directory(={init.directory})"){
     s,c -> {c with directory = s}
    },
    CommandLine.string(["--avoid-dir"],"Directory to avoid","directory(={init.avoid_dir})"){
     s,c -> {c with avoid_dir=[s|c.avoid_dir]}
    },
    CommandLine.string(["--avoid-ext"],"Extension to avoid","extension(={init.avoid_ext})"){
     s,c -> {c with avoid_ext=[s|c.avoid_ext]}
    },
    CommandLine.string(["--command"],"Command to build","command(=inferred)"){
     s,c -> {c with command = [s|c.command]}
    },
    CommandLine.string(["--opa-opt"],"Option for opa command","opt"){
     s,c -> {c with opa_opt = "{c.opa_opt} {s}"}
    },
    CommandLine.switch(["--no-launch"],"Last command is not a launch command"){
     c -> {c with launch = false}
    },
    cases = [
      ("no", {none}),
      ("linux", {notify_send}),
      ("mac"  , {terminal_notifier}),
      ("windows", {none})
    ]
    CommandLine.case(["--notifier"],cases,
     "Force platform notifier", String.concat(" or ",List.map(_.f1, cases))
    ){
     n,c -> {c with notifier = n}
    },
  ]
}} : CommandLine.family(config)

avoid_file(config)(file) =
      List.mem(file,config.avoid_dir)
  ||  List.exists(String.has_suffix(_,file), config.avoid_ext)
  ||  List.exists(String.has_prefix(_,file), config.avoid_prefix)



type command_queue_state = MutexRef.t({
  need : bool
  in_progress : option(Date.date) // Some => in progress
})

type job = {
  name : string
  command : -> void
}

/** projecting a function to an exclusive access queue,
    only one function can be executed at the same time,
    new execution demand during a previous exectution will be merge into one exectution
    new execution demand that are very near in time are merged
*/
exclusive_queue(command: -> void) = (

  stateR = MutexRef.create({need=false progress=none})
  finished() = do debug("XQUEUE FINISHED") MutexRef.update(stateR){ state -> {state with progress=none} }
  start() = do debug("XQUEUE START") MutexRef.update(stateR){ state -> {state with progress=some(Date.now()) need=false} }
  consume_need()  = MutexRef.exclusive(stateR){ ->
    state = MutexRef.get(stateR)
    do debug("XQUEUE CONSUME NEED {state}")
    do MutexRef.set(stateR, {MutexRef.get(stateR) with need=false})
    state.need
  }
  rec proceed(id) =
    do debug("TRY PROCEED {id}")
    if MutexRef.get(stateR).progress!=none then debug("STILL IN PROGRESS") else
    do debug("PROCEED")
    do start()
    do command()
    do finished()
    if consume_need() then proceed(id)
    else debug("XQUEUE END LOOP, WAITING")

  -> MutexRef.update(stateR){ state ->
    // any transition to need=true without any in_progress work starts a new proceed_loop
    if state.need then do debug("STILL IN NEED") state else
    need = match state.progress
      {none} -> true
      {some=t} ->
        dt = Date.between(t,Date.now())
        do debug("DT = {Duration.in_milliseconds(dt)} ms")
        dt > DConfig.grouping_delay // if far enough after starting progress
      end
    do if need then Scheduler.push( -> proceed(Random.string(4)) )
    do debug("NEED = {need}")
    {state with ~need}
  }

)

/**
  * Listening a set of file, and triggering command on any change that is not about avoided files.
  */
job_on_files_changes(avoid_file,files,_name,command)=
  command=exclusive_queue(command)
  handler_map = MutexRef.create(StringMap.empty)
  _remove_file(file) = MutexRef.update(handler_map){
    map -> StringMap.remove(file, map)
  }
  command_if_not_avoid(file) = if not(avoid_file(file)) then command() else debug("AVOIDED {file}")
  add_file(file) =
    do debug("Listen {file} ")
    is_directory = File.is_directory(file)
    handler = File.onchange(file,none){file1, event ->
      do  debug("EVENT[{file} {file1}] = {Debug.dump(event)}")
      match event
      {change=file2} ->
        do if is_directory then debug("file-change") else debug("mod-or-delete")
        command_if_not_avoid(file2)
      {rename=file2} ->
        do if is_directory then debug("new-or-delete") // TODO : HANDLE NEW FILE LISTENING HERE
                           else debug("delete")
        command_if_not_avoid(file2)
    }
    do MutexRef.update(handler_map){ map ->
      StringMap.add(file, handler, map)
    }
    debug("OK")
    do StringSet.iter(add_file,files)
    command()

/** Collect recusively all path in a directory using an avoidance rule */
collect_files(avoid_file, rpath,dir) =
  path = List.rev([dir|rpath])
  pathstr = String.concat("/", path)
  v = File.readdir(pathstr):outcome(llarray(string), string)<commit_msg>[fix] big project: reduce the number of files actually listened<commit_after>/** opa-dynamic
  * Monitor a directory conataining a project, recompiles it and relaunch it when needed
  */

import stdlib.io.file
import stdlib.core.concurrency
import stdlib.system

do Scheduler.sleep(1, -> main(CommandLine.filter(CommandLineConf)) ) // Workaround

@expand
debug = #<Ifstatic:DEBUG> s -> println("[DEBUG] {s}") #<Else> _ -> void #<End>

type config.notifier = {notify_send} / {terminal_notifier} / {none}

/** Configuration for one project directory */
type config = {
  command : list(string) // list of command , inferred if empty
  directory : string     // project directory
  avoid_dir : list(string)
  avoid_ext : list(string)
  avoid_prefix : list(string)
  launch : bool
  opa_opt : string
  notifier : config.notifier
}


// Add browser support, launch and reload
DConfig = {{
  /** Control grouping delay of sequential events */
  grouping_delay = Duration.ms(300)
  config = {
    command = [] // empty => inferred
    directory="."
    avoid_dir=[".git", "_build", "_tracks","node_modules"]
    avoid_ext=[".opx", "_depends"/*opa*/, ".log", ".bak", ".old", "~"/*emacs*/, ".exe"]
    avoid_prefix =[ ".#"/*emacs*/, "#" ]
    launch = true
    opa_opt = ""
    notifier = {notify_send}
  }
}}

CommandLineConf = {{
  title = "OpaDyn"
  init = DConfig.config
  anonymous = []
  parsers = [
    CommandLine.string(["--src-dir"],"Directory to auto-build","directory(={init.directory})"){
     s,c -> {c with directory = s}
    },
    CommandLine.string(["--avoid-dir"],"Directory to avoid","directory(={init.avoid_dir})"){
     s,c -> {c with avoid_dir=[s|c.avoid_dir]}
    },
    CommandLine.string(["--avoid-ext"],"Extension to avoid","extension(={init.avoid_ext})"){
     s,c -> {c with avoid_ext=[s|c.avoid_ext]}
    },
    CommandLine.string(["--command"],"Command to build","command(=inferred)"){
     s,c -> {c with command = [s|c.command]}
    },
    CommandLine.string(["--opa-opt"],"Option for opa command","opt"){
     s,c -> {c with opa_opt = "{c.opa_opt} {s}"}
    },
    CommandLine.switch(["--no-launch"],"Last command is not a launch command"){
     c -> {c with launch = false}
    },
    cases = [
      ("no", {none}),
      ("linux", {notify_send}),
      ("mac"  , {terminal_notifier}),
      ("windows", {none})
    ]
    CommandLine.case(["--notifier"],cases,
     "Force platform notifier", String.concat(" or ",List.map(_.f1, cases))
    ){
     n,c -> {c with notifier = n}
    },
  ]
}} : CommandLine.family(config)

avoid_file(config)(file) =
      List.mem(file,config.avoid_dir)
  ||  List.exists(String.has_suffix(_,file), config.avoid_ext)
  ||  List.exists(String.has_prefix(_,file), config.avoid_prefix)



type command_queue_state = MutexRef.t({
  need : bool
  in_progress : option(Date.date) // Some => in progress
})

type job = {
  name : string
  command : -> void
}

/** projecting a function to an exclusive access queue,
    only one function can be executed at the same time,
    new execution demand during a previous exectution will be merge into one exectution
    new execution demand that are very near in time are merged
*/
exclusive_queue(command: -> void) = (

  stateR = MutexRef.create({need=false progress=none})
  finished() = do debug("XQUEUE FINISHED") MutexRef.update(stateR){ state -> {state with progress=none} }
  start() = do debug("XQUEUE START") MutexRef.update(stateR){ state -> {state with progress=some(Date.now()) need=false} }
  consume_need()  = MutexRef.exclusive(stateR){ ->
    state = MutexRef.get(stateR)
    do debug("XQUEUE CONSUME NEED {state}")
    do MutexRef.set(stateR, {MutexRef.get(stateR) with need=false})
    state.need
  }
  rec proceed(id) =
    do debug("TRY PROCEED {id}")
    if MutexRef.get(stateR).progress!=none then debug("STILL IN PROGRESS") else
    do debug("PROCEED")
    do start()
    do command()
    do finished()
    if consume_need() then proceed(id)
    else debug("XQUEUE END LOOP, WAITING")

  -> MutexRef.update(stateR){ state ->
    // any transition to need=true without any in_progress work starts a new proceed_loop
    if state.need then do debug("STILL IN NEED") state else
    need = match state.progress
      {none} -> true
      {some=t} ->
        dt = Date.between(t,Date.now())
        do debug("DT = {Duration.in_milliseconds(dt)} ms")
        dt > DConfig.grouping_delay // if far enough after starting progress
      end
    do if need then Scheduler.push( -> proceed(Random.string(4)) )
    do debug("NEED = {need}")
    {state with ~need}
  }

)

/**
  * Listening a set of file, and triggering command on any change that is not about avoided files.
  */
job_on_files_changes(avoid_file,files,_name,command)=
  command=exclusive_queue(command)
  handler_map = MutexRef.create(StringMap.empty)
  _remove_file(file) = MutexRef.update(handler_map){
    map -> StringMap.remove(file, map)
  }
  command_if_not_avoid(file) = if not(avoid_file(file)) then command() else debug("AVOIDED {file}")
  add_file(file) =
    do debug("Listen {file} ")
    is_directory = File.is_directory(file)
    if not(is_directory) then void else
    handler = File.onchange(file,none){file1, event ->
      do  debug("EVENT[{file} {file1}] = {Debug.dump(event)}")
      match event
      {change=file2} ->
        do if is_directory then debug("file-change") else debug("mod-or-delete")
        command_if_not_avoid(file2)
      {rename=file2} ->
        do if is_directory then debug("new-or-delete") // TODO : HANDLE NEW FILE LISTENING HERE
                           else debug("delete")
        command_if_not_avoid(file2)
    }
    do MutexRef.update(handler_map){ map ->
      StringMap.add(file, handler, map)
    }
    debug("OK")

  do StringSet.iter(add_file,files)
  command()


/** Collect recusively all path in a directory using an avoidance rule */
collect_files(avoid_file, rpath,dir) =
  path = List.rev([dir|rpath])
  pathstr = String.concat("/", path)
  v = File.readdir(pathstr):outcome(llarray(string), string)
{'fraction_non_alphanumeric': 0.1526596631649158, 'fraction_numerical': 0.002001000500250125, 'mean_word_length': 3.4115483633688855, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4390960', 'n_tokens_mistral': 4035, 'n_tokens_neox': 3910, 'n_words': 977}