<commit_before>with GNAT.IO;  use GNAT.IO;
with Babel;
with Babel.Files;
with Ada.Directories;
with Ada.Strings.Unbounded;
with Util.Encoders;
with Util.Encoders.Base16;
with Babel.Filters;
with Babel.Files.Buffers;
with Babel.Files.Queues;
with Babel.Stores.Local;
with Babel.Strategies.Default;
with Babel.Strategies.Workers;
procedure babel_main is

   use Ada.Strings.Unbounded;

   Dir         : Babel.Files.Directory_Type;
   Hex_Encoder : Util.Encoders.Base16.Encoder;
   Exclude     : aliased Babel.Filters.Exclude_Directory_Filter_Type;
   Local       : aliased Babel.Stores.Local.Local_Store_Type;
   Backup      : aliased Babel.Strategies.Default.Default_Strategy_Type;
   Buffers     : aliased Babel.Files.Buffers.Buffer_Pools.Pool;
   Store       : aliased Babel.Stores.Local.Local_Store_Type;
--
--     procedure Print_Sha (Path : in String;
--                          File : in out Babel.Files.File) is
--        Sha : constant String := Hex_Encoder.Transform (File.SHA1);
--     begin
--        Put_Line (Path & "/" & To_String (File.Name) & " => " & Sha);
--     end Print_Sha;

   procedure Do_Backup (Count : in Positive) is
      Workers   : Babel.Strategies.Workers.Worker_Type (Count);
      Container : Babel.Files.Default_Container;
      Queue     : Babel.Files.Queues.Directory_Queue;
   begin
      Babel.Files.Queues.Add_Directory (Queue, Dir);
      Babel.Strategies.Workers.Start (Workers, Backup'Unchecked_Access);
      Backup.Scan (Queue, Container);
   end Do_Backup;

begin
   Dir := Babel.Files.Allocate (Name => ".",
                                Dir  => Babel.Files.NO_DIRECTORY);
   Exclude.Add_Exclude (".svn");
   Exclude.Add_Exclude ("obj");
   Babel.Files.Buffers.Create_Pool (Into => Buffers, Count => 10, Size => 1_000_000);
   Store.Set_Root_Directory ("/tmp/babel-store");
   Local.Set_Root_Directory (".");
   Backup.Set_Filters (Exclude'Unchecked_Access);
   Backup.Set_Stores (Read => Local'Unchecked_Access, Write => Store'Unchecked_Access);
   Backup.Set_Buffers (Buffers'Unchecked_Access);

   Put_Line ("Size: " & Natural'Image (Ada.Directories.File_Size'Size));
   Do_Backup (2);
--     Bkp.Files.Scan (".", Dir);
--     Bkp.Files.Iterate_Files (".", Dir, 10, Print_Sha'Access);
--     Put_Line ("Total size: " & Ada.Directories.File_Size'Image (Dir.Tot_Size));
--     Put_Line ("File count: " & Natural'Image (Dir.Tot_Files));
--     Put_Line ("Dir count:  " & Natural'Image (Dir.Tot_Dirs));
end babel_main;<commit_msg>Update the main for a simple backup<commit_after>with GNAT.IO;  use GNAT.IO;
with Babel;
with Babel.Files;
with Ada.Directories;
with Ada.Strings.Unbounded;
with Util.Encoders;
with Util.Encoders.Base16;
with Util.Log.Loggers;
with Babel.Filters;
with Babel.Files.Buffers;
with Babel.Files.Queues;
with Babel.Stores.Local;
with Babel.Strategies.Default;
with Babel.Strategies.Workers;
with Babel.Base.Text;
with Babel.Base.Users;
with Babel.Streams;
with Babel.Streams.XZ;
with Babel.Streams.Cached;
with Babel.Streams.Files;
with Tar;
procedure babel_main is

   use Ada.Strings.Unbounded;

   Dir         : Babel.Files.Directory_Type;
   Hex_Encoder : Util.Encoders.Base16.Encoder;
   Exclude     : aliased Babel.Filters.Exclude_Directory_Filter_Type;
   Local       : aliased Babel.Stores.Local.Local_Store_Type;
   Backup      : aliased Babel.Strategies.Default.Default_Strategy_Type;
   Buffers     : aliased Babel.Files.Buffers.Buffer_Pool;
   Store       : aliased Babel.Stores.Local.Local_Store_Type;
   Database    : aliased Babel.Base.Text.Text_Database;

   --
--     procedure Print_Sha (Path : in String;
--                          File : in out Babel.Files.File) is
--        Sha : constant String := Hex_Encoder.Transform (File.SHA1);
--     begin
--        Put_Line (Path & "/" & To_String (File.Name) & " => " & Sha);
--     end Print_Sha;

   procedure Do_Backup (Count : in Positive) is
      Workers   : Babel.Strategies.Workers.Worker_Type (Count);
      Container : Babel.Files.Default_Container;
      Queue     : Babel.Files.Queues.Directory_Queue;
   begin
      Babel.Files.Queues.Add_Directory (Queue, Dir);
      Babel.Strategies.Workers.Start (Workers, Backup'Unchecked_Access);
      Backup.Scan (Queue, Container);
   end Do_Backup;

begin
   Util.Log.Loggers.Initialize ("babel.properties");

   Dir := Babel.Files.Allocate (Name => ".",
                                Dir  => Babel.Files.NO_DIRECTORY);
   Exclude.Add_Exclude (".svn");
   Exclude.Add_Exclude ("obj");
   Babel.Files.Buffers.Create_Pool (Into => Buffers, Count => 10, Size => 1_000_000);
   Store.Set_Root_Directory ("/tmp/babel-store");
   Local.Set_Root_Directory (".");
   Backup.Set_Filters (Exclude'Unchecked_Access);
   Backup.Set_Stores (Read => Local'Unchecked_Access, Write => Store'Unchecked_Access);
   Backup.Set_Buffers (Buffers'Unchecked_Access);
   Backup.Set_Database (Database'Unchecked_Access);

   Put_Line ("Size: " & Natural'Image (Ada.Directories.File_Size'Size));
   Do_Backup (2);
   Database.Save ("database.txt");
--     Bkp.Files.Scan (".", Dir);
--     Bkp.Files.Iterate_Files (".", Dir, 10, Print_Sha'Access);
--     Put_Line ("Total size: " & Ada.Directories.File_Size'Image (Dir.Tot_Size));
--     Put_Line ("File count: " & Natural'Image (Dir.Tot_Files));
--     Put_Line ("Dir count:  " & Natural'Image (Dir.Tot_Dirs));
end babel_main;
{'fraction_non_alphanumeric': 0.13016644847578082, 'fraction_numerical': 0.0063587058163456145, 'mean_word_length': 3.3024939662107804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2085225', 'n_tokens_mistral': 1906, 'n_tokens_neox': 1832, 'n_words': 346}