Hugging Face's logo

csujeong
/
Gemma-7B-Finetuning-JCS-Ko

PEFT
TensorBoard
Safetensors
trl
sft
Generated from Trainer
Model card Files Metrics Community
csujeong commited on
Commit
781ce8d
•
1 Parent(s): b5091ae

End of training

Browse files
Files changed (18) hide show
  1. .gitattributes +1 -0
  2. README.md +55 -0
  3. adapter_config.json +30 -0
  4. adapter_model.safetensors +3 -0
  5. runs/Feb26_11-29-58_336d6f2eb0af/events.out.tfevents.1708948250.336d6f2eb0af.191.0 +3 -0
  6. special_tokens_map.json +24 -0
  7. tokenizer.json +3 -0
  8. tokenizer.model +3 -0
  9. tokenizer_config.json +49 -0
  10. training_args.bin +3 -0
  11. wandb/debug-internal.log +566 -0
  12. wandb/debug.log +42 -0
  13. wandb/run-20240226_115124-aiyp9xfp/files/config.yaml +664 -0
  14. wandb/run-20240226_115124-aiyp9xfp/files/requirements.txt +481 -0
  15. wandb/run-20240226_115124-aiyp9xfp/files/wandb-metadata.json +52 -0
  16. wandb/run-20240226_115124-aiyp9xfp/logs/debug-internal.log +566 -0
  17. wandb/run-20240226_115124-aiyp9xfp/logs/debug.log +42 -0
  18. wandb/run-20240226_115124-aiyp9xfp/run-aiyp9xfp.wandb +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: other
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - sft
7
+ - generated_from_trainer
8
+ base_model: google/gemma-7b
9
+ model-index:
10
+ - name: Gemma-7B-Finetuning-JCS-Ko
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # Gemma-7B-Finetuning-JCS-Ko
18
+
19
+ This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 0.0002
39
+ - train_batch_size: 2
40
+ - eval_batch_size: 8
41
+ - seed: 42
42
+ - gradient_accumulation_steps: 2
43
+ - total_train_batch_size: 4
44
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
45
+ - lr_scheduler_type: cosine
46
+ - lr_scheduler_warmup_ratio: 0.03
47
+ - training_steps: 60
48
+
49
+ ### Framework versions
50
+
51
+ - PEFT 0.8.2
52
+ - Transformers 4.38.1
53
+ - Pytorch 2.1.0+cu121
54
+ - Datasets 2.17.1
55
+ - Tokenizers 0.15.2
adapter_config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "google/gemma-7b",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 32,
13
+ "lora_dropout": 0.05,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 32,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "gate_proj",
23
+ "k_proj",
24
+ "v_proj",
25
+ "o_proj",
26
+ "q_proj"
27
+ ],
28
+ "task_type": "CAUSAL_LM",
29
+ "use_rslora": false
30
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e51909a9018a8c95fd75b7305e0fa909104fba65b92eb1987e4e607d2475ae28
3
+ size 201888736
runs/Feb26_11-29-58_336d6f2eb0af/events.out.tfevents.1708948250.336d6f2eb0af.191.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b4780eecbce2f00a61b2a9e44dfce147b6a92551e2dc002ad17d2faa6a44dba
3
+ size 15286
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<bos>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<eos>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<eos>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb7e592b7314175501c7fc56b904d581de569169ac90e5aa2ec11a860c2cbbaa
3
+ size 17477652
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
3
+ size 4241003
tokenizer_config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<pad>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<eos>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "<bos>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "3": {
30
+ "content": "<unk>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ }
37
+ },
38
+ "bos_token": "<bos>",
39
+ "clean_up_tokenization_spaces": false,
40
+ "eos_token": "<eos>",
41
+ "legacy": null,
42
+ "model_max_length": 1000000000000000019884624838656,
43
+ "pad_token": "<eos>",
44
+ "sp_model_kwargs": {},
45
+ "spaces_between_special_tokens": false,
46
+ "tokenizer_class": "GemmaTokenizer",
47
+ "unk_token": "<unk>",
48
+ "use_default_system_prompt": false
49
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e385b8a3303aafaed246b9061da0da8749925927191f283921eb2923f2757cbf
3
+ size 4984
wandb/debug-internal.log ADDED
@@ -0,0 +1,566 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-02-26 11:51:24,269 INFO StreamThr :8773 [internal.py:wandb_internal():86] W&B internal server running at pid: 8773, started at: 2024-02-26 11:51:24.265817
2
+ 2024-02-26 11:51:24,273 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status
3
+ 2024-02-26 11:51:26,744 INFO WriterThread:8773 [datastore.py:open_for_write():87] open: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/run-aiyp9xfp.wandb
4
+ 2024-02-26 11:51:26,751 DEBUG SenderThread:8773 [sender.py:send():382] send: header
5
+ 2024-02-26 11:51:26,756 DEBUG SenderThread:8773 [sender.py:send():382] send: run
6
+ 2024-02-26 11:51:27,013 INFO SenderThread:8773 [dir_watcher.py:__init__():211] watching files in: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files
7
+ 2024-02-26 11:51:27,014 INFO SenderThread:8773 [sender.py:_start_run_threads():1136] run started: aiyp9xfp with start time 1708948284.268213
8
+ 2024-02-26 11:51:27,024 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: check_version
9
+ 2024-02-26 11:51:27,025 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: check_version
10
+ 2024-02-26 11:51:27,158 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: run_start
11
+ 2024-02-26 11:51:27,168 DEBUG HandlerThread:8773 [system_info.py:__init__():27] System info init
12
+ 2024-02-26 11:51:27,168 DEBUG HandlerThread:8773 [system_info.py:__init__():42] System info init done
13
+ 2024-02-26 11:51:27,168 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
14
+ 2024-02-26 11:51:27,168 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
15
+ 2024-02-26 11:51:27,169 INFO HandlerThread:8773 [system_monitor.py:probe():214] Collecting system info
16
+ 2024-02-26 11:51:27,169 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
17
+ 2024-02-26 11:51:27,170 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
18
+ 2024-02-26 11:51:27,172 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
19
+ 2024-02-26 11:51:27,172 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
20
+ 2024-02-26 11:51:27,173 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
21
+ 2024-02-26 11:51:27,182 DEBUG HandlerThread:8773 [system_info.py:probe():151] Probing system
22
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [gitlib.py:_init_repo():56] git repository is invalid
23
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [system_info.py:probe():199] Probing system done
24
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-02-26T11:51:27.182772', 'startedAt': '2024-02-26T11:51:24.188771', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Gemma-7B-Finetuning-databricks-dolly-15k-ko.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1wMe78fcAk4lrbXJnoOMsmSpeUqlmneh3', 'host': '336d6f2eb0af', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 1, 'cpu_count_logical': 2, 'cpu_freq': {'current': 2299.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 78.1898422241211, 'used': 42.809574127197266}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 12.674789428710938}}
25
+ 2024-02-26 11:51:27,188 INFO HandlerThread:8773 [system_monitor.py:probe():224] Finished collecting system info
26
+ 2024-02-26 11:51:27,189 INFO HandlerThread:8773 [system_monitor.py:probe():227] Publishing system info
27
+ 2024-02-26 11:51:27,207 INFO HandlerThread:8773 [system_monitor.py:probe():229] Finished publishing system info
28
+ 2024-02-26 11:51:27,216 DEBUG SenderThread:8773 [sender.py:send():382] send: files
29
+ 2024-02-26 11:51:27,216 INFO SenderThread:8773 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
30
+ 2024-02-26 11:51:27,244 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: python_packages
31
+ 2024-02-26 11:51:27,248 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: python_packages
32
+ 2024-02-26 11:51:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
33
+ 2024-02-26 11:51:27,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
34
+ 2024-02-26 11:51:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
35
+ 2024-02-26 11:51:27,471 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
36
+ 2024-02-26 11:51:27,471 DEBUG SenderThread:8773 [sender.py:send():382] send: config
37
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: metric
38
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
39
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: metric
40
+ 2024-02-26 11:51:27,473 WARNING SenderThread:8773 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen)
41
+ 2024-02-26 11:51:27,474 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
42
+ 2024-02-26 11:51:27,529 INFO wandb-upload_0:8773 [upload_job.py:push():131] Uploaded file /tmp/tmp7ayn3ah6wandb/rx7k0how-wandb-metadata.json
43
+ 2024-02-26 11:51:28,019 INFO Thread-12 :8773 [dir_watcher.py:_on_file_created():271] file/dir created: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/wandb-metadata.json
44
+ 2024-02-26 11:51:28,020 INFO Thread-12 :8773 [dir_watcher.py:_on_file_created():271] file/dir created: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/requirements.txt
45
+ 2024-02-26 11:51:29,474 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
46
+ 2024-02-26 11:51:34,475 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
47
+ 2024-02-26 11:51:38,566 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
48
+ 2024-02-26 11:51:38,566 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
49
+ 2024-02-26 11:51:38,566 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
50
+ 2024-02-26 11:51:38,568 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
51
+ 2024-02-26 11:51:38,568 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
52
+ 2024-02-26 11:51:38,569 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
53
+ 2024-02-26 11:51:38,569 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
54
+ 2024-02-26 11:51:38,569 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
55
+ 2024-02-26 11:51:38,584 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
56
+ 2024-02-26 11:51:38,585 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
57
+ 2024-02-26 11:51:38,585 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
58
+ 2024-02-26 11:51:38,586 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
59
+ 2024-02-26 11:51:39,586 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
60
+ 2024-02-26 11:51:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
61
+ 2024-02-26 11:51:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
62
+ 2024-02-26 11:51:42,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
63
+ 2024-02-26 11:51:45,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
64
+ 2024-02-26 11:51:50,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
65
+ 2024-02-26 11:51:55,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
66
+ 2024-02-26 11:51:56,068 INFO Thread-12 :8773 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/config.yaml
67
+ 2024-02-26 11:51:57,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
68
+ 2024-02-26 11:51:57,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
69
+ 2024-02-26 11:51:57,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
70
+ 2024-02-26 11:52:01,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
71
+ 2024-02-26 11:52:06,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
72
+ 2024-02-26 11:52:11,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
73
+ 2024-02-26 11:52:12,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
74
+ 2024-02-26 11:52:12,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
75
+ 2024-02-26 11:52:12,288 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
76
+ 2024-02-26 11:52:17,321 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
77
+ 2024-02-26 11:52:19,911 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
78
+ 2024-02-26 11:52:19,916 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
79
+ 2024-02-26 11:52:19,916 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
80
+ 2024-02-26 11:52:19,925 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
81
+ 2024-02-26 11:52:19,926 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
82
+ 2024-02-26 11:52:19,931 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
83
+ 2024-02-26 11:52:19,931 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
84
+ 2024-02-26 11:52:19,943 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
85
+ 2024-02-26 11:52:19,944 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
86
+ 2024-02-26 11:52:22,323 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
87
+ 2024-02-26 11:52:27,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
88
+ 2024-02-26 11:52:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
89
+ 2024-02-26 11:52:27,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
90
+ 2024-02-26 11:52:27,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
91
+ 2024-02-26 11:52:29,235 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
92
+ 2024-02-26 11:52:29,236 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
93
+ 2024-02-26 11:52:29,236 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
94
+ 2024-02-26 11:52:29,236 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
95
+ 2024-02-26 11:52:29,238 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
96
+ 2024-02-26 11:52:29,239 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
97
+ 2024-02-26 11:52:29,240 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
98
+ 2024-02-26 11:52:29,240 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
99
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
100
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
101
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
102
+ 2024-02-26 11:52:29,250 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
103
+ 2024-02-26 11:52:33,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
104
+ 2024-02-26 11:52:38,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
105
+ 2024-02-26 11:52:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
106
+ 2024-02-26 11:52:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
107
+ 2024-02-26 11:52:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
108
+ 2024-02-26 11:52:43,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
109
+ 2024-02-26 11:52:48,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
110
+ 2024-02-26 11:52:53,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
111
+ 2024-02-26 11:52:57,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
112
+ 2024-02-26 11:52:57,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
113
+ 2024-02-26 11:52:57,271 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
114
+ 2024-02-26 11:52:59,272 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
115
+ 2024-02-26 11:53:04,273 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
116
+ 2024-02-26 11:53:09,274 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
117
+ 2024-02-26 11:53:12,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
118
+ 2024-02-26 11:53:12,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
119
+ 2024-02-26 11:53:12,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
120
+ 2024-02-26 11:53:14,368 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
121
+ 2024-02-26 11:53:19,368 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
122
+ 2024-02-26 11:53:24,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
123
+ 2024-02-26 11:53:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
124
+ 2024-02-26 11:53:27,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
125
+ 2024-02-26 11:53:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
126
+ 2024-02-26 11:53:30,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
127
+ 2024-02-26 11:53:30,386 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
128
+ 2024-02-26 11:53:30,387 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
129
+ 2024-02-26 11:53:30,390 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
130
+ 2024-02-26 11:53:30,394 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
131
+ 2024-02-26 11:53:30,395 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
132
+ 2024-02-26 11:53:30,395 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
133
+ 2024-02-26 11:53:30,399 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
134
+ 2024-02-26 11:53:30,406 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
135
+ 2024-02-26 11:53:30,409 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
136
+ 2024-02-26 11:53:35,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
137
+ 2024-02-26 11:53:40,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
138
+ 2024-02-26 11:53:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
139
+ 2024-02-26 11:53:42,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
140
+ 2024-02-26 11:53:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
141
+ 2024-02-26 11:53:43,588 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
142
+ 2024-02-26 11:53:43,588 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
143
+ 2024-02-26 11:53:43,588 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
144
+ 2024-02-26 11:53:43,592 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
145
+ 2024-02-26 11:53:43,596 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
146
+ 2024-02-26 11:53:43,597 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
147
+ 2024-02-26 11:53:43,598 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
148
+ 2024-02-26 11:53:43,598 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
149
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
150
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
151
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
152
+ 2024-02-26 11:53:43,604 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
153
+ 2024-02-26 11:53:45,605 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
154
+ 2024-02-26 11:53:50,606 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
155
+ 2024-02-26 11:53:55,607 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
156
+ 2024-02-26 11:53:57,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
157
+ 2024-02-26 11:53:57,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
158
+ 2024-02-26 11:53:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
159
+ 2024-02-26 11:54:01,347 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
160
+ 2024-02-26 11:54:06,348 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
161
+ 2024-02-26 11:54:11,349 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
162
+ 2024-02-26 11:54:12,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
163
+ 2024-02-26 11:54:12,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
164
+ 2024-02-26 11:54:12,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
165
+ 2024-02-26 11:54:16,559 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
166
+ 2024-02-26 11:54:21,560 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
167
+ 2024-02-26 11:54:26,561 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
168
+ 2024-02-26 11:54:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
169
+ 2024-02-26 11:54:27,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
170
+ 2024-02-26 11:54:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
171
+ 2024-02-26 11:54:32,307 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
172
+ 2024-02-26 11:54:37,307 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
173
+ 2024-02-26 11:54:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
174
+ 2024-02-26 11:54:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
175
+ 2024-02-26 11:54:42,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
176
+ 2024-02-26 11:54:46,023 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
177
+ 2024-02-26 11:54:51,024 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
178
+ 2024-02-26 11:54:56,025 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
179
+ 2024-02-26 11:54:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
180
+ 2024-02-26 11:54:57,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
181
+ 2024-02-26 11:54:57,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
182
+ 2024-02-26 11:55:01,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
183
+ 2024-02-26 11:55:06,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
184
+ 2024-02-26 11:55:11,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
185
+ 2024-02-26 11:55:12,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
186
+ 2024-02-26 11:55:12,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
187
+ 2024-02-26 11:55:12,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
188
+ 2024-02-26 11:55:17,339 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
189
+ 2024-02-26 11:55:22,340 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
190
+ 2024-02-26 11:55:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
191
+ 2024-02-26 11:55:27,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
192
+ 2024-02-26 11:55:27,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
193
+ 2024-02-26 11:55:27,389 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
194
+ 2024-02-26 11:55:32,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
195
+ 2024-02-26 11:55:37,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
196
+ 2024-02-26 11:55:42,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
197
+ 2024-02-26 11:55:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
198
+ 2024-02-26 11:55:42,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
199
+ 2024-02-26 11:55:43,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
200
+ 2024-02-26 11:55:48,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
201
+ 2024-02-26 11:55:53,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
202
+ 2024-02-26 11:55:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
203
+ 2024-02-26 11:55:57,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
204
+ 2024-02-26 11:55:57,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
205
+ 2024-02-26 11:55:58,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
206
+ 2024-02-26 11:56:03,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
207
+ 2024-02-26 11:56:08,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
208
+ 2024-02-26 11:56:12,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
209
+ 2024-02-26 11:56:12,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
210
+ 2024-02-26 11:56:12,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
211
+ 2024-02-26 11:56:14,365 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
212
+ 2024-02-26 11:56:19,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
213
+ 2024-02-26 11:56:24,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
214
+ 2024-02-26 11:56:27,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
215
+ 2024-02-26 11:56:27,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
216
+ 2024-02-26 11:56:27,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
217
+ 2024-02-26 11:56:30,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
218
+ 2024-02-26 11:56:35,309 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
219
+ 2024-02-26 11:56:40,310 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
220
+ 2024-02-26 11:56:42,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
221
+ 2024-02-26 11:56:42,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
222
+ 2024-02-26 11:56:42,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
223
+ 2024-02-26 11:56:46,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
224
+ 2024-02-26 11:56:51,308 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
225
+ 2024-02-26 11:56:56,315 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
226
+ 2024-02-26 11:56:57,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
227
+ 2024-02-26 11:56:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
228
+ 2024-02-26 11:56:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
229
+ 2024-02-26 11:57:01,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
230
+ 2024-02-26 11:57:06,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
231
+ 2024-02-26 11:57:11,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
232
+ 2024-02-26 11:57:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
233
+ 2024-02-26 11:57:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
234
+ 2024-02-26 11:57:12,252 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
235
+ 2024-02-26 11:57:17,324 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
236
+ 2024-02-26 11:57:22,325 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
237
+ 2024-02-26 11:57:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
238
+ 2024-02-26 11:57:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
239
+ 2024-02-26 11:57:27,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
240
+ 2024-02-26 11:57:28,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
241
+ 2024-02-26 11:57:33,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
242
+ 2024-02-26 11:57:38,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
243
+ 2024-02-26 11:57:42,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
244
+ 2024-02-26 11:57:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
245
+ 2024-02-26 11:57:42,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
246
+ 2024-02-26 11:57:44,282 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
247
+ 2024-02-26 11:57:49,283 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
248
+ 2024-02-26 11:57:54,284 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
249
+ 2024-02-26 11:57:57,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
250
+ 2024-02-26 11:57:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
251
+ 2024-02-26 11:57:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
252
+ 2024-02-26 11:57:59,314 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
253
+ 2024-02-26 11:58:04,315 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
254
+ 2024-02-26 11:58:09,316 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
255
+ 2024-02-26 11:58:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
256
+ 2024-02-26 11:58:12,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
257
+ 2024-02-26 11:58:12,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
258
+ 2024-02-26 11:58:16,193 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
259
+ 2024-02-26 11:58:21,194 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
260
+ 2024-02-26 11:58:26,196 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
261
+ 2024-02-26 11:58:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
262
+ 2024-02-26 11:58:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
263
+ 2024-02-26 11:58:27,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
264
+ 2024-02-26 11:58:31,358 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
265
+ 2024-02-26 11:58:36,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
266
+ 2024-02-26 11:58:41,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
267
+ 2024-02-26 11:58:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
268
+ 2024-02-26 11:58:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
269
+ 2024-02-26 11:58:42,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
270
+ 2024-02-26 11:58:47,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
271
+ 2024-02-26 11:58:52,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
272
+ 2024-02-26 11:58:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
273
+ 2024-02-26 11:58:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
274
+ 2024-02-26 11:58:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
275
+ 2024-02-26 11:58:57,349 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
276
+ 2024-02-26 11:59:02,350 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
277
+ 2024-02-26 11:59:07,351 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
278
+ 2024-02-26 11:59:12,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
279
+ 2024-02-26 11:59:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
280
+ 2024-02-26 11:59:12,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
281
+ 2024-02-26 11:59:13,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
282
+ 2024-02-26 11:59:18,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
283
+ 2024-02-26 11:59:23,337 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
284
+ 2024-02-26 11:59:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
285
+ 2024-02-26 11:59:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
286
+ 2024-02-26 11:59:27,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
287
+ 2024-02-26 11:59:28,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
288
+ 2024-02-26 11:59:33,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
289
+ 2024-02-26 11:59:38,377 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
290
+ 2024-02-26 11:59:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
291
+ 2024-02-26 11:59:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
292
+ 2024-02-26 11:59:42,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
293
+ 2024-02-26 11:59:44,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
294
+ 2024-02-26 11:59:49,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
295
+ 2024-02-26 11:59:54,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
296
+ 2024-02-26 11:59:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
297
+ 2024-02-26 11:59:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
298
+ 2024-02-26 11:59:57,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
299
+ 2024-02-26 11:59:59,331 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
300
+ 2024-02-26 12:00:04,332 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
301
+ 2024-02-26 12:00:09,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
302
+ 2024-02-26 12:00:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
303
+ 2024-02-26 12:00:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
304
+ 2024-02-26 12:00:12,295 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
305
+ 2024-02-26 12:00:15,324 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
306
+ 2024-02-26 12:00:20,325 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
307
+ 2024-02-26 12:00:25,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
308
+ 2024-02-26 12:00:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
309
+ 2024-02-26 12:00:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
310
+ 2024-02-26 12:00:27,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
311
+ 2024-02-26 12:00:30,345 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
312
+ 2024-02-26 12:00:35,346 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
313
+ 2024-02-26 12:00:40,347 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
314
+ 2024-02-26 12:00:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
315
+ 2024-02-26 12:00:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
316
+ 2024-02-26 12:00:42,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
317
+ 2024-02-26 12:00:45,389 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
318
+ 2024-02-26 12:00:50,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
319
+ 2024-02-26 12:00:55,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
320
+ 2024-02-26 12:00:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
321
+ 2024-02-26 12:00:57,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
322
+ 2024-02-26 12:00:57,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
323
+ 2024-02-26 12:01:01,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
324
+ 2024-02-26 12:01:06,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
325
+ 2024-02-26 12:01:11,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
326
+ 2024-02-26 12:01:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
327
+ 2024-02-26 12:01:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
328
+ 2024-02-26 12:01:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
329
+ 2024-02-26 12:01:17,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
330
+ 2024-02-26 12:01:22,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
331
+ 2024-02-26 12:01:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
332
+ 2024-02-26 12:01:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
333
+ 2024-02-26 12:01:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
334
+ 2024-02-26 12:01:27,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
335
+ 2024-02-26 12:01:32,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
336
+ 2024-02-26 12:01:37,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
337
+ 2024-02-26 12:01:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
338
+ 2024-02-26 12:01:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
339
+ 2024-02-26 12:01:42,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
340
+ 2024-02-26 12:01:43,316 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
341
+ 2024-02-26 12:01:48,317 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
342
+ 2024-02-26 12:01:53,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
343
+ 2024-02-26 12:01:57,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
344
+ 2024-02-26 12:01:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
345
+ 2024-02-26 12:01:57,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
346
+ 2024-02-26 12:01:58,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
347
+ 2024-02-26 12:02:03,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
348
+ 2024-02-26 12:02:08,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
349
+ 2024-02-26 12:02:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
350
+ 2024-02-26 12:02:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
351
+ 2024-02-26 12:02:12,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
352
+ 2024-02-26 12:02:14,353 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
353
+ 2024-02-26 12:02:19,354 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
354
+ 2024-02-26 12:02:24,355 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
355
+ 2024-02-26 12:02:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
356
+ 2024-02-26 12:02:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
357
+ 2024-02-26 12:02:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
358
+ 2024-02-26 12:02:30,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
359
+ 2024-02-26 12:02:35,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
360
+ 2024-02-26 12:02:40,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
361
+ 2024-02-26 12:02:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
362
+ 2024-02-26 12:02:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
363
+ 2024-02-26 12:02:42,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
364
+ 2024-02-26 12:02:45,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
365
+ 2024-02-26 12:02:50,364 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
366
+ 2024-02-26 12:02:55,364 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
367
+ 2024-02-26 12:02:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
368
+ 2024-02-26 12:02:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
369
+ 2024-02-26 12:02:57,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
370
+ 2024-02-26 12:03:01,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
371
+ 2024-02-26 12:03:06,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
372
+ 2024-02-26 12:03:11,335 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
373
+ 2024-02-26 12:03:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
374
+ 2024-02-26 12:03:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
375
+ 2024-02-26 12:03:12,295 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
376
+ 2024-02-26 12:03:16,991 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
377
+ 2024-02-26 12:03:21,992 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
378
+ 2024-02-26 12:03:26,993 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
379
+ 2024-02-26 12:03:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
380
+ 2024-02-26 12:03:27,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
381
+ 2024-02-26 12:03:27,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
382
+ 2024-02-26 12:03:32,341 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
383
+ 2024-02-26 12:03:37,342 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
384
+ 2024-02-26 12:03:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
385
+ 2024-02-26 12:03:42,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
386
+ 2024-02-26 12:03:42,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
387
+ 2024-02-26 12:03:43,301 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
388
+ 2024-02-26 12:03:48,302 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
389
+ 2024-02-26 12:03:53,303 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
390
+ 2024-02-26 12:03:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
391
+ 2024-02-26 12:03:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
392
+ 2024-02-26 12:03:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
393
+ 2024-02-26 12:03:58,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
394
+ 2024-02-26 12:04:03,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
395
+ 2024-02-26 12:04:08,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
396
+ 2024-02-26 12:04:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
397
+ 2024-02-26 12:04:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
398
+ 2024-02-26 12:04:12,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
399
+ 2024-02-26 12:04:13,386 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
400
+ 2024-02-26 12:04:18,387 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
401
+ 2024-02-26 12:04:23,388 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
402
+ 2024-02-26 12:04:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
403
+ 2024-02-26 12:04:27,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
404
+ 2024-02-26 12:04:27,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
405
+ 2024-02-26 12:04:29,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
406
+ 2024-02-26 12:04:34,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
407
+ 2024-02-26 12:04:39,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
408
+ 2024-02-26 12:04:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
409
+ 2024-02-26 12:04:42,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
410
+ 2024-02-26 12:04:42,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
411
+ 2024-02-26 12:04:45,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
412
+ 2024-02-26 12:04:50,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
413
+ 2024-02-26 12:04:55,306 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
414
+ 2024-02-26 12:04:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
415
+ 2024-02-26 12:04:57,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
416
+ 2024-02-26 12:04:57,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
417
+ 2024-02-26 12:05:00,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
418
+ 2024-02-26 12:05:05,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
419
+ 2024-02-26 12:05:10,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
420
+ 2024-02-26 12:05:12,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
421
+ 2024-02-26 12:05:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
422
+ 2024-02-26 12:05:12,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
423
+ 2024-02-26 12:05:16,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
424
+ 2024-02-26 12:05:21,335 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
425
+ 2024-02-26 12:05:26,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
426
+ 2024-02-26 12:05:27,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
427
+ 2024-02-26 12:05:27,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
428
+ 2024-02-26 12:05:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
429
+ 2024-02-26 12:05:32,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
430
+ 2024-02-26 12:05:37,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
431
+ 2024-02-26 12:05:42,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
432
+ 2024-02-26 12:05:42,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
433
+ 2024-02-26 12:05:42,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
434
+ 2024-02-26 12:05:42,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
435
+ 2024-02-26 12:05:47,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
436
+ 2024-02-26 12:05:52,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
437
+ 2024-02-26 12:05:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
438
+ 2024-02-26 12:05:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
439
+ 2024-02-26 12:05:57,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
440
+ 2024-02-26 12:05:58,317 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
441
+ 2024-02-26 12:06:03,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
442
+ 2024-02-26 12:06:08,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
443
+ 2024-02-26 12:06:12,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
444
+ 2024-02-26 12:06:12,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
445
+ 2024-02-26 12:06:12,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
446
+ 2024-02-26 12:06:13,353 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
447
+ 2024-02-26 12:06:18,354 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
448
+ 2024-02-26 12:06:23,355 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
449
+ 2024-02-26 12:06:27,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
450
+ 2024-02-26 12:06:27,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
451
+ 2024-02-26 12:06:27,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
452
+ 2024-02-26 12:06:28,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
453
+ 2024-02-26 12:06:33,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
454
+ 2024-02-26 12:06:38,392 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
455
+ 2024-02-26 12:06:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
456
+ 2024-02-26 12:06:42,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
457
+ 2024-02-26 12:06:42,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
458
+ 2024-02-26 12:06:44,303 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
459
+ 2024-02-26 12:06:49,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
460
+ 2024-02-26 12:06:54,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
461
+ 2024-02-26 12:06:57,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
462
+ 2024-02-26 12:06:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
463
+ 2024-02-26 12:06:57,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
464
+ 2024-02-26 12:06:59,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
465
+ 2024-02-26 12:07:04,327 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
466
+ 2024-02-26 12:07:09,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
467
+ 2024-02-26 12:07:12,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
468
+ 2024-02-26 12:07:12,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
469
+ 2024-02-26 12:07:12,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
470
+ 2024-02-26 12:07:14,372 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
471
+ 2024-02-26 12:07:19,373 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
472
+ 2024-02-26 12:07:24,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
473
+ 2024-02-26 12:07:27,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
474
+ 2024-02-26 12:07:27,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
475
+ 2024-02-26 12:07:27,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
476
+ 2024-02-26 12:07:30,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
477
+ 2024-02-26 12:07:35,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
478
+ 2024-02-26 12:07:40,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
479
+ 2024-02-26 12:07:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
480
+ 2024-02-26 12:07:42,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
481
+ 2024-02-26 12:07:42,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
482
+ 2024-02-26 12:07:46,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
483
+ 2024-02-26 12:07:51,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
484
+ 2024-02-26 12:07:56,372 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
485
+ 2024-02-26 12:07:57,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
486
+ 2024-02-26 12:07:57,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
487
+ 2024-02-26 12:07:57,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
488
+ 2024-02-26 12:08:01,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
489
+ 2024-02-26 12:08:06,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
490
+ 2024-02-26 12:08:11,393 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
491
+ 2024-02-26 12:08:12,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
492
+ 2024-02-26 12:08:12,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
493
+ 2024-02-26 12:08:12,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
494
+ 2024-02-26 12:08:17,340 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
495
+ 2024-02-26 12:08:22,341 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
496
+ 2024-02-26 12:08:27,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
497
+ 2024-02-26 12:08:27,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
498
+ 2024-02-26 12:08:27,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
499
+ 2024-02-26 12:08:28,300 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
500
+ 2024-02-26 12:08:33,300 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
501
+ 2024-02-26 12:08:38,301 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
502
+ 2024-02-26 12:08:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
503
+ 2024-02-26 12:08:42,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
504
+ 2024-02-26 12:08:42,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
505
+ 2024-02-26 12:08:43,311 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
506
+ 2024-02-26 12:08:48,312 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
507
+ 2024-02-26 12:08:53,313 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
508
+ 2024-02-26 12:08:57,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
509
+ 2024-02-26 12:08:57,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
510
+ 2024-02-26 12:08:57,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
511
+ 2024-02-26 12:08:58,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
512
+ 2024-02-26 12:09:03,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
513
+ 2024-02-26 12:09:08,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
514
+ 2024-02-26 12:09:12,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
515
+ 2024-02-26 12:09:12,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
516
+ 2024-02-26 12:09:12,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
517
+ 2024-02-26 12:09:13,393 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
518
+ 2024-02-26 12:09:18,394 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
519
+ 2024-02-26 12:09:19,077 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
520
+ 2024-02-26 12:09:19,080 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
521
+ 2024-02-26 12:09:19,080 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
522
+ 2024-02-26 12:09:19,081 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
523
+ 2024-02-26 12:09:19,088 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
524
+ 2024-02-26 12:09:19,092 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
525
+ 2024-02-26 12:09:19,101 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
526
+ 2024-02-26 12:09:19,106 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
527
+ 2024-02-26 12:09:19,106 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
528
+ 2024-02-26 12:09:23,394 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
529
+ 2024-02-26 12:09:27,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
530
+ 2024-02-26 12:09:27,259 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
531
+ 2024-02-26 12:09:27,259 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
532
+ 2024-02-26 12:09:29,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
533
+ 2024-02-26 12:09:30,621 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
534
+ 2024-02-26 12:09:30,621 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
535
+ 2024-02-26 12:09:30,622 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
536
+ 2024-02-26 12:09:30,624 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
537
+ 2024-02-26 12:09:30,626 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
538
+ 2024-02-26 12:09:30,626 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
539
+ 2024-02-26 12:09:30,624 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
540
+ 2024-02-26 12:09:30,632 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
541
+ 2024-02-26 12:09:30,642 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
542
+ 2024-02-26 12:09:30,642 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
543
+ 2024-02-26 12:09:30,643 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
544
+ 2024-02-26 12:09:30,644 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
545
+ 2024-02-26 12:09:34,646 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
546
+ 2024-02-26 12:09:39,647 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
547
+ 2024-02-26 12:09:42,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
548
+ 2024-02-26 12:09:42,259 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
549
+ 2024-02-26 12:09:42,259 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
550
+ 2024-02-26 12:09:45,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
551
+ 2024-02-26 12:09:50,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
552
+ 2024-02-26 12:09:55,327 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
553
+ 2024-02-26 12:09:56,160 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
554
+ 2024-02-26 12:09:56,161 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
555
+ 2024-02-26 12:09:56,161 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
556
+ 2024-02-26 12:09:56,164 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
557
+ 2024-02-26 12:09:56,164 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
558
+ 2024-02-26 12:09:56,168 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
559
+ 2024-02-26 12:09:56,169 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
560
+ 2024-02-26 12:09:56,175 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
561
+ 2024-02-26 12:09:56,176 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
562
+ 2024-02-26 12:09:57,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
563
+ 2024-02-26 12:09:57,781 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
564
+ 2024-02-26 12:09:57,782 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
565
+ 2024-02-26 12:10:00,830 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
566
+ 2024-02-26 12:10:05,832 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
wandb/debug.log ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-02-26 11:51:24,250 INFO MainThread:191 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
2
+ 2024-02-26 11:51:24,253 INFO MainThread:191 [wandb_setup.py:_flush():76] Configure stats pid to 191
3
+ 2024-02-26 11:51:24,254 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
4
+ 2024-02-26 11:51:24,255 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/settings
5
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
6
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
7
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': '<python with no main file>'}
8
+ 2024-02-26 11:51:24,257 INFO MainThread:191 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
9
+ 2024-02-26 11:51:24,257 INFO MainThread:191 [wandb_init.py:_log_setup():526] Logging user logs to /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/logs/debug.log
10
+ 2024-02-26 11:51:24,258 INFO MainThread:191 [wandb_init.py:_log_setup():527] Logging internal logs to /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/logs/debug-internal.log
11
+ 2024-02-26 11:51:24,258 INFO MainThread:191 [wandb_init.py:_jupyter_setup():472] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7882d69a6a10>
12
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():566] calling init triggers
13
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
14
+ config: {}
15
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():616] starting backend
16
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():620] setting up manager
17
+ 2024-02-26 11:51:24,264 INFO MainThread:191 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-02-26 11:51:24,266 INFO MainThread:191 [wandb_init.py:init():628] backend started and connected
19
+ 2024-02-26 11:51:24,296 INFO MainThread:191 [wandb_run.py:_label_probe_notebook():1295] probe notebook
20
+ 2024-02-26 11:51:26,743 INFO MainThread:191 [wandb_init.py:init():720] updated telemetry
21
+ 2024-02-26 11:51:26,755 INFO MainThread:191 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
22
+ 2024-02-26 11:51:27,024 INFO MainThread:191 [wandb_run.py:_on_init():2262] communicating current version
23
+ 2024-02-26 11:51:27,146 INFO MainThread:191 [wandb_run.py:_on_init():2271] got version response
24
+ 2024-02-26 11:51:27,146 INFO MainThread:191 [wandb_init.py:init():804] starting run threads in backend
25
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_console_start():2241] atexit reg
26
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_redirect():2096] redirect: wrap_raw
27
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_redirect():2161] Wrapping output streams.
28
+ 2024-02-26 11:51:27,249 INFO MainThread:191 [wandb_run.py:_redirect():2186] Redirects installed.
29
+ 2024-02-26 11:51:27,251 INFO MainThread:191 [wandb_init.py:init():847] run started, returning control to user process
30
+ 2024-02-26 11:51:27,258 INFO MainThread:191 [wandb_run.py:_config_callback():1343] config_cb None None {'vocab_size': 256000, 'max_position_embeddings': 8192, 'hidden_size': 3072, 'intermediate_size': 24576, 'num_hidden_layers': 28, 'num_attention_heads': 16, 'head_dim': 256, 'num_key_value_heads': 16, 'hidden_act': 'gelu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 10000.0, 'attention_bias': False, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GemmaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 2, 'pad_token_id': 0, 'eos_token_id': 1, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'google/gemma-7b', 'transformers_version': '4.38.1', 'model_type': 'gemma', 'rope_scaling': None, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'bnb_4bit_compute_dtype': 'bfloat16', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0002, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 3.0, 'max_steps': 60, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/runs/Feb26_11-29-58_336d6f2eb0af', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 10, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': False, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': True, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}
31
+ 2024-02-26 11:51:38,564 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
32
+ 2024-02-26 11:51:38,565 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
33
+ 2024-02-26 11:52:19,910 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
34
+ 2024-02-26 11:52:29,234 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
35
+ 2024-02-26 11:52:29,234 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
36
+ 2024-02-26 11:53:30,386 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
37
+ 2024-02-26 11:53:43,585 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
38
+ 2024-02-26 11:53:43,585 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
39
+ 2024-02-26 12:09:19,076 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
40
+ 2024-02-26 12:09:30,620 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
41
+ 2024-02-26 12:09:30,620 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
42
+ 2024-02-26 12:09:56,159 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
wandb/run-20240226_115124-aiyp9xfp/files/config.yaml ADDED
@@ -0,0 +1,664 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ python_version: 3.10.12
7
+ cli_version: 0.16.3
8
+ framework: huggingface
9
+ huggingface_version: 4.38.1
10
+ is_jupyter_run: true
11
+ is_kaggle_kernel: false
12
+ start_time: 1708948284.268213
13
+ t:
14
+ 1:
15
+ - 1
16
+ - 2
17
+ - 3
18
+ - 5
19
+ - 11
20
+ - 12
21
+ - 49
22
+ - 51
23
+ - 53
24
+ - 55
25
+ - 71
26
+ - 84
27
+ - 98
28
+ 2:
29
+ - 1
30
+ - 2
31
+ - 3
32
+ - 5
33
+ - 11
34
+ - 12
35
+ - 49
36
+ - 51
37
+ - 53
38
+ - 55
39
+ - 71
40
+ - 84
41
+ - 98
42
+ 3:
43
+ - 7
44
+ - 23
45
+ 4: 3.10.12
46
+ 5: 0.16.3
47
+ 6: 4.38.1
48
+ 8:
49
+ - 1
50
+ - 5
51
+ - 12
52
+ 9:
53
+ 1: transformers_trainer
54
+ 13: linux-x86_64
55
+ m:
56
+ - 1: train/global_step
57
+ 6:
58
+ - 3
59
+ vocab_size:
60
+ desc: null
61
+ value: 256000
62
+ max_position_embeddings:
63
+ desc: null
64
+ value: 8192
65
+ hidden_size:
66
+ desc: null
67
+ value: 3072
68
+ intermediate_size:
69
+ desc: null
70
+ value: 24576
71
+ num_hidden_layers:
72
+ desc: null
73
+ value: 28
74
+ num_attention_heads:
75
+ desc: null
76
+ value: 16
77
+ head_dim:
78
+ desc: null
79
+ value: 256
80
+ num_key_value_heads:
81
+ desc: null
82
+ value: 16
83
+ hidden_act:
84
+ desc: null
85
+ value: gelu
86
+ initializer_range:
87
+ desc: null
88
+ value: 0.02
89
+ rms_norm_eps:
90
+ desc: null
91
+ value: 1.0e-06
92
+ use_cache:
93
+ desc: null
94
+ value: false
95
+ rope_theta:
96
+ desc: null
97
+ value: 10000.0
98
+ attention_bias:
99
+ desc: null
100
+ value: false
101
+ attention_dropout:
102
+ desc: null
103
+ value: 0.0
104
+ return_dict:
105
+ desc: null
106
+ value: true
107
+ output_hidden_states:
108
+ desc: null
109
+ value: false
110
+ output_attentions:
111
+ desc: null
112
+ value: false
113
+ torchscript:
114
+ desc: null
115
+ value: false
116
+ torch_dtype:
117
+ desc: null
118
+ value: bfloat16
119
+ use_bfloat16:
120
+ desc: null
121
+ value: false
122
+ tf_legacy_loss:
123
+ desc: null
124
+ value: false
125
+ pruned_heads:
126
+ desc: null
127
+ value: {}
128
+ tie_word_embeddings:
129
+ desc: null
130
+ value: true
131
+ chunk_size_feed_forward:
132
+ desc: null
133
+ value: 0
134
+ is_encoder_decoder:
135
+ desc: null
136
+ value: false
137
+ is_decoder:
138
+ desc: null
139
+ value: false
140
+ cross_attention_hidden_size:
141
+ desc: null
142
+ value: null
143
+ add_cross_attention:
144
+ desc: null
145
+ value: false
146
+ tie_encoder_decoder:
147
+ desc: null
148
+ value: false
149
+ max_length:
150
+ desc: null
151
+ value: 20
152
+ min_length:
153
+ desc: null
154
+ value: 0
155
+ do_sample:
156
+ desc: null
157
+ value: false
158
+ early_stopping:
159
+ desc: null
160
+ value: false
161
+ num_beams:
162
+ desc: null
163
+ value: 1
164
+ num_beam_groups:
165
+ desc: null
166
+ value: 1
167
+ diversity_penalty:
168
+ desc: null
169
+ value: 0.0
170
+ temperature:
171
+ desc: null
172
+ value: 1.0
173
+ top_k:
174
+ desc: null
175
+ value: 50
176
+ top_p:
177
+ desc: null
178
+ value: 1.0
179
+ typical_p:
180
+ desc: null
181
+ value: 1.0
182
+ repetition_penalty:
183
+ desc: null
184
+ value: 1.0
185
+ length_penalty:
186
+ desc: null
187
+ value: 1.0
188
+ no_repeat_ngram_size:
189
+ desc: null
190
+ value: 0
191
+ encoder_no_repeat_ngram_size:
192
+ desc: null
193
+ value: 0
194
+ bad_words_ids:
195
+ desc: null
196
+ value: null
197
+ num_return_sequences:
198
+ desc: null
199
+ value: 1
200
+ output_scores:
201
+ desc: null
202
+ value: false
203
+ return_dict_in_generate:
204
+ desc: null
205
+ value: false
206
+ forced_bos_token_id:
207
+ desc: null
208
+ value: null
209
+ forced_eos_token_id:
210
+ desc: null
211
+ value: null
212
+ remove_invalid_values:
213
+ desc: null
214
+ value: false
215
+ exponential_decay_length_penalty:
216
+ desc: null
217
+ value: null
218
+ suppress_tokens:
219
+ desc: null
220
+ value: null
221
+ begin_suppress_tokens:
222
+ desc: null
223
+ value: null
224
+ architectures:
225
+ desc: null
226
+ value:
227
+ - GemmaForCausalLM
228
+ finetuning_task:
229
+ desc: null
230
+ value: null
231
+ id2label:
232
+ desc: null
233
+ value:
234
+ '0': LABEL_0
235
+ '1': LABEL_1
236
+ label2id:
237
+ desc: null
238
+ value:
239
+ LABEL_0: 0
240
+ LABEL_1: 1
241
+ tokenizer_class:
242
+ desc: null
243
+ value: null
244
+ prefix:
245
+ desc: null
246
+ value: null
247
+ bos_token_id:
248
+ desc: null
249
+ value: 2
250
+ pad_token_id:
251
+ desc: null
252
+ value: 0
253
+ eos_token_id:
254
+ desc: null
255
+ value: 1
256
+ sep_token_id:
257
+ desc: null
258
+ value: null
259
+ decoder_start_token_id:
260
+ desc: null
261
+ value: null
262
+ task_specific_params:
263
+ desc: null
264
+ value: null
265
+ problem_type:
266
+ desc: null
267
+ value: null
268
+ _name_or_path:
269
+ desc: null
270
+ value: google/gemma-7b
271
+ transformers_version:
272
+ desc: null
273
+ value: 4.38.1
274
+ model_type:
275
+ desc: null
276
+ value: gemma
277
+ rope_scaling:
278
+ desc: null
279
+ value: null
280
+ quantization_config:
281
+ desc: null
282
+ value:
283
+ quant_method: QuantizationMethod.BITS_AND_BYTES
284
+ _load_in_8bit: false
285
+ _load_in_4bit: true
286
+ llm_int8_threshold: 6.0
287
+ llm_int8_skip_modules: null
288
+ llm_int8_enable_fp32_cpu_offload: false
289
+ llm_int8_has_fp16_weight: false
290
+ bnb_4bit_quant_type: nf4
291
+ bnb_4bit_use_double_quant: true
292
+ bnb_4bit_compute_dtype: bfloat16
293
+ load_in_4bit: true
294
+ load_in_8bit: false
295
+ output_dir:
296
+ desc: null
297
+ value: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko
298
+ overwrite_output_dir:
299
+ desc: null
300
+ value: false
301
+ do_train:
302
+ desc: null
303
+ value: false
304
+ do_eval:
305
+ desc: null
306
+ value: false
307
+ do_predict:
308
+ desc: null
309
+ value: false
310
+ evaluation_strategy:
311
+ desc: null
312
+ value: 'no'
313
+ prediction_loss_only:
314
+ desc: null
315
+ value: false
316
+ per_device_train_batch_size:
317
+ desc: null
318
+ value: 2
319
+ per_device_eval_batch_size:
320
+ desc: null
321
+ value: 8
322
+ per_gpu_train_batch_size:
323
+ desc: null
324
+ value: null
325
+ per_gpu_eval_batch_size:
326
+ desc: null
327
+ value: null
328
+ gradient_accumulation_steps:
329
+ desc: null
330
+ value: 2
331
+ eval_accumulation_steps:
332
+ desc: null
333
+ value: null
334
+ eval_delay:
335
+ desc: null
336
+ value: 0
337
+ learning_rate:
338
+ desc: null
339
+ value: 0.0002
340
+ weight_decay:
341
+ desc: null
342
+ value: 0.0
343
+ adam_beta1:
344
+ desc: null
345
+ value: 0.9
346
+ adam_beta2:
347
+ desc: null
348
+ value: 0.999
349
+ adam_epsilon:
350
+ desc: null
351
+ value: 1.0e-08
352
+ max_grad_norm:
353
+ desc: null
354
+ value: 0.3
355
+ num_train_epochs:
356
+ desc: null
357
+ value: 3.0
358
+ max_steps:
359
+ desc: null
360
+ value: 60
361
+ lr_scheduler_type:
362
+ desc: null
363
+ value: cosine
364
+ lr_scheduler_kwargs:
365
+ desc: null
366
+ value: {}
367
+ warmup_ratio:
368
+ desc: null
369
+ value: 0.03
370
+ warmup_steps:
371
+ desc: null
372
+ value: 0
373
+ log_level:
374
+ desc: null
375
+ value: passive
376
+ log_level_replica:
377
+ desc: null
378
+ value: warning
379
+ log_on_each_node:
380
+ desc: null
381
+ value: true
382
+ logging_dir:
383
+ desc: null
384
+ value: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/runs/Feb26_11-29-58_336d6f2eb0af
385
+ logging_strategy:
386
+ desc: null
387
+ value: steps
388
+ logging_first_step:
389
+ desc: null
390
+ value: false
391
+ logging_steps:
392
+ desc: null
393
+ value: 10
394
+ logging_nan_inf_filter:
395
+ desc: null
396
+ value: true
397
+ save_strategy:
398
+ desc: null
399
+ value: steps
400
+ save_steps:
401
+ desc: null
402
+ value: 10
403
+ save_total_limit:
404
+ desc: null
405
+ value: null
406
+ save_safetensors:
407
+ desc: null
408
+ value: true
409
+ save_on_each_node:
410
+ desc: null
411
+ value: false
412
+ save_only_model:
413
+ desc: null
414
+ value: false
415
+ no_cuda:
416
+ desc: null
417
+ value: false
418
+ use_cpu:
419
+ desc: null
420
+ value: false
421
+ use_mps_device:
422
+ desc: null
423
+ value: false
424
+ seed:
425
+ desc: null
426
+ value: 42
427
+ data_seed:
428
+ desc: null
429
+ value: null
430
+ jit_mode_eval:
431
+ desc: null
432
+ value: false
433
+ use_ipex:
434
+ desc: null
435
+ value: false
436
+ bf16:
437
+ desc: null
438
+ value: false
439
+ fp16:
440
+ desc: null
441
+ value: false
442
+ fp16_opt_level:
443
+ desc: null
444
+ value: O1
445
+ half_precision_backend:
446
+ desc: null
447
+ value: auto
448
+ bf16_full_eval:
449
+ desc: null
450
+ value: false
451
+ fp16_full_eval:
452
+ desc: null
453
+ value: false
454
+ tf32:
455
+ desc: null
456
+ value: false
457
+ local_rank:
458
+ desc: null
459
+ value: 0
460
+ ddp_backend:
461
+ desc: null
462
+ value: null
463
+ tpu_num_cores:
464
+ desc: null
465
+ value: null
466
+ tpu_metrics_debug:
467
+ desc: null
468
+ value: false
469
+ debug:
470
+ desc: null
471
+ value: []
472
+ dataloader_drop_last:
473
+ desc: null
474
+ value: false
475
+ eval_steps:
476
+ desc: null
477
+ value: null
478
+ dataloader_num_workers:
479
+ desc: null
480
+ value: 0
481
+ dataloader_prefetch_factor:
482
+ desc: null
483
+ value: null
484
+ past_index:
485
+ desc: null
486
+ value: -1
487
+ run_name:
488
+ desc: null
489
+ value: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko
490
+ disable_tqdm:
491
+ desc: null
492
+ value: false
493
+ remove_unused_columns:
494
+ desc: null
495
+ value: true
496
+ label_names:
497
+ desc: null
498
+ value: null
499
+ load_best_model_at_end:
500
+ desc: null
501
+ value: false
502
+ metric_for_best_model:
503
+ desc: null
504
+ value: null
505
+ greater_is_better:
506
+ desc: null
507
+ value: null
508
+ ignore_data_skip:
509
+ desc: null
510
+ value: false
511
+ fsdp:
512
+ desc: null
513
+ value: []
514
+ fsdp_min_num_params:
515
+ desc: null
516
+ value: 0
517
+ fsdp_config:
518
+ desc: null
519
+ value:
520
+ min_num_params: 0
521
+ xla: false
522
+ xla_fsdp_v2: false
523
+ xla_fsdp_grad_ckpt: false
524
+ fsdp_transformer_layer_cls_to_wrap:
525
+ desc: null
526
+ value: null
527
+ accelerator_config:
528
+ desc: null
529
+ value:
530
+ split_batches: false
531
+ dispatch_batches: null
532
+ even_batches: true
533
+ use_seedable_sampler: true
534
+ deepspeed:
535
+ desc: null
536
+ value: null
537
+ label_smoothing_factor:
538
+ desc: null
539
+ value: 0.0
540
+ optim:
541
+ desc: null
542
+ value: paged_adamw_32bit
543
+ optim_args:
544
+ desc: null
545
+ value: null
546
+ adafactor:
547
+ desc: null
548
+ value: false
549
+ group_by_length:
550
+ desc: null
551
+ value: true
552
+ length_column_name:
553
+ desc: null
554
+ value: length
555
+ report_to:
556
+ desc: null
557
+ value:
558
+ - tensorboard
559
+ - wandb
560
+ ddp_find_unused_parameters:
561
+ desc: null
562
+ value: null
563
+ ddp_bucket_cap_mb:
564
+ desc: null
565
+ value: null
566
+ ddp_broadcast_buffers:
567
+ desc: null
568
+ value: null
569
+ dataloader_pin_memory:
570
+ desc: null
571
+ value: true
572
+ dataloader_persistent_workers:
573
+ desc: null
574
+ value: false
575
+ skip_memory_metrics:
576
+ desc: null
577
+ value: true
578
+ use_legacy_prediction_loop:
579
+ desc: null
580
+ value: false
581
+ push_to_hub:
582
+ desc: null
583
+ value: true
584
+ resume_from_checkpoint:
585
+ desc: null
586
+ value: null
587
+ hub_model_id:
588
+ desc: null
589
+ value: null
590
+ hub_strategy:
591
+ desc: null
592
+ value: every_save
593
+ hub_token:
594
+ desc: null
595
+ value: <HUB_TOKEN>
596
+ hub_private_repo:
597
+ desc: null
598
+ value: false
599
+ hub_always_push:
600
+ desc: null
601
+ value: false
602
+ gradient_checkpointing:
603
+ desc: null
604
+ value: false
605
+ gradient_checkpointing_kwargs:
606
+ desc: null
607
+ value: null
608
+ include_inputs_for_metrics:
609
+ desc: null
610
+ value: false
611
+ fp16_backend:
612
+ desc: null
613
+ value: auto
614
+ push_to_hub_model_id:
615
+ desc: null
616
+ value: null
617
+ push_to_hub_organization:
618
+ desc: null
619
+ value: null
620
+ push_to_hub_token:
621
+ desc: null
622
+ value: <PUSH_TO_HUB_TOKEN>
623
+ mp_parameters:
624
+ desc: null
625
+ value: ''
626
+ auto_find_batch_size:
627
+ desc: null
628
+ value: false
629
+ full_determinism:
630
+ desc: null
631
+ value: false
632
+ torchdynamo:
633
+ desc: null
634
+ value: null
635
+ ray_scope:
636
+ desc: null
637
+ value: last
638
+ ddp_timeout:
639
+ desc: null
640
+ value: 1800
641
+ torch_compile:
642
+ desc: null
643
+ value: false
644
+ torch_compile_backend:
645
+ desc: null
646
+ value: null
647
+ torch_compile_mode:
648
+ desc: null
649
+ value: null
650
+ dispatch_batches:
651
+ desc: null
652
+ value: null
653
+ split_batches:
654
+ desc: null
655
+ value: null
656
+ include_tokens_per_second:
657
+ desc: null
658
+ value: false
659
+ include_num_input_tokens_seen:
660
+ desc: null
661
+ value: false
662
+ neftune_noise_alpha:
663
+ desc: null
664
+ value: null
wandb/run-20240226_115124-aiyp9xfp/files/requirements.txt ADDED
@@ -0,0 +1,481 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==1.4.0
2
+ aiohttp==3.9.3
3
+ aiosignal==1.3.1
4
+ alabaster==0.7.16
5
+ albumentations==1.3.1
6
+ altair==4.2.2
7
+ annotated-types==0.6.0
8
+ anyio==3.7.1
9
+ appdirs==1.4.4
10
+ argon2-cffi-bindings==21.2.0
11
+ argon2-cffi==23.1.0
12
+ array-record==0.5.0
13
+ arviz==0.15.1
14
+ astropy==5.3.4
15
+ astunparse==1.6.3
16
+ async-timeout==4.0.3
17
+ atpublic==4.0
18
+ attrs==23.2.0
19
+ audioread==3.0.1
20
+ autograd==1.6.2
21
+ babel==2.14.0
22
+ backcall==0.2.0
23
+ beautifulsoup4==4.12.3
24
+ bidict==0.23.0
25
+ bigframes==0.21.0
26
+ bleach==6.1.0
27
+ blinker==1.4
28
+ blis==0.7.11
29
+ blosc2==2.0.0
30
+ bokeh==3.3.4
31
+ bqplot==0.12.42
32
+ branca==0.7.1
33
+ build==1.0.3
34
+ cachecontrol==0.14.0
35
+ cachetools==5.3.2
36
+ catalogue==2.0.10
37
+ certifi==2024.2.2
38
+ cffi==1.16.0
39
+ chardet==5.2.0
40
+ charset-normalizer==3.3.2
41
+ chex==0.1.85
42
+ click-plugins==1.1.1
43
+ click==8.1.7
44
+ cligj==0.7.2
45
+ cloudpathlib==0.16.0
46
+ cloudpickle==2.2.1
47
+ cmake==3.27.9
48
+ cmdstanpy==1.2.1
49
+ colorcet==3.0.1
50
+ colorlover==0.3.0
51
+ colour==0.1.5
52
+ community==1.0.0b1
53
+ confection==0.1.4
54
+ cons==0.4.6
55
+ contextlib2==21.6.0
56
+ contourpy==1.2.0
57
+ cryptography==42.0.3
58
+ cufflinks==0.17.3
59
+ cupy-cuda12x==12.2.0
60
+ cvxopt==1.3.2
61
+ cvxpy==1.3.3
62
+ cycler==0.12.1
63
+ cymem==2.0.8
64
+ cython==3.0.8
65
+ dask==2023.8.1
66
+ datascience==0.17.6
67
+ db-dtypes==1.2.0
68
+ dbus-python==1.2.18
69
+ debugpy==1.6.6
70
+ decorator==4.4.2
71
+ defusedxml==0.7.1
72
+ distributed==2023.8.1
73
+ distro==1.7.0
74
+ dlib==19.24.2
75
+ dm-tree==0.1.8
76
+ docutils==0.18.1
77
+ dopamine-rl==4.0.6
78
+ duckdb==0.9.2
79
+ earthengine-api==0.1.390
80
+ easydict==1.12
81
+ ecos==2.0.13
82
+ editdistance==0.6.2
83
+ eerepr==0.0.4
84
+ en-core-web-sm==3.7.1
85
+ entrypoints==0.4
86
+ et-xmlfile==1.1.0
87
+ etils==1.7.0
88
+ etuples==0.3.9
89
+ exceptiongroup==1.2.0
90
+ fastai==2.7.14
91
+ fastcore==1.5.29
92
+ fastdownload==0.0.7
93
+ fastjsonschema==2.19.1
94
+ fastprogress==1.0.3
95
+ fastrlock==0.8.2
96
+ filelock==3.13.1
97
+ fiona==1.9.5
98
+ firebase-admin==5.3.0
99
+ flask==2.2.5
100
+ flatbuffers==23.5.26
101
+ flax==0.8.1
102
+ folium==0.14.0
103
+ fonttools==4.49.0
104
+ frozendict==2.4.0
105
+ frozenlist==1.4.1
106
+ fsspec==2023.6.0
107
+ future==0.18.3
108
+ gast==0.5.4
109
+ gcsfs==2023.6.0
110
+ gdal==3.6.4
111
+ gdown==4.7.3
112
+ geemap==0.31.0
113
+ gensim==4.3.2
114
+ geocoder==1.38.1
115
+ geographiclib==2.0
116
+ geopandas==0.13.2
117
+ geopy==2.3.0
118
+ gin-config==0.5.0
119
+ glob2==0.7
120
+ google-ai-generativelanguage==0.4.0
121
+ google-api-core==2.11.1
122
+ google-api-python-client==2.84.0
123
+ google-auth-httplib2==0.1.1
124
+ google-auth-oauthlib==1.2.0
125
+ google-auth==2.27.0
126
+ google-cloud-aiplatform==1.42.1
127
+ google-cloud-bigquery-connection==1.12.1
128
+ google-cloud-bigquery-storage==2.24.0
129
+ google-cloud-bigquery==3.12.0
130
+ google-cloud-core==2.3.3
131
+ google-cloud-datastore==2.15.2
132
+ google-cloud-firestore==2.11.1
133
+ google-cloud-functions==1.13.3
134
+ google-cloud-iam==2.14.1
135
+ google-cloud-language==2.13.1
136
+ google-cloud-resource-manager==1.12.1
137
+ google-cloud-storage==2.8.0
138
+ google-cloud-translate==3.11.3
139
+ google-colab==1.0.0
140
+ google-crc32c==1.5.0
141
+ google-generativeai==0.3.2
142
+ google-pasta==0.2.0
143
+ google-resumable-media==2.7.0
144
+ google==2.0.3
145
+ googleapis-common-protos==1.62.0
146
+ googledrivedownloader==0.4
147
+ graphviz==0.20.1
148
+ greenlet==3.0.3
149
+ grpc-google-iam-v1==0.13.0
150
+ grpcio-status==1.48.2
151
+ grpcio==1.60.1
152
+ gspread-dataframe==3.3.1
153
+ gspread==3.4.2
154
+ gym-notices==0.0.8
155
+ gym==0.25.2
156
+ h5netcdf==1.3.0
157
+ h5py==3.9.0
158
+ holidays==0.42
159
+ holoviews==1.17.1
160
+ html5lib==1.1
161
+ httpimport==1.3.1
162
+ httplib2==0.22.0
163
+ huggingface-hub==0.20.3
164
+ humanize==4.7.0
165
+ hyperopt==0.2.7
166
+ ibis-framework==7.1.0
167
+ idna==3.6
168
+ imageio-ffmpeg==0.4.9
169
+ imageio==2.31.6
170
+ imagesize==1.4.1
171
+ imbalanced-learn==0.10.1
172
+ imgaug==0.4.0
173
+ importlib-metadata==7.0.1
174
+ importlib-resources==6.1.1
175
+ imutils==0.5.4
176
+ inflect==7.0.0
177
+ iniconfig==2.0.0
178
+ intel-openmp==2023.2.3
179
+ ipyevents==2.0.2
180
+ ipyfilechooser==0.6.0
181
+ ipykernel==5.5.6
182
+ ipyleaflet==0.18.2
183
+ ipython-genutils==0.2.0
184
+ ipython-sql==0.5.0
185
+ ipython==7.34.0
186
+ ipytree==0.2.2
187
+ ipywidgets==7.7.1
188
+ itsdangerous==2.1.2
189
+ jax==0.4.23
190
+ jaxlib==0.4.23+cuda12.cudnn89
191
+ jeepney==0.7.1
192
+ jieba==0.42.1
193
+ jinja2==3.1.3
194
+ joblib==1.3.2
195
+ jsonpickle==3.0.2
196
+ jsonschema-specifications==2023.12.1
197
+ jsonschema==4.19.2
198
+ jupyter-client==6.1.12
199
+ jupyter-console==6.1.0
200
+ jupyter-core==5.7.1
201
+ jupyter-server==1.24.0
202
+ jupyterlab-pygments==0.3.0
203
+ jupyterlab-widgets==3.0.10
204
+ kaggle==1.5.16
205
+ kagglehub==0.1.9
206
+ keras==2.15.0
207
+ keyring==23.5.0
208
+ kiwisolver==1.4.5
209
+ langcodes==3.3.0
210
+ launchpadlib==1.10.16
211
+ lazr.restfulclient==0.14.4
212
+ lazr.uri==1.0.6
213
+ lazy-loader==0.3
214
+ libclang==16.0.6
215
+ librosa==0.10.1
216
+ lightgbm==4.1.0
217
+ linkify-it-py==2.0.3
218
+ llvmlite==0.41.1
219
+ locket==1.0.0
220
+ logical-unification==0.4.6
221
+ lxml==4.9.4
222
+ malloy==2023.1067
223
+ markdown-it-py==3.0.0
224
+ markdown==3.5.2
225
+ markupsafe==2.1.5
226
+ matplotlib-inline==0.1.6
227
+ matplotlib-venn==0.11.10
228
+ matplotlib==3.7.1
229
+ mdit-py-plugins==0.4.0
230
+ mdurl==0.1.2
231
+ minikanren==1.0.3
232
+ missingno==0.5.2
233
+ mistune==0.8.4
234
+ mizani==0.9.3
235
+ mkl==2023.2.0
236
+ ml-dtypes==0.2.0
237
+ mlxtend==0.22.0
238
+ more-itertools==10.1.0
239
+ moviepy==1.0.3
240
+ mpmath==1.3.0
241
+ msgpack==1.0.7
242
+ multidict==6.0.5
243
+ multipledispatch==1.0.0
244
+ multitasking==0.0.11
245
+ murmurhash==1.0.10
246
+ music21==9.1.0
247
+ natsort==8.4.0
248
+ nbclassic==1.0.0
249
+ nbclient==0.9.0
250
+ nbconvert==6.5.4
251
+ nbformat==5.9.2
252
+ nest-asyncio==1.6.0
253
+ networkx==3.2.1
254
+ nibabel==4.0.2
255
+ nltk==3.8.1
256
+ notebook-shim==0.2.4
257
+ notebook==6.5.5
258
+ numba==0.58.1
259
+ numexpr==2.9.0
260
+ numpy==1.25.2
261
+ oauth2client==4.1.3
262
+ oauthlib==3.2.2
263
+ opencv-contrib-python==4.8.0.76
264
+ opencv-python-headless==4.9.0.80
265
+ opencv-python==4.8.0.76
266
+ openpyxl==3.1.2
267
+ opt-einsum==3.3.0
268
+ optax==0.1.9
269
+ orbax-checkpoint==0.4.4
270
+ osqp==0.6.2.post8
271
+ packaging==23.2
272
+ pandas-datareader==0.10.0
273
+ pandas-gbq==0.19.2
274
+ pandas-stubs==1.5.3.230304
275
+ pandas==1.5.3
276
+ pandocfilters==1.5.1
277
+ panel==1.3.8
278
+ param==2.0.2
279
+ parso==0.8.3
280
+ parsy==2.1
281
+ partd==1.4.1
282
+ pathlib==1.0.1
283
+ patsy==0.5.6
284
+ peewee==3.17.1
285
+ pexpect==4.9.0
286
+ pickleshare==0.7.5
287
+ pillow==9.4.0
288
+ pins==0.8.4
289
+ pip-tools==6.13.0
290
+ pip==23.1.2
291
+ platformdirs==4.2.0
292
+ plotly==5.15.0
293
+ plotnine==0.12.4
294
+ pluggy==1.4.0
295
+ polars==0.20.2
296
+ pooch==1.8.0
297
+ portpicker==1.5.2
298
+ prefetch-generator==1.0.3
299
+ preshed==3.0.9
300
+ prettytable==3.9.0
301
+ proglog==0.1.10
302
+ progressbar2==4.2.0
303
+ prometheus-client==0.20.0
304
+ promise==2.3
305
+ prompt-toolkit==3.0.43
306
+ prophet==1.1.5
307
+ proto-plus==1.23.0
308
+ protobuf==3.20.3
309
+ psutil==5.9.5
310
+ psycopg2==2.9.9
311
+ ptyprocess==0.7.0
312
+ py-cpuinfo==9.0.0
313
+ py4j==0.10.9.7
314
+ pyarrow-hotfix==0.6
315
+ pyarrow==14.0.2
316
+ pyasn1-modules==0.3.0
317
+ pyasn1==0.5.1
318
+ pycocotools==2.0.7
319
+ pycparser==2.21
320
+ pyct==0.5.0
321
+ pydantic-core==2.16.2
322
+ pydantic==2.6.1
323
+ pydata-google-auth==1.8.2
324
+ pydot-ng==2.0.0
325
+ pydot==1.4.2
326
+ pydotplus==2.0.2
327
+ pydrive2==1.6.3
328
+ pydrive==1.3.1
329
+ pyerfa==2.0.1.1
330
+ pygame==2.5.2
331
+ pygments==2.16.1
332
+ pygobject==3.42.1
333
+ pyjwt==2.3.0
334
+ pymc==5.7.2
335
+ pymystem3==0.2.0
336
+ pyopengl==3.1.7
337
+ pyopenssl==24.0.0
338
+ pyparsing==3.1.1
339
+ pyperclip==1.8.2
340
+ pyproj==3.6.1
341
+ pyproject-hooks==1.0.0
342
+ pyshp==2.3.1
343
+ pysocks==1.7.1
344
+ pytensor==2.14.2
345
+ pytest==7.4.4
346
+ python-apt==0.0.0
347
+ python-box==7.1.1
348
+ python-dateutil==2.8.2
349
+ python-louvain==0.16
350
+ python-slugify==8.0.4
351
+ python-utils==3.8.2
352
+ pytz==2023.4
353
+ pyviz-comms==3.0.1
354
+ pywavelets==1.5.0
355
+ pyyaml==6.0.1
356
+ pyzmq==23.2.1
357
+ qdldl==0.1.7.post0
358
+ qudida==0.0.4
359
+ ratelim==0.1.6
360
+ referencing==0.33.0
361
+ regex==2023.12.25
362
+ requests-oauthlib==1.3.1
363
+ requests==2.31.0
364
+ requirements-parser==0.5.0
365
+ rich==13.7.0
366
+ rpds-py==0.18.0
367
+ rpy2==3.4.2
368
+ rsa==4.9
369
+ safetensors==0.4.2
370
+ scikit-image==0.19.3
371
+ scikit-learn==1.2.2
372
+ scipy==1.11.4
373
+ scooby==0.9.2
374
+ scs==3.2.4.post1
375
+ seaborn==0.13.1
376
+ secretstorage==3.3.1
377
+ send2trash==1.8.2
378
+ sentencepiece==0.1.99
379
+ setuptools==67.7.2
380
+ shapely==2.0.3
381
+ six==1.16.0
382
+ sklearn-pandas==2.2.0
383
+ smart-open==6.4.0
384
+ sniffio==1.3.0
385
+ snowballstemmer==2.2.0
386
+ sortedcontainers==2.4.0
387
+ soundfile==0.12.1
388
+ soupsieve==2.5
389
+ soxr==0.3.7
390
+ spacy-legacy==3.0.12
391
+ spacy-loggers==1.0.5
392
+ spacy==3.7.4
393
+ sphinx==5.0.2
394
+ sphinxcontrib-applehelp==1.0.8
395
+ sphinxcontrib-devhelp==1.0.6
396
+ sphinxcontrib-htmlhelp==2.0.5
397
+ sphinxcontrib-jsmath==1.0.1
398
+ sphinxcontrib-qthelp==1.0.7
399
+ sphinxcontrib-serializinghtml==1.1.10
400
+ sqlalchemy==2.0.27
401
+ sqlglot==19.9.0
402
+ sqlparse==0.4.4
403
+ srsly==2.4.8
404
+ stanio==0.3.0
405
+ statsmodels==0.14.1
406
+ sympy==1.12
407
+ tables==3.8.0
408
+ tabulate==0.9.0
409
+ tbb==2021.11.0
410
+ tblib==3.0.0
411
+ tenacity==8.2.3
412
+ tensorboard-data-server==0.7.2
413
+ tensorboard==2.15.2
414
+ tensorflow-datasets==4.9.4
415
+ tensorflow-estimator==2.15.0
416
+ tensorflow-gcs-config==2.15.0
417
+ tensorflow-hub==0.16.1
418
+ tensorflow-io-gcs-filesystem==0.36.0
419
+ tensorflow-metadata==1.14.0
420
+ tensorflow-probability==0.23.0
421
+ tensorflow==2.15.0
422
+ tensorstore==0.1.45
423
+ termcolor==2.4.0
424
+ terminado==0.18.0
425
+ text-unidecode==1.3
426
+ textblob==0.17.1
427
+ tf-keras==2.15.0
428
+ tf-slim==1.1.0
429
+ thinc==8.2.3
430
+ threadpoolctl==3.3.0
431
+ tifffile==2024.2.12
432
+ tinycss2==1.2.1
433
+ tokenizers==0.15.2
434
+ toml==0.10.2
435
+ tomli==2.0.1
436
+ toolz==0.12.1
437
+ torch==2.1.0+cu121
438
+ torchaudio==2.1.0+cu121
439
+ torchdata==0.7.0
440
+ torchsummary==1.5.1
441
+ torchtext==0.16.0
442
+ torchvision==0.16.0+cu121
443
+ tornado==6.3.2
444
+ tqdm==4.66.2
445
+ traitlets==5.7.1
446
+ traittypes==0.2.1
447
+ transformers==4.37.2
448
+ triton==2.1.0
449
+ tweepy==4.14.0
450
+ typer==0.9.0
451
+ types-pytz==2024.1.0.20240203
452
+ types-setuptools==69.1.0.20240217
453
+ typing-extensions==4.9.0
454
+ tzlocal==5.2
455
+ uc-micro-py==1.0.3
456
+ uritemplate==4.1.1
457
+ urllib3==2.0.7
458
+ vega-datasets==0.9.0
459
+ wadllib==1.3.6
460
+ wasabi==1.1.2
461
+ wcwidth==0.2.13
462
+ weasel==0.3.4
463
+ webcolors==1.13
464
+ webencodings==0.5.1
465
+ websocket-client==1.7.0
466
+ werkzeug==3.0.1
467
+ wheel==0.42.0
468
+ widgetsnbextension==3.6.6
469
+ wordcloud==1.9.3
470
+ wrapt==1.14.1
471
+ xarray-einstats==0.7.0
472
+ xarray==2023.7.0
473
+ xgboost==2.0.3
474
+ xlrd==2.0.1
475
+ xxhash==3.4.1
476
+ xyzservices==2023.10.1
477
+ yarl==1.9.4
478
+ yellowbrick==1.5
479
+ yfinance==0.2.36
480
+ zict==3.0.0
481
+ zipp==3.17.0
wandb/run-20240226_115124-aiyp9xfp/files/wandb-metadata.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.1.58+-x86_64-with-glibc2.35",
3
+ "python": "3.10.12",
4
+ "heartbeatAt": "2024-02-26T11:51:27.182772",
5
+ "startedAt": "2024-02-26T11:51:24.188771",
6
+ "docker": null,
7
+ "cuda": null,
8
+ "args": [],
9
+ "state": "running",
10
+ "program": "Gemma-7B-Finetuning-databricks-dolly-15k-ko.ipynb",
11
+ "codePathLocal": null,
12
+ "colab": "https://colab.research.google.com/notebook#fileId=1wMe78fcAk4lrbXJnoOMsmSpeUqlmneh3",
13
+ "host": "336d6f2eb0af",
14
+ "username": "root",
15
+ "executable": "/usr/bin/python3",
16
+ "cpu_count": 1,
17
+ "cpu_count_logical": 2,
18
+ "cpu_freq": {
19
+ "current": 2299.998,
20
+ "min": 0.0,
21
+ "max": 0.0
22
+ },
23
+ "cpu_freq_per_core": [
24
+ {
25
+ "current": 2299.998,
26
+ "min": 0.0,
27
+ "max": 0.0
28
+ },
29
+ {
30
+ "current": 2299.998,
31
+ "min": 0.0,
32
+ "max": 0.0
33
+ }
34
+ ],
35
+ "disk": {
36
+ "/": {
37
+ "total": 78.1898422241211,
38
+ "used": 42.809574127197266
39
+ }
40
+ },
41
+ "gpu": "Tesla T4",
42
+ "gpu_count": 1,
43
+ "gpu_devices": [
44
+ {
45
+ "name": "Tesla T4",
46
+ "memory_total": 16106127360
47
+ }
48
+ ],
49
+ "memory": {
50
+ "total": 12.674789428710938
51
+ }
52
+ }
wandb/run-20240226_115124-aiyp9xfp/logs/debug-internal.log ADDED
@@ -0,0 +1,566 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-02-26 11:51:24,269 INFO StreamThr :8773 [internal.py:wandb_internal():86] W&B internal server running at pid: 8773, started at: 2024-02-26 11:51:24.265817
2
+ 2024-02-26 11:51:24,273 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status
3
+ 2024-02-26 11:51:26,744 INFO WriterThread:8773 [datastore.py:open_for_write():87] open: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/run-aiyp9xfp.wandb
4
+ 2024-02-26 11:51:26,751 DEBUG SenderThread:8773 [sender.py:send():382] send: header
5
+ 2024-02-26 11:51:26,756 DEBUG SenderThread:8773 [sender.py:send():382] send: run
6
+ 2024-02-26 11:51:27,013 INFO SenderThread:8773 [dir_watcher.py:__init__():211] watching files in: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files
7
+ 2024-02-26 11:51:27,014 INFO SenderThread:8773 [sender.py:_start_run_threads():1136] run started: aiyp9xfp with start time 1708948284.268213
8
+ 2024-02-26 11:51:27,024 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: check_version
9
+ 2024-02-26 11:51:27,025 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: check_version
10
+ 2024-02-26 11:51:27,158 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: run_start
11
+ 2024-02-26 11:51:27,168 DEBUG HandlerThread:8773 [system_info.py:__init__():27] System info init
12
+ 2024-02-26 11:51:27,168 DEBUG HandlerThread:8773 [system_info.py:__init__():42] System info init done
13
+ 2024-02-26 11:51:27,168 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
14
+ 2024-02-26 11:51:27,168 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
15
+ 2024-02-26 11:51:27,169 INFO HandlerThread:8773 [system_monitor.py:probe():214] Collecting system info
16
+ 2024-02-26 11:51:27,169 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
17
+ 2024-02-26 11:51:27,170 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
18
+ 2024-02-26 11:51:27,172 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
19
+ 2024-02-26 11:51:27,172 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
20
+ 2024-02-26 11:51:27,173 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
21
+ 2024-02-26 11:51:27,182 DEBUG HandlerThread:8773 [system_info.py:probe():151] Probing system
22
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [gitlib.py:_init_repo():56] git repository is invalid
23
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [system_info.py:probe():199] Probing system done
24
+ 2024-02-26 11:51:27,188 DEBUG HandlerThread:8773 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-02-26T11:51:27.182772', 'startedAt': '2024-02-26T11:51:24.188771', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Gemma-7B-Finetuning-databricks-dolly-15k-ko.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1wMe78fcAk4lrbXJnoOMsmSpeUqlmneh3', 'host': '336d6f2eb0af', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 1, 'cpu_count_logical': 2, 'cpu_freq': {'current': 2299.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 78.1898422241211, 'used': 42.809574127197266}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 12.674789428710938}}
25
+ 2024-02-26 11:51:27,188 INFO HandlerThread:8773 [system_monitor.py:probe():224] Finished collecting system info
26
+ 2024-02-26 11:51:27,189 INFO HandlerThread:8773 [system_monitor.py:probe():227] Publishing system info
27
+ 2024-02-26 11:51:27,207 INFO HandlerThread:8773 [system_monitor.py:probe():229] Finished publishing system info
28
+ 2024-02-26 11:51:27,216 DEBUG SenderThread:8773 [sender.py:send():382] send: files
29
+ 2024-02-26 11:51:27,216 INFO SenderThread:8773 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
30
+ 2024-02-26 11:51:27,244 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: python_packages
31
+ 2024-02-26 11:51:27,248 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: python_packages
32
+ 2024-02-26 11:51:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
33
+ 2024-02-26 11:51:27,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
34
+ 2024-02-26 11:51:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
35
+ 2024-02-26 11:51:27,471 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
36
+ 2024-02-26 11:51:27,471 DEBUG SenderThread:8773 [sender.py:send():382] send: config
37
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: metric
38
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
39
+ 2024-02-26 11:51:27,473 DEBUG SenderThread:8773 [sender.py:send():382] send: metric
40
+ 2024-02-26 11:51:27,473 WARNING SenderThread:8773 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen)
41
+ 2024-02-26 11:51:27,474 DEBUG SenderThread:8773 [sender.py:send():382] send: telemetry
42
+ 2024-02-26 11:51:27,529 INFO wandb-upload_0:8773 [upload_job.py:push():131] Uploaded file /tmp/tmp7ayn3ah6wandb/rx7k0how-wandb-metadata.json
43
+ 2024-02-26 11:51:28,019 INFO Thread-12 :8773 [dir_watcher.py:_on_file_created():271] file/dir created: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/wandb-metadata.json
44
+ 2024-02-26 11:51:28,020 INFO Thread-12 :8773 [dir_watcher.py:_on_file_created():271] file/dir created: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/requirements.txt
45
+ 2024-02-26 11:51:29,474 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
46
+ 2024-02-26 11:51:34,475 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
47
+ 2024-02-26 11:51:38,566 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
48
+ 2024-02-26 11:51:38,566 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
49
+ 2024-02-26 11:51:38,566 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
50
+ 2024-02-26 11:51:38,568 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
51
+ 2024-02-26 11:51:38,568 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
52
+ 2024-02-26 11:51:38,569 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
53
+ 2024-02-26 11:51:38,569 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
54
+ 2024-02-26 11:51:38,569 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
55
+ 2024-02-26 11:51:38,584 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
56
+ 2024-02-26 11:51:38,585 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
57
+ 2024-02-26 11:51:38,585 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
58
+ 2024-02-26 11:51:38,586 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
59
+ 2024-02-26 11:51:39,586 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
60
+ 2024-02-26 11:51:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
61
+ 2024-02-26 11:51:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
62
+ 2024-02-26 11:51:42,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
63
+ 2024-02-26 11:51:45,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
64
+ 2024-02-26 11:51:50,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
65
+ 2024-02-26 11:51:55,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
66
+ 2024-02-26 11:51:56,068 INFO Thread-12 :8773 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/files/config.yaml
67
+ 2024-02-26 11:51:57,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
68
+ 2024-02-26 11:51:57,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
69
+ 2024-02-26 11:51:57,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
70
+ 2024-02-26 11:52:01,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
71
+ 2024-02-26 11:52:06,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
72
+ 2024-02-26 11:52:11,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
73
+ 2024-02-26 11:52:12,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
74
+ 2024-02-26 11:52:12,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
75
+ 2024-02-26 11:52:12,288 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
76
+ 2024-02-26 11:52:17,321 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
77
+ 2024-02-26 11:52:19,911 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
78
+ 2024-02-26 11:52:19,916 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
79
+ 2024-02-26 11:52:19,916 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
80
+ 2024-02-26 11:52:19,925 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
81
+ 2024-02-26 11:52:19,926 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
82
+ 2024-02-26 11:52:19,931 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
83
+ 2024-02-26 11:52:19,931 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
84
+ 2024-02-26 11:52:19,943 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
85
+ 2024-02-26 11:52:19,944 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
86
+ 2024-02-26 11:52:22,323 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
87
+ 2024-02-26 11:52:27,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
88
+ 2024-02-26 11:52:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
89
+ 2024-02-26 11:52:27,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
90
+ 2024-02-26 11:52:27,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
91
+ 2024-02-26 11:52:29,235 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
92
+ 2024-02-26 11:52:29,236 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
93
+ 2024-02-26 11:52:29,236 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
94
+ 2024-02-26 11:52:29,236 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
95
+ 2024-02-26 11:52:29,238 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
96
+ 2024-02-26 11:52:29,239 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
97
+ 2024-02-26 11:52:29,240 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
98
+ 2024-02-26 11:52:29,240 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
99
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
100
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
101
+ 2024-02-26 11:52:29,249 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
102
+ 2024-02-26 11:52:29,250 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
103
+ 2024-02-26 11:52:33,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
104
+ 2024-02-26 11:52:38,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
105
+ 2024-02-26 11:52:42,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
106
+ 2024-02-26 11:52:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
107
+ 2024-02-26 11:52:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
108
+ 2024-02-26 11:52:43,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
109
+ 2024-02-26 11:52:48,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
110
+ 2024-02-26 11:52:53,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
111
+ 2024-02-26 11:52:57,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
112
+ 2024-02-26 11:52:57,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
113
+ 2024-02-26 11:52:57,271 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
114
+ 2024-02-26 11:52:59,272 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
115
+ 2024-02-26 11:53:04,273 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
116
+ 2024-02-26 11:53:09,274 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
117
+ 2024-02-26 11:53:12,248 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
118
+ 2024-02-26 11:53:12,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
119
+ 2024-02-26 11:53:12,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
120
+ 2024-02-26 11:53:14,368 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
121
+ 2024-02-26 11:53:19,368 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
122
+ 2024-02-26 11:53:24,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
123
+ 2024-02-26 11:53:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
124
+ 2024-02-26 11:53:27,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
125
+ 2024-02-26 11:53:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
126
+ 2024-02-26 11:53:30,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
127
+ 2024-02-26 11:53:30,386 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
128
+ 2024-02-26 11:53:30,387 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
129
+ 2024-02-26 11:53:30,390 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
130
+ 2024-02-26 11:53:30,394 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
131
+ 2024-02-26 11:53:30,395 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
132
+ 2024-02-26 11:53:30,395 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
133
+ 2024-02-26 11:53:30,399 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
134
+ 2024-02-26 11:53:30,406 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
135
+ 2024-02-26 11:53:30,409 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
136
+ 2024-02-26 11:53:35,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
137
+ 2024-02-26 11:53:40,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
138
+ 2024-02-26 11:53:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
139
+ 2024-02-26 11:53:42,249 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
140
+ 2024-02-26 11:53:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
141
+ 2024-02-26 11:53:43,588 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
142
+ 2024-02-26 11:53:43,588 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
143
+ 2024-02-26 11:53:43,588 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
144
+ 2024-02-26 11:53:43,592 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
145
+ 2024-02-26 11:53:43,596 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
146
+ 2024-02-26 11:53:43,597 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
147
+ 2024-02-26 11:53:43,598 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
148
+ 2024-02-26 11:53:43,598 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
149
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
150
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
151
+ 2024-02-26 11:53:43,603 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
152
+ 2024-02-26 11:53:43,604 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
153
+ 2024-02-26 11:53:45,605 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
154
+ 2024-02-26 11:53:50,606 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
155
+ 2024-02-26 11:53:55,607 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
156
+ 2024-02-26 11:53:57,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
157
+ 2024-02-26 11:53:57,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
158
+ 2024-02-26 11:53:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
159
+ 2024-02-26 11:54:01,347 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
160
+ 2024-02-26 11:54:06,348 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
161
+ 2024-02-26 11:54:11,349 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
162
+ 2024-02-26 11:54:12,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
163
+ 2024-02-26 11:54:12,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
164
+ 2024-02-26 11:54:12,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
165
+ 2024-02-26 11:54:16,559 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
166
+ 2024-02-26 11:54:21,560 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
167
+ 2024-02-26 11:54:26,561 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
168
+ 2024-02-26 11:54:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
169
+ 2024-02-26 11:54:27,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
170
+ 2024-02-26 11:54:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
171
+ 2024-02-26 11:54:32,307 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
172
+ 2024-02-26 11:54:37,307 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
173
+ 2024-02-26 11:54:42,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
174
+ 2024-02-26 11:54:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
175
+ 2024-02-26 11:54:42,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
176
+ 2024-02-26 11:54:46,023 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
177
+ 2024-02-26 11:54:51,024 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
178
+ 2024-02-26 11:54:56,025 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
179
+ 2024-02-26 11:54:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
180
+ 2024-02-26 11:54:57,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
181
+ 2024-02-26 11:54:57,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
182
+ 2024-02-26 11:55:01,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
183
+ 2024-02-26 11:55:06,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
184
+ 2024-02-26 11:55:11,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
185
+ 2024-02-26 11:55:12,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
186
+ 2024-02-26 11:55:12,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
187
+ 2024-02-26 11:55:12,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
188
+ 2024-02-26 11:55:17,339 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
189
+ 2024-02-26 11:55:22,340 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
190
+ 2024-02-26 11:55:27,249 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
191
+ 2024-02-26 11:55:27,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
192
+ 2024-02-26 11:55:27,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
193
+ 2024-02-26 11:55:27,389 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
194
+ 2024-02-26 11:55:32,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
195
+ 2024-02-26 11:55:37,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
196
+ 2024-02-26 11:55:42,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
197
+ 2024-02-26 11:55:42,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
198
+ 2024-02-26 11:55:42,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
199
+ 2024-02-26 11:55:43,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
200
+ 2024-02-26 11:55:48,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
201
+ 2024-02-26 11:55:53,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
202
+ 2024-02-26 11:55:57,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
203
+ 2024-02-26 11:55:57,250 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
204
+ 2024-02-26 11:55:57,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
205
+ 2024-02-26 11:55:58,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
206
+ 2024-02-26 11:56:03,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
207
+ 2024-02-26 11:56:08,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
208
+ 2024-02-26 11:56:12,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
209
+ 2024-02-26 11:56:12,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
210
+ 2024-02-26 11:56:12,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
211
+ 2024-02-26 11:56:14,365 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
212
+ 2024-02-26 11:56:19,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
213
+ 2024-02-26 11:56:24,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
214
+ 2024-02-26 11:56:27,250 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
215
+ 2024-02-26 11:56:27,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
216
+ 2024-02-26 11:56:27,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
217
+ 2024-02-26 11:56:30,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
218
+ 2024-02-26 11:56:35,309 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
219
+ 2024-02-26 11:56:40,310 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
220
+ 2024-02-26 11:56:42,251 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
221
+ 2024-02-26 11:56:42,251 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
222
+ 2024-02-26 11:56:42,291 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
223
+ 2024-02-26 11:56:46,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
224
+ 2024-02-26 11:56:51,308 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
225
+ 2024-02-26 11:56:56,315 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
226
+ 2024-02-26 11:56:57,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
227
+ 2024-02-26 11:56:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
228
+ 2024-02-26 11:56:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
229
+ 2024-02-26 11:57:01,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
230
+ 2024-02-26 11:57:06,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
231
+ 2024-02-26 11:57:11,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
232
+ 2024-02-26 11:57:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
233
+ 2024-02-26 11:57:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
234
+ 2024-02-26 11:57:12,252 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
235
+ 2024-02-26 11:57:17,324 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
236
+ 2024-02-26 11:57:22,325 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
237
+ 2024-02-26 11:57:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
238
+ 2024-02-26 11:57:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
239
+ 2024-02-26 11:57:27,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
240
+ 2024-02-26 11:57:28,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
241
+ 2024-02-26 11:57:33,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
242
+ 2024-02-26 11:57:38,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
243
+ 2024-02-26 11:57:42,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
244
+ 2024-02-26 11:57:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
245
+ 2024-02-26 11:57:42,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
246
+ 2024-02-26 11:57:44,282 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
247
+ 2024-02-26 11:57:49,283 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
248
+ 2024-02-26 11:57:54,284 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
249
+ 2024-02-26 11:57:57,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
250
+ 2024-02-26 11:57:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
251
+ 2024-02-26 11:57:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
252
+ 2024-02-26 11:57:59,314 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
253
+ 2024-02-26 11:58:04,315 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
254
+ 2024-02-26 11:58:09,316 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
255
+ 2024-02-26 11:58:12,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
256
+ 2024-02-26 11:58:12,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
257
+ 2024-02-26 11:58:12,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
258
+ 2024-02-26 11:58:16,193 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
259
+ 2024-02-26 11:58:21,194 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
260
+ 2024-02-26 11:58:26,196 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
261
+ 2024-02-26 11:58:27,252 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
262
+ 2024-02-26 11:58:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
263
+ 2024-02-26 11:58:27,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
264
+ 2024-02-26 11:58:31,358 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
265
+ 2024-02-26 11:58:36,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
266
+ 2024-02-26 11:58:41,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
267
+ 2024-02-26 11:58:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
268
+ 2024-02-26 11:58:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
269
+ 2024-02-26 11:58:42,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
270
+ 2024-02-26 11:58:47,292 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
271
+ 2024-02-26 11:58:52,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
272
+ 2024-02-26 11:58:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
273
+ 2024-02-26 11:58:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
274
+ 2024-02-26 11:58:57,253 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
275
+ 2024-02-26 11:58:57,349 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
276
+ 2024-02-26 11:59:02,350 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
277
+ 2024-02-26 11:59:07,351 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
278
+ 2024-02-26 11:59:12,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
279
+ 2024-02-26 11:59:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
280
+ 2024-02-26 11:59:12,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
281
+ 2024-02-26 11:59:13,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
282
+ 2024-02-26 11:59:18,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
283
+ 2024-02-26 11:59:23,337 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
284
+ 2024-02-26 11:59:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
285
+ 2024-02-26 11:59:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
286
+ 2024-02-26 11:59:27,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
287
+ 2024-02-26 11:59:28,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
288
+ 2024-02-26 11:59:33,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
289
+ 2024-02-26 11:59:38,377 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
290
+ 2024-02-26 11:59:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
291
+ 2024-02-26 11:59:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
292
+ 2024-02-26 11:59:42,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
293
+ 2024-02-26 11:59:44,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
294
+ 2024-02-26 11:59:49,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
295
+ 2024-02-26 11:59:54,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
296
+ 2024-02-26 11:59:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
297
+ 2024-02-26 11:59:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
298
+ 2024-02-26 11:59:57,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
299
+ 2024-02-26 11:59:59,331 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
300
+ 2024-02-26 12:00:04,332 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
301
+ 2024-02-26 12:00:09,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
302
+ 2024-02-26 12:00:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
303
+ 2024-02-26 12:00:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
304
+ 2024-02-26 12:00:12,295 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
305
+ 2024-02-26 12:00:15,324 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
306
+ 2024-02-26 12:00:20,325 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
307
+ 2024-02-26 12:00:25,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
308
+ 2024-02-26 12:00:27,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
309
+ 2024-02-26 12:00:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
310
+ 2024-02-26 12:00:27,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
311
+ 2024-02-26 12:00:30,345 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
312
+ 2024-02-26 12:00:35,346 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
313
+ 2024-02-26 12:00:40,347 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
314
+ 2024-02-26 12:00:42,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
315
+ 2024-02-26 12:00:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
316
+ 2024-02-26 12:00:42,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
317
+ 2024-02-26 12:00:45,389 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
318
+ 2024-02-26 12:00:50,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
319
+ 2024-02-26 12:00:55,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
320
+ 2024-02-26 12:00:57,253 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
321
+ 2024-02-26 12:00:57,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
322
+ 2024-02-26 12:00:57,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
323
+ 2024-02-26 12:01:01,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
324
+ 2024-02-26 12:01:06,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
325
+ 2024-02-26 12:01:11,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
326
+ 2024-02-26 12:01:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
327
+ 2024-02-26 12:01:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
328
+ 2024-02-26 12:01:12,254 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
329
+ 2024-02-26 12:01:17,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
330
+ 2024-02-26 12:01:22,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
331
+ 2024-02-26 12:01:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
332
+ 2024-02-26 12:01:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
333
+ 2024-02-26 12:01:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
334
+ 2024-02-26 12:01:27,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
335
+ 2024-02-26 12:01:32,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
336
+ 2024-02-26 12:01:37,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
337
+ 2024-02-26 12:01:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
338
+ 2024-02-26 12:01:42,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
339
+ 2024-02-26 12:01:42,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
340
+ 2024-02-26 12:01:43,316 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
341
+ 2024-02-26 12:01:48,317 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
342
+ 2024-02-26 12:01:53,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
343
+ 2024-02-26 12:01:57,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
344
+ 2024-02-26 12:01:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
345
+ 2024-02-26 12:01:57,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
346
+ 2024-02-26 12:01:58,366 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
347
+ 2024-02-26 12:02:03,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
348
+ 2024-02-26 12:02:08,367 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
349
+ 2024-02-26 12:02:12,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
350
+ 2024-02-26 12:02:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
351
+ 2024-02-26 12:02:12,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
352
+ 2024-02-26 12:02:14,353 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
353
+ 2024-02-26 12:02:19,354 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
354
+ 2024-02-26 12:02:24,355 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
355
+ 2024-02-26 12:02:27,254 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
356
+ 2024-02-26 12:02:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
357
+ 2024-02-26 12:02:27,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
358
+ 2024-02-26 12:02:30,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
359
+ 2024-02-26 12:02:35,293 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
360
+ 2024-02-26 12:02:40,294 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
361
+ 2024-02-26 12:02:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
362
+ 2024-02-26 12:02:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
363
+ 2024-02-26 12:02:42,255 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
364
+ 2024-02-26 12:02:45,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
365
+ 2024-02-26 12:02:50,364 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
366
+ 2024-02-26 12:02:55,364 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
367
+ 2024-02-26 12:02:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
368
+ 2024-02-26 12:02:57,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
369
+ 2024-02-26 12:02:57,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
370
+ 2024-02-26 12:03:01,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
371
+ 2024-02-26 12:03:06,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
372
+ 2024-02-26 12:03:11,335 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
373
+ 2024-02-26 12:03:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
374
+ 2024-02-26 12:03:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
375
+ 2024-02-26 12:03:12,295 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
376
+ 2024-02-26 12:03:16,991 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
377
+ 2024-02-26 12:03:21,992 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
378
+ 2024-02-26 12:03:26,993 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
379
+ 2024-02-26 12:03:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
380
+ 2024-02-26 12:03:27,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
381
+ 2024-02-26 12:03:27,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
382
+ 2024-02-26 12:03:32,341 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
383
+ 2024-02-26 12:03:37,342 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
384
+ 2024-02-26 12:03:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
385
+ 2024-02-26 12:03:42,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
386
+ 2024-02-26 12:03:42,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
387
+ 2024-02-26 12:03:43,301 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
388
+ 2024-02-26 12:03:48,302 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
389
+ 2024-02-26 12:03:53,303 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
390
+ 2024-02-26 12:03:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
391
+ 2024-02-26 12:03:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
392
+ 2024-02-26 12:03:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
393
+ 2024-02-26 12:03:58,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
394
+ 2024-02-26 12:04:03,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
395
+ 2024-02-26 12:04:08,329 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
396
+ 2024-02-26 12:04:12,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
397
+ 2024-02-26 12:04:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
398
+ 2024-02-26 12:04:12,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
399
+ 2024-02-26 12:04:13,386 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
400
+ 2024-02-26 12:04:18,387 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
401
+ 2024-02-26 12:04:23,388 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
402
+ 2024-02-26 12:04:27,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
403
+ 2024-02-26 12:04:27,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
404
+ 2024-02-26 12:04:27,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
405
+ 2024-02-26 12:04:29,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
406
+ 2024-02-26 12:04:34,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
407
+ 2024-02-26 12:04:39,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
408
+ 2024-02-26 12:04:42,255 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
409
+ 2024-02-26 12:04:42,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
410
+ 2024-02-26 12:04:42,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
411
+ 2024-02-26 12:04:45,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
412
+ 2024-02-26 12:04:50,305 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
413
+ 2024-02-26 12:04:55,306 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
414
+ 2024-02-26 12:04:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
415
+ 2024-02-26 12:04:57,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
416
+ 2024-02-26 12:04:57,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
417
+ 2024-02-26 12:05:00,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
418
+ 2024-02-26 12:05:05,362 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
419
+ 2024-02-26 12:05:10,363 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
420
+ 2024-02-26 12:05:12,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
421
+ 2024-02-26 12:05:12,256 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
422
+ 2024-02-26 12:05:12,296 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
423
+ 2024-02-26 12:05:16,334 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
424
+ 2024-02-26 12:05:21,335 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
425
+ 2024-02-26 12:05:26,336 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
426
+ 2024-02-26 12:05:27,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
427
+ 2024-02-26 12:05:27,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
428
+ 2024-02-26 12:05:27,289 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
429
+ 2024-02-26 12:05:32,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
430
+ 2024-02-26 12:05:37,290 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
431
+ 2024-02-26 12:05:42,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
432
+ 2024-02-26 12:05:42,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
433
+ 2024-02-26 12:05:42,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
434
+ 2024-02-26 12:05:42,359 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
435
+ 2024-02-26 12:05:47,360 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
436
+ 2024-02-26 12:05:52,361 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
437
+ 2024-02-26 12:05:57,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
438
+ 2024-02-26 12:05:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
439
+ 2024-02-26 12:05:57,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
440
+ 2024-02-26 12:05:58,317 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
441
+ 2024-02-26 12:06:03,318 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
442
+ 2024-02-26 12:06:08,319 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
443
+ 2024-02-26 12:06:12,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
444
+ 2024-02-26 12:06:12,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
445
+ 2024-02-26 12:06:12,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
446
+ 2024-02-26 12:06:13,353 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
447
+ 2024-02-26 12:06:18,354 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
448
+ 2024-02-26 12:06:23,355 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
449
+ 2024-02-26 12:06:27,256 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
450
+ 2024-02-26 12:06:27,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
451
+ 2024-02-26 12:06:27,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
452
+ 2024-02-26 12:06:28,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
453
+ 2024-02-26 12:06:33,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
454
+ 2024-02-26 12:06:38,392 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
455
+ 2024-02-26 12:06:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
456
+ 2024-02-26 12:06:42,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
457
+ 2024-02-26 12:06:42,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
458
+ 2024-02-26 12:06:44,303 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
459
+ 2024-02-26 12:06:49,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
460
+ 2024-02-26 12:06:54,304 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
461
+ 2024-02-26 12:06:57,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
462
+ 2024-02-26 12:06:57,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
463
+ 2024-02-26 12:06:57,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
464
+ 2024-02-26 12:06:59,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
465
+ 2024-02-26 12:07:04,327 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
466
+ 2024-02-26 12:07:09,328 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
467
+ 2024-02-26 12:07:12,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
468
+ 2024-02-26 12:07:12,257 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
469
+ 2024-02-26 12:07:12,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
470
+ 2024-02-26 12:07:14,372 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
471
+ 2024-02-26 12:07:19,373 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
472
+ 2024-02-26 12:07:24,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
473
+ 2024-02-26 12:07:27,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
474
+ 2024-02-26 12:07:27,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
475
+ 2024-02-26 12:07:27,297 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
476
+ 2024-02-26 12:07:30,369 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
477
+ 2024-02-26 12:07:35,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
478
+ 2024-02-26 12:07:40,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
479
+ 2024-02-26 12:07:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
480
+ 2024-02-26 12:07:42,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
481
+ 2024-02-26 12:07:42,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
482
+ 2024-02-26 12:07:46,370 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
483
+ 2024-02-26 12:07:51,371 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
484
+ 2024-02-26 12:07:56,372 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
485
+ 2024-02-26 12:07:57,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
486
+ 2024-02-26 12:07:57,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
487
+ 2024-02-26 12:07:57,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
488
+ 2024-02-26 12:08:01,390 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
489
+ 2024-02-26 12:08:06,391 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
490
+ 2024-02-26 12:08:11,393 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
491
+ 2024-02-26 12:08:12,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
492
+ 2024-02-26 12:08:12,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
493
+ 2024-02-26 12:08:12,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
494
+ 2024-02-26 12:08:17,340 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
495
+ 2024-02-26 12:08:22,341 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
496
+ 2024-02-26 12:08:27,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
497
+ 2024-02-26 12:08:27,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
498
+ 2024-02-26 12:08:27,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
499
+ 2024-02-26 12:08:28,300 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
500
+ 2024-02-26 12:08:33,300 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
501
+ 2024-02-26 12:08:38,301 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
502
+ 2024-02-26 12:08:42,257 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
503
+ 2024-02-26 12:08:42,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
504
+ 2024-02-26 12:08:42,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
505
+ 2024-02-26 12:08:43,311 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
506
+ 2024-02-26 12:08:48,312 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
507
+ 2024-02-26 12:08:53,313 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
508
+ 2024-02-26 12:08:57,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
509
+ 2024-02-26 12:08:57,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
510
+ 2024-02-26 12:08:57,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
511
+ 2024-02-26 12:08:58,374 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
512
+ 2024-02-26 12:09:03,375 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
513
+ 2024-02-26 12:09:08,376 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
514
+ 2024-02-26 12:09:12,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
515
+ 2024-02-26 12:09:12,258 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
516
+ 2024-02-26 12:09:12,298 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
517
+ 2024-02-26 12:09:13,393 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
518
+ 2024-02-26 12:09:18,394 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
519
+ 2024-02-26 12:09:19,077 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
520
+ 2024-02-26 12:09:19,080 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
521
+ 2024-02-26 12:09:19,080 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
522
+ 2024-02-26 12:09:19,081 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
523
+ 2024-02-26 12:09:19,088 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
524
+ 2024-02-26 12:09:19,092 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
525
+ 2024-02-26 12:09:19,101 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
526
+ 2024-02-26 12:09:19,106 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
527
+ 2024-02-26 12:09:19,106 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
528
+ 2024-02-26 12:09:23,394 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
529
+ 2024-02-26 12:09:27,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
530
+ 2024-02-26 12:09:27,259 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
531
+ 2024-02-26 12:09:27,259 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
532
+ 2024-02-26 12:09:29,333 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
533
+ 2024-02-26 12:09:30,621 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: pause
534
+ 2024-02-26 12:09:30,621 INFO HandlerThread:8773 [handler.py:handle_request_pause():708] stopping system metrics thread
535
+ 2024-02-26 12:09:30,622 INFO HandlerThread:8773 [system_monitor.py:finish():203] Stopping system monitor
536
+ 2024-02-26 12:09:30,624 DEBUG SystemMonitor:8773 [system_monitor.py:_start():172] Starting system metrics aggregation loop
537
+ 2024-02-26 12:09:30,626 DEBUG SystemMonitor:8773 [system_monitor.py:_start():179] Finished system metrics aggregation loop
538
+ 2024-02-26 12:09:30,626 DEBUG SystemMonitor:8773 [system_monitor.py:_start():183] Publishing last batch of metrics
539
+ 2024-02-26 12:09:30,624 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined cpu monitor
540
+ 2024-02-26 12:09:30,632 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined disk monitor
541
+ 2024-02-26 12:09:30,642 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined gpu monitor
542
+ 2024-02-26 12:09:30,642 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined memory monitor
543
+ 2024-02-26 12:09:30,643 INFO HandlerThread:8773 [interfaces.py:finish():202] Joined network monitor
544
+ 2024-02-26 12:09:30,644 DEBUG SenderThread:8773 [sender.py:send():382] send: stats
545
+ 2024-02-26 12:09:34,646 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
546
+ 2024-02-26 12:09:39,647 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
547
+ 2024-02-26 12:09:42,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
548
+ 2024-02-26 12:09:42,259 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
549
+ 2024-02-26 12:09:42,259 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
550
+ 2024-02-26 12:09:45,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
551
+ 2024-02-26 12:09:50,326 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
552
+ 2024-02-26 12:09:55,327 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
553
+ 2024-02-26 12:09:56,160 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: resume
554
+ 2024-02-26 12:09:56,161 INFO HandlerThread:8773 [handler.py:handle_request_resume():699] starting system metrics thread
555
+ 2024-02-26 12:09:56,161 INFO HandlerThread:8773 [system_monitor.py:start():194] Starting system monitor
556
+ 2024-02-26 12:09:56,164 INFO SystemMonitor:8773 [system_monitor.py:_start():158] Starting system asset monitoring threads
557
+ 2024-02-26 12:09:56,164 INFO SystemMonitor:8773 [interfaces.py:start():190] Started cpu monitoring
558
+ 2024-02-26 12:09:56,168 INFO SystemMonitor:8773 [interfaces.py:start():190] Started disk monitoring
559
+ 2024-02-26 12:09:56,169 INFO SystemMonitor:8773 [interfaces.py:start():190] Started gpu monitoring
560
+ 2024-02-26 12:09:56,175 INFO SystemMonitor:8773 [interfaces.py:start():190] Started memory monitoring
561
+ 2024-02-26 12:09:56,176 INFO SystemMonitor:8773 [interfaces.py:start():190] Started network monitoring
562
+ 2024-02-26 12:09:57,258 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: stop_status
563
+ 2024-02-26 12:09:57,781 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: internal_messages
564
+ 2024-02-26 12:09:57,782 DEBUG SenderThread:8773 [sender.py:send_request():409] send_request: stop_status
565
+ 2024-02-26 12:10:00,830 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
566
+ 2024-02-26 12:10:05,832 DEBUG HandlerThread:8773 [handler.py:handle_request():146] handle_request: status_report
wandb/run-20240226_115124-aiyp9xfp/logs/debug.log ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2024-02-26 11:51:24,250 INFO MainThread:191 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
2
+ 2024-02-26 11:51:24,253 INFO MainThread:191 [wandb_setup.py:_flush():76] Configure stats pid to 191
3
+ 2024-02-26 11:51:24,254 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
4
+ 2024-02-26 11:51:24,255 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/settings
5
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
6
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
7
+ 2024-02-26 11:51:24,256 INFO MainThread:191 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': '<python with no main file>'}
8
+ 2024-02-26 11:51:24,257 INFO MainThread:191 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
9
+ 2024-02-26 11:51:24,257 INFO MainThread:191 [wandb_init.py:_log_setup():526] Logging user logs to /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/logs/debug.log
10
+ 2024-02-26 11:51:24,258 INFO MainThread:191 [wandb_init.py:_log_setup():527] Logging internal logs to /content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/wandb/run-20240226_115124-aiyp9xfp/logs/debug-internal.log
11
+ 2024-02-26 11:51:24,258 INFO MainThread:191 [wandb_init.py:_jupyter_setup():472] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7882d69a6a10>
12
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():566] calling init triggers
13
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
14
+ config: {}
15
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():616] starting backend
16
+ 2024-02-26 11:51:24,259 INFO MainThread:191 [wandb_init.py:init():620] setting up manager
17
+ 2024-02-26 11:51:24,264 INFO MainThread:191 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
18
+ 2024-02-26 11:51:24,266 INFO MainThread:191 [wandb_init.py:init():628] backend started and connected
19
+ 2024-02-26 11:51:24,296 INFO MainThread:191 [wandb_run.py:_label_probe_notebook():1295] probe notebook
20
+ 2024-02-26 11:51:26,743 INFO MainThread:191 [wandb_init.py:init():720] updated telemetry
21
+ 2024-02-26 11:51:26,755 INFO MainThread:191 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
22
+ 2024-02-26 11:51:27,024 INFO MainThread:191 [wandb_run.py:_on_init():2262] communicating current version
23
+ 2024-02-26 11:51:27,146 INFO MainThread:191 [wandb_run.py:_on_init():2271] got version response
24
+ 2024-02-26 11:51:27,146 INFO MainThread:191 [wandb_init.py:init():804] starting run threads in backend
25
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_console_start():2241] atexit reg
26
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_redirect():2096] redirect: wrap_raw
27
+ 2024-02-26 11:51:27,248 INFO MainThread:191 [wandb_run.py:_redirect():2161] Wrapping output streams.
28
+ 2024-02-26 11:51:27,249 INFO MainThread:191 [wandb_run.py:_redirect():2186] Redirects installed.
29
+ 2024-02-26 11:51:27,251 INFO MainThread:191 [wandb_init.py:init():847] run started, returning control to user process
30
+ 2024-02-26 11:51:27,258 INFO MainThread:191 [wandb_run.py:_config_callback():1343] config_cb None None {'vocab_size': 256000, 'max_position_embeddings': 8192, 'hidden_size': 3072, 'intermediate_size': 24576, 'num_hidden_layers': 28, 'num_attention_heads': 16, 'head_dim': 256, 'num_key_value_heads': 16, 'hidden_act': 'gelu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 10000.0, 'attention_bias': False, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GemmaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 2, 'pad_token_id': 0, 'eos_token_id': 1, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'google/gemma-7b', 'transformers_version': '4.38.1', 'model_type': 'gemma', 'rope_scaling': None, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'bnb_4bit_compute_dtype': 'bfloat16', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0002, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 3.0, 'max_steps': 60, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko/runs/Feb26_11-29-58_336d6f2eb0af', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 10, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': False, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/content/gdrive/MyDrive/LLM/Gemma-7B-Finetuning-JCS-Ko', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': True, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}
31
+ 2024-02-26 11:51:38,564 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
32
+ 2024-02-26 11:51:38,565 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
33
+ 2024-02-26 11:52:19,910 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
34
+ 2024-02-26 11:52:29,234 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
35
+ 2024-02-26 11:52:29,234 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
36
+ 2024-02-26 11:53:30,386 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
37
+ 2024-02-26 11:53:43,585 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
38
+ 2024-02-26 11:53:43,585 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
39
+ 2024-02-26 12:09:19,076 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
40
+ 2024-02-26 12:09:30,620 INFO MainThread:191 [jupyter.py:save_ipynb():373] not saving jupyter notebook
41
+ 2024-02-26 12:09:30,620 INFO MainThread:191 [wandb_init.py:_pause_backend():437] pausing backend
42
+ 2024-02-26 12:09:56,159 INFO MainThread:191 [wandb_init.py:_resume_backend():442] resuming backend
wandb/run-20240226_115124-aiyp9xfp/run-aiyp9xfp.wandb ADDED
File without changes