diff --git "a/slurm.out" "b/slurm.out"
new file mode 100644--- /dev/null
+++ "b/slurm.out"
@@ -0,0 +1,3005 @@
+1: W1123 14:35:36.265000 2700626 torch/distributed/run.py:792] 
+1: W1123 14:35:36.265000 2700626 torch/distributed/run.py:792] *****************************************
+1: W1123 14:35:36.265000 2700626 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+1: W1123 14:35:36.265000 2700626 torch/distributed/run.py:792] *****************************************
+2: W1123 14:35:36.265000 1476038 torch/distributed/run.py:792] 
+2: W1123 14:35:36.265000 1476038 torch/distributed/run.py:792] *****************************************
+2: W1123 14:35:36.265000 1476038 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+2: W1123 14:35:36.265000 1476038 torch/distributed/run.py:792] *****************************************
+0: W1123 14:35:36.265000 3958233 torch/distributed/run.py:792] 
+0: W1123 14:35:36.265000 3958233 torch/distributed/run.py:792] *****************************************
+0: W1123 14:35:36.265000 3958233 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+0: W1123 14:35:36.265000 3958233 torch/distributed/run.py:792] *****************************************
+3: W1123 14:35:36.265000 1471917 torch/distributed/run.py:792] 
+3: W1123 14:35:36.265000 1471917 torch/distributed/run.py:792] *****************************************
+3: W1123 14:35:36.265000 1471917 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+3: W1123 14:35:36.265000 1471917 torch/distributed/run.py:792] *****************************************
+0: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:3958317] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+0: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:3958317] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+2: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:1476125] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+1: [2025-11-23 14:40:32,705] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:2700705] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+3: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.check_eval_packing:119] [PID:1472009] [RANK:0] explicitly setting `eval_sample_packing` to match `sample_packing`
+2: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:1476125] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+1: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:2700705] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+3: [2025-11-23 14:40:32,706] [INFO] [axolotl.utils.schemas.validation.hint_sample_packing_padding:218] [PID:1472009] [RANK:0] Setting `pad_to_sequence_len: true` to prevent memory leaks when sample_packing
+0: [2025-11-23 14:41:03,102] [WARNING] [axolotl.utils.config.normalize_config:139] [PID:3958317] [RANK:0] Invalid value for save_steps (1.6666666666666667) from saves_per_epoch and/or num_epochs. Saving at training end only.
+0: [2025-11-23 14:41:03,272] [INFO] [axolotl.cli.config.load_cfg:245] [PID:3958317] [RANK:0] config:
+0: {
+0:   "activation_offloading": false,
+0:   "auto_resume_from_checkpoints": true,
+0:   "axolotl_config_path": "/lustre/fswork/projects/rech/dgo/udv55np/train/tmp/1763904854646355858.yaml",
+0:   "base_model": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "base_model_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "batch_size": 16,
+0:   "bf16": true,
+0:   "capabilities": {
+0:     "bf16": true,
+0:     "compute_capability": "sm_90",
+0:     "fp8": false,
+0:     "n_gpu": 16,
+0:     "n_node": 1
+0:   },
+0:   "chat_template": "gemma3",
+0:   "context_parallel_size": 1,
+0:   "curriculum_sampling": true,
+0:   "dataloader_num_workers": 2,
+0:   "dataset_prepared_path": "/lustre/fswork/projects/rech/dgo/udv55np/dataset_gemma/Nemotron-Super-49B-v1_5/split_0.5",
+0:   "dataset_processes": 32,
+0:   "datasets": [
+0:     {
+0:       "chat_template": "tokenizer_default",
+0:       "data_files": [
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0001.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0002.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0013.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0015.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0004.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0011.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0000.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking/0003.jsonl"
+0:       ],
+0:       "ds_type": "json",
+0:       "field_messages": "conversations",
+0:       "message_property_mappings": {
+0:         "content": "content",
+0:         "role": "role"
+0:       },
+0:       "path": "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking",
+0:       "trust_remote_code": false,
+0:       "type": "chat_template"
+0:     },
+0:     {
+0:       "chat_template": "tokenizer_default",
+0:       "data_files": [
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0007.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0009.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0005.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0006.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0014.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0010.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0012.jsonl",
+0:         "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking/0008.jsonl"
+0:       ],
+0:       "ds_type": "json",
+0:       "field_messages": "conversations",
+0:       "message_property_mappings": {
+0:         "content": "content",
+0:         "role": "role"
+0:       },
+0:       "path": "/lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking",
+0:       "trust_remote_code": false,
+0:       "type": "chat_template"
+0:     }
+0:   ],
+0:   "ddp": true,
+0:   "deepspeed": {
+0:     "bf16": {
+0:       "enabled": true
+0:     },
+0:     "gradient_accumulation_steps": "auto",
+0:     "gradient_clipping": "auto",
+0:     "train_batch_size": "auto",
+0:     "train_micro_batch_size_per_gpu": "auto",
+0:     "wall_clock_breakdown": false,
+0:     "zero_optimization": {
+0:       "contiguous_gradients": true,
+0:       "overlap_comm": true,
+0:       "reduce_bucket_size": "auto",
+0:       "stage": 3,
+0:       "stage3_gather_16bit_weights_on_model_save": true,
+0:       "stage3_param_persistence_threshold": "auto",
+0:       "stage3_prefetch_bucket_size": "auto",
+0:       "sub_group_size": 0
+0:     }
+0:   },
+0:   "device": "cuda:0",
+0:   "device_map": {
+0:     "": 0
+0:   },
+0:   "dion_rank_fraction": 1.0,
+0:   "dion_rank_multiple_of": 1,
+0:   "env_capabilities": {
+0:     "torch_version": "2.6.0"
+0:   },
+0:   "eot_tokens": [
+0:     "<end_of_turn>"
+0:   ],
+0:   "eval_batch_size": 1,
+0:   "eval_causal_lm_metrics": [
+0:     "sacrebleu",
+0:     "comet",
+0:     "ter",
+0:     "chrf"
+0:   ],
+0:   "eval_max_new_tokens": 128,
+0:   "eval_sample_packing": true,
+0:   "eval_table_size": 0,
+0:   "evals_per_epoch": 0,
+0:   "flash_attention": true,
+0:   "fp16": false,
+0:   "gradient_accumulation_steps": 1,
+0:   "gradient_checkpointing": true,
+0:   "gradient_checkpointing_kwargs": {
+0:     "use_reentrant": true
+0:   },
+0:   "is_multimodal": true,
+0:   "learning_rate": 5e-06,
+0:   "lisa_layers_attribute": "model.layers",
+0:   "load_best_model_at_end": false,
+0:   "load_in_4bit": false,
+0:   "load_in_8bit": false,
+0:   "local_rank": 0,
+0:   "logging_steps": 10,
+0:   "lora_dropout": 0.0,
+0:   "loraplus_lr_embedding": 1e-06,
+0:   "lr_scheduler": "warmup_stable_decay",
+0:   "lr_scheduler_kwargs": {
+0:     "min_lr_ratio": 0.1,
+0:     "num_decay_steps": 200
+0:   },
+0:   "max_prompt_len": 512,
+0:   "mean_resizing_embeddings": false,
+0:   "micro_batch_size": 1,
+0:   "model_config_type": "gemma3",
+0:   "num_epochs": 0.6,
+0:   "optimizer": "adamw_torch_fused",
+0:   "output_dir": "/lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5",
+0:   "pad_to_sequence_len": true,
+0:   "pretrain_multipack_attn": true,
+0:   "pretrain_multipack_buffer_size": 10000,
+0:   "processor_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-4b",
+0:   "profiler_steps_start": 0,
+0:   "qlora_sharded_model_loading": false,
+0:   "ray_num_workers": 1,
+0:   "resources_per_worker": {
+0:     "GPU": 1
+0:   },
+0:   "sample_packing": true,
+0:   "sample_packing_bin_size": 200,
+0:   "sample_packing_group_size": 100000,
+0:   "sample_packing_sequentially": true,
+0:   "save_only_model": true,
+0:   "save_safetensors": true,
+0:   "save_total_limit": 20,
+0:   "saves_per_epoch": 1,
+0:   "sequence_len": 16384,
+0:   "shuffle_before_merging_datasets": true,
+0:   "shuffle_merged_datasets": false,
+0:   "skip_prepare_dataset": false,
+0:   "strict": false,
+0:   "tensor_parallel_size": 1,
+0:   "tf32": false,
+0:   "tiled_mlp_use_original_mlp": true,
+0:   "tokenizer_config": "/lustre/fswork/projects/rech/qwv/udv55np/Gemma/base/gemma-3-27b",
+0:   "torch_dtype": "torch.bfloat16",
+0:   "train_on_inputs": false,
+0:   "trl": {
+0:     "log_completions": false,
+0:     "mask_truncated_completions": false,
+0:     "ref_model_mixup_alpha": 0.9,
+0:     "ref_model_sync_steps": 64,
+0:     "scale_rewards": true,
+0:     "sync_ref_model": false,
+0:     "use_vllm": false,
+0:     "vllm_server_host": "0.0.0.0",
+0:     "vllm_server_port": 8000
+0:   },
+0:   "use_ray": false,
+0:   "use_tensorboard": true,
+0:   "val_set_size": 0.0,
+0:   "vllm": {
+0:     "device": "auto",
+0:     "dtype": "auto",
+0:     "gpu_memory_utilization": 0.9,
+0:     "host": "0.0.0.0",
+0:     "port": 8000
+0:   },
+0:   "warmup_steps": 100,
+0:   "weight_decay": 0.0,
+0:   "world_size": 16
+0: }
+0: [2025-11-23 14:41:03,273] [INFO] [axolotl.cli.checks.check_user_token:35] [PID:3958317] [RANK:0] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
+2: [2025-11-23 14:41:05,147] [INFO] [axolotl.utils.data.sft._load_raw_datasets:314] [PID:1476128] [RANK:3] Loading raw datasets...
+2: 
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 4638 examples [00:00, 12849.35 examples/s]
Generating train split: 9310 examples [00:00, 21307.04 examples/s]
Generating train split: 28012 examples [00:00, 60097.01 examples/s]
Generating train split: 39449 examples [00:00, 64467.45 examples/s]
Generating train split: 53392 examples [00:00, 69259.07 examples/s]
Generating train split: 69660 examples [00:01, 85958.64 examples/s]
Generating train split: 88447 examples [00:01, 80181.70 examples/s]
Generating train split: 109184 examples [00:01, 85361.85 examples/s]
Generating train split: 123087 examples [00:01, 86829.67 examples/s]
Generating train split: 139318 examples [00:01, 97144.19 examples/s]
Generating train split: 157992 examples [00:02, 92776.98 examples/s]
Generating train split: 178803 examples [00:02, 99095.05 examples/s]
Generating train split: 192677 examples [00:02, 98292.92 examples/s]
Generating train split: 208978 examples [00:02, 106980.59 examples/s]
Generati
+2: ng train split: 227918 examples [00:02, 91560.59 examples/s] 
Generating train split: 241725 examples [00:02, 95407.49 examples/s]
Generating train split: 253114 examples [00:03, 81285.99 examples/s]
Generating train split: 271907 examples [00:03, 93881.27 examples/s]
Generating train split: 278638 examples [00:03, 84392.86 examples/s]
+2: [2025-11-23 14:41:10,911] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:88] [PID:1476128] [RANK:3] Loading dataset: /lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/no_thinking with base_type: chat_template and prompt_style: None
+2: 
Tokenizing Prompts (num_proc=32):   0%|          | 0/278638 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=32):   0%|          | 1000/278638 [00:04<18:53, 245.01 examples/s]
Tokenizing Prompts (num_proc=32):   1%|          | 2000/278638 [00:04<08:28, 543.90 examples/s]
Tokenizing Prompts (num_proc=32):   1%|          | 3000/278638 [00:04<05:08, 894.11 examples/s]
Tokenizing Prompts (num_proc=32):   1%|▏         | 4000/278638 [00:04<03:34, 1279.25 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 5000/278638 [00:05<02:41, 1699.60 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 6000/278638 [00:05<02:09, 2108.26 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 7000/278638 [00:05<01:48, 2507.91 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 8000/278638 [00:05<01:37, 2770.51 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 9000/278638 [00:06<01:26, 3128.81 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▎         | 1000
+2: 0/278638 [00:06<01:20, 3323.61 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 11000/278638 [00:06<01:17, 3448.23 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 12000/278638 [00:06<01:15, 3535.26 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 14000/278638 [00:07<00:56, 4717.80 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 15000/278638 [00:07<01:00, 4387.58 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▋         | 18000/278638 [00:07<00:40, 6386.97 examples/s]
Tokenizing Prompts (num_proc=32):   7%|▋         | 19000/278638 [00:08<00:46, 5530.43 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 21000/278638 [00:08<00:42, 6055.15 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 23000/278638 [00:08<00:39, 6412.80 examples/s]
Tokenizing Prompts (num_proc=32):  10%|▉         | 27000/278638 [00:08<00:29, 8497.60 examples/s]
Tokenizing Prompts (num_proc=32):  10%|█         | 29000/278638 [00:08<00:25, 9
+2: 896.74 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█         | 31000/278638 [00:09<00:26, 9402.37 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 33000/278638 [00:09<00:32, 7671.98 examples/s]
Tokenizing Prompts (num_proc=32):  13%|█▎        | 35000/278638 [00:09<00:30, 7902.29 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▍        | 39000/278638 [00:09<00:20, 11674.30 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▍        | 41000/278638 [00:10<00:20, 11666.23 examples/s]
Tokenizing Prompts (num_proc=32):  16%|█▌        | 44000/278638 [00:10<00:23, 9988.83 examples/s] 
Tokenizing Prompts (num_proc=32):  17%|█▋        | 46000/278638 [00:10<00:20, 11297.09 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 48000/278638 [00:10<00:23, 9980.45 examples/s] 
Tokenizing Prompts (num_proc=32):  18%|█▊        | 50000/278638 [00:11<00:25, 8902.40 examples/s]
Tokenizing Prompts (num_proc=32):  19%|█▉        | 53000/278638 [00:11<00:21, 10
+2: 523.45 examples/s]
Tokenizing Prompts (num_proc=32):  20%|█▉        | 55000/278638 [00:11<00:18, 11775.72 examples/s]
Tokenizing Prompts (num_proc=32):  20%|██        | 57000/278638 [00:11<00:20, 10889.98 examples/s]
Tokenizing Prompts (num_proc=32):  21%|██        | 59000/278638 [00:11<00:19, 11162.43 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 61000/278638 [00:12<00:18, 11926.21 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 63000/278638 [00:12<00:18, 11923.90 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 65000/278638 [00:12<00:18, 11323.07 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▍       | 67000/278638 [00:12<00:17, 12315.11 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▍       | 69000/278638 [00:12<00:16, 12623.81 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▌       | 71000/278638 [00:12<00:15, 13313.13 examples/s]
Tokenizing Prompts (num_proc=32):  27%|██▋       | 76000/278
+2: 638 [00:13<00:19, 10205.31 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 79000/278638 [00:13<00:15, 12683.59 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▉       | 81000/278638 [00:13<00:17, 11411.53 examples/s]
Tokenizing Prompts (num_proc=32):  30%|██▉       | 83000/278638 [00:14<00:19, 10144.70 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███       | 85000/278638 [00:14<00:17, 10929.46 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███       | 87000/278638 [00:14<00:15, 12345.37 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 89000/278638 [00:14<00:14, 12746.70 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 91000/278638 [00:14<00:16, 11576.82 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 93000/278638 [00:14<00:16, 11544.36 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▍      | 95000/278638 [00:14<00:14, 12902.65 examples/s]
Tokenizing Prompts (num_proc=32)
+2: :  35%|███▍      | 97000/278638 [00:15<00:24, 7322.49 examples/s] 
Tokenizing Prompts (num_proc=32):  36%|███▌      | 100000/278638 [00:15<00:17, 10204.71 examples/s]
Tokenizing Prompts (num_proc=32):  38%|███▊      | 105000/278638 [00:15<00:10, 15908.15 examples/s]
Tokenizing Prompts (num_proc=32):  39%|███▉      | 108000/278638 [00:15<00:12, 13374.01 examples/s]
Tokenizing Prompts (num_proc=32):  39%|███▉      | 110000/278638 [00:16<00:15, 11234.50 examples/s]
Tokenizing Prompts (num_proc=32):  41%|████      | 113000/278638 [00:16<00:13, 12707.37 examples/s]
Tokenizing Prompts (num_proc=32):  41%|████▏     | 115000/278638 [00:16<00:15, 10509.35 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 117000/278638 [00:16<00:16, 9854.66 examples/s] 
Tokenizing Prompts (num_proc=32):  43%|████▎     | 120000/278638 [00:17<00:12, 12334.44 examples/s]
Tokenizing Prompts (num_proc=32):  44%|████▍     | 122000/278638 [00:
+2: 17<00:13, 11233.92 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▍     | 125000/278638 [00:17<00:12, 12797.67 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▌     | 127000/278638 [00:17<00:14, 10415.59 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▋     | 129000/278638 [00:18<00:15, 9892.72 examples/s] 
Tokenizing Prompts (num_proc=32):  47%|████▋     | 131000/278638 [00:18<00:14, 10348.21 examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 134000/278638 [00:18<00:11, 13120.24 examples/s]
Tokenizing Prompts (num_proc=32):  50%|████▉     | 139000/278638 [00:18<00:07, 17990.36 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████     | 141000/278638 [00:18<00:11, 12055.98 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████▏    | 143000/278638 [00:19<00:11, 11572.55 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 145000/278638 [00:19<00:15, 8896.30 examples/s] 
+2: 
Tokenizing Prompts (num_proc=32):  53%|█████▎    | 148000/278638 [00:19<00:12, 10620.08 examples/s]
Tokenizing Prompts (num_proc=32):  54%|█████▍    | 150000/278638 [00:19<00:12, 10518.58 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▍    | 153000/278638 [00:20<00:10, 11992.17 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 156000/278638 [00:20<00:09, 12770.80 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 158000/278638 [00:20<00:10, 11223.80 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 160000/278638 [00:20<00:10, 11388.34 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 162000/278638 [00:20<00:11, 9925.07 examples/s] 
Tokenizing Prompts (num_proc=32):  59%|█████▉    | 165000/278638 [00:21<00:08, 12748.85 examples/s]
Tokenizing Prompts (num_proc=32):  60%|█████▉    | 167000/278638 [00:21<00:08, 13400.86 examples/s]
Tokenizing Prom
+2: pts (num_proc=32):  61%|██████    | 170000/278638 [00:21<00:06, 15949.17 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 172000/278638 [00:21<00:07, 13854.05 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 174000/278638 [00:21<00:07, 13081.76 examples/s]
Tokenizing Prompts (num_proc=32):  63%|██████▎   | 176000/278638 [00:21<00:10, 9854.93 examples/s] 
Tokenizing Prompts (num_proc=32):  64%|██████▍   | 178000/278638 [00:22<00:11, 8798.25 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▍   | 181000/278638 [00:22<00:08, 11495.62 examples/s]
Tokenizing Prompts (num_proc=32):  66%|██████▌   | 184000/278638 [00:22<00:07, 12076.63 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 186000/278638 [00:22<00:07, 12088.23 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 188000/278638 [00:22<00:06, 13364.90 examples/s]
Tokenizing Promp
+2: ts (num_proc=32):  68%|██████▊   | 190000/278638 [00:23<00:07, 11652.07 examples/s]
Tokenizing Prompts (num_proc=32):  69%|██████▉   | 192000/278638 [00:23<00:10, 8350.15 examples/s] 
Tokenizing Prompts (num_proc=32):  70%|██████▉   | 194000/278638 [00:23<00:08, 9560.84 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████   | 197000/278638 [00:23<00:07, 11127.36 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████▏  | 199000/278638 [00:23<00:06, 12382.87 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 202000/278638 [00:24<00:04, 15385.73 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▎  | 205000/278638 [00:24<00:06, 11963.03 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▍  | 207000/278638 [00:24<00:08, 8550.78 examples/s] 
Tokenizing Prompts (num_proc=32):  75%|███████▌  | 209000/278638 [00:25<00:07, 9146.96 examples/s]
Tokeni
+2: zing Prompts (num_proc=32):  76%|███████▌  | 211000/278638 [00:25<00:07, 9554.23 examples/s]
Tokenizing Prompts (num_proc=32):  77%|███████▋  | 214000/278638 [00:25<00:05, 12109.17 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 216000/278638 [00:25<00:04, 12712.08 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 218000/278638 [00:25<00:05, 11648.15 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███████▉  | 220000/278638 [00:25<00:05, 11408.90 examples/s]
Tokenizing Prompts (num_proc=32):  80%|████████  | 223416/278638 [00:26<00:04, 12100.46 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████  | 225124/278638 [00:26<00:04, 11007.36 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 228832/278638 [00:26<00:04, 12123.91 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 231832/278638 [00:26<00:03, 1235
+2: 2.91 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▍ | 234540/278638 [00:27<00:03, 13160.37 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▍ | 236540/278638 [00:27<00:03, 12347.00 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▌ | 237956/278638 [00:27<00:03, 11033.36 examples/s]
Tokenizing Prompts (num_proc=32):  86%|████████▋ | 240664/278638 [00:27<00:04, 8645.28 examples/s] 
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 244080/278638 [00:28<00:03, 11467.76 examples/s]
Tokenizing Prompts (num_proc=32):  89%|████████▊ | 246788/278638 [00:28<00:02, 11856.22 examples/s]
Tokenizing Prompts (num_proc=32):  89%|████████▉ | 248788/278638 [00:28<00:03, 8143.09 examples/s] 
Tokenizing Prompts (num_proc=32):  90%|█████████ | 251204/278638 [00:29<00:03, 7346.70 examples/s]
Tokenizing Prompts (num_proc=32):  91%|████████�
+2: �� | 252912/278638 [00:29<00:03, 8254.78 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████▏| 254912/278638 [00:29<00:02, 8067.63 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 256619/278638 [00:29<00:02, 9145.53 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 258326/278638 [00:29<00:02, 8978.95 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 261326/278638 [00:30<00:01, 8880.09 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 263033/278638 [00:30<00:02, 7081.64 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▌| 264740/278638 [00:30<00:02, 6157.14 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▌| 266740/278638 [00:31<00:01, 7645.09 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 269154/278638 [00:31<00:01, 7375.99 examples/s]
Tokenizing Prompts 
+2: (num_proc=32):  97%|█████████▋| 270568/278638 [00:31<00:01, 7465.72 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 272275/278638 [00:31<00:00, 8784.42 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 273689/278638 [00:31<00:00, 7126.43 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▊| 275103/278638 [00:32<00:00, 6328.08 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 276517/278638 [00:32<00:00, 4792.87 examples/s]
Tokenizing Prompts (num_proc=32): 100%|█████████▉| 277931/278638 [00:32<00:00, 5838.07 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 278638/278638 [00:33<00:00, 8278.46 examples/s]
+2: 
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 2058 examples [00:00, 11669.40 examples/s]
Generating train split: 6090 examples [00:00, 19221.42 examples/s]
Generating train split: 10251 examples [00:00, 23063.87 examples/s]
Generating train split: 14481 examples [00:00, 26497.83 examples/s]
Generating train split: 18527 examples [00:00, 28159.87 examples/s]
Generating train split: 22610 examples [00:00, 29559.86 examples/s]
Generating train split: 26777 examples [00:00, 30683.45 examples/s]
Generating train split: 31039 examples [00:01, 31682.40 examples/s]
Generating train split: 34830 examples [00:01, 31997.87 examples/s]
Generating train split: 39052 examples [00:01, 25031.74 examples/s]
Generating train split: 43192 examples [00:01, 25049.69 examples/s]
Generating train split: 47447 examples [00:01, 27090.59 examples/s]
Generating train split: 51624 examples [00:01, 28647.99 examples/s]
Generating train split: 55743 examples [00:02, 29572.53 examples/s]
Generating train
+2:  split: 59877 examples [00:02, 30351.39 examples/s]
Generating train split: 64063 examples [00:02, 31079.15 examples/s]
Generating train split: 68083 examples [00:02, 31297.55 examples/s]
Generating train split: 71703 examples [00:02, 25797.07 examples/s]
Generating train split: 75608 examples [00:02, 25632.63 examples/s]
Generating train split: 79766 examples [00:02, 26493.35 examples/s]
Generating train split: 83874 examples [00:03, 28212.45 examples/s]
Generating train split: 88086 examples [00:03, 29737.88 examples/s]
Generating train split: 92253 examples [00:03, 30820.47 examples/s]
Generating train split: 96493 examples [00:03, 31809.21 examples/s]
Generating train split: 100792 examples [00:03, 32488.11 examples/s]
Generating train split: 104490 examples [00:03, 31418.86 examples/s]
Generating train split: 108718 examples [00:03, 22394.09 examples/s]
Generating train split: 112729 examples [00:04, 23091.32 examples/s]
Generating train split: 116714 examples [00:04, 25077.40 examples/s]
Generating trai
+2: n split: 120963 examples [00:04, 27200.99 examples/s]
Generating train split: 124982 examples [00:04, 28474.15 examples/s]
Generating train split: 129148 examples [00:04, 29537.86 examples/s]
Generating train split: 133327 examples [00:04, 30449.72 examples/s]
Generating train split: 137471 examples [00:04, 31114.47 examples/s]
Generating train split: 141370 examples [00:05, 26213.45 examples/s]
Generating train split: 145578 examples [00:05, 26502.74 examples/s]
Generating train split: 149736 examples [00:05, 27197.96 examples/s]
Generating train split: 153939 examples [00:05, 28886.83 examples/s]
Generating train split: 158019 examples [00:05, 29394.76 examples/s]
Generating train split: 162121 examples [00:05, 30229.45 examples/s]
Generating train split: 166314 examples [00:05, 31068.28 examples/s]
Generating train split: 170440 examples [00:06, 31565.58 examples/s]
Generating train split: 175147 examples [00:06, 29345.60 examples/s]
Generating train split: 178210 examples [00:06, 26777.07 examples/s]
Gene
+2: rating train split: 182405 examples [00:06, 26669.36 examples/s]
Generating train split: 186452 examples [00:06, 28111.71 examples/s]
Generating train split: 190598 examples [00:06, 29625.50 examples/s]
Generating train split: 194647 examples [00:06, 30405.53 examples/s]
Generating train split: 198879 examples [00:07, 31385.15 examples/s]
Generating train split: 203078 examples [00:07, 32100.19 examples/s]
Generating train split: 208225 examples [00:07, 30966.34 examples/s]
Generating train split: 213172 examples [00:07, 26412.40 examples/s]
Generating train split: 217397 examples [00:07, 26571.62 examples/s]
Generating train split: 221516 examples [00:07, 27889.87 examples/s]
Generating train split: 225610 examples [00:07, 29114.16 examples/s]
Generating train split: 229634 examples [00:08, 29933.69 examples/s]
Generating train split: 233748 examples [00:08, 30689.67 examples/s]
Generating train split: 238054 examples [00:08, 31615.39 examples/s]
Generating train split: 242070 examples [00:08, 31742.88 examp
+2: les/s]
Generating train split: 245907 examples [00:08, 24052.99 examples/s]
Generating train split: 250178 examples [00:08, 24906.69 examples/s]
Generating train split: 254319 examples [00:09, 25950.98 examples/s]
Generating train split: 257368 examples [00:09, 25732.57 examples/s]
Generating train split: 261480 examples [00:09, 27550.66 examples/s]
Generating train split: 265701 examples [00:09, 29111.02 examples/s]
Generating train split: 269895 examples [00:09, 30099.14 examples/s]
Generating train split: 274079 examples [00:09, 30889.77 examples/s]
Generating train split: 278193 examples [00:09, 31323.95 examples/s]
Generating train split: 278639 examples [00:09, 28369.87 examples/s]
+2: [2025-11-23 14:41:54,865] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:88] [PID:1476128] [RANK:3] Loading dataset: /lustre/fswork/projects/rech/qwv/udv55np/dataset/ift/Nemotron-Super-49B-v1_5/thinking with base_type: chat_template and prompt_style: None
+2: 
Tokenizing Prompts (num_proc=32):   0%|          | 0/278639 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=32):   0%|          | 1000/278639 [00:11<54:33, 84.80 examples/s]
Tokenizing Prompts (num_proc=32):   1%|          | 2000/278639 [00:12<24:22, 189.15 examples/s]
Tokenizing Prompts (num_proc=32):   1%|▏         | 4000/278639 [00:12<09:15, 494.67 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 5000/278639 [00:12<06:31, 699.21 examples/s]
Tokenizing Prompts (num_proc=32):   2%|▏         | 6000/278639 [00:13<05:05, 893.11 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 7000/278639 [00:13<03:44, 1207.47 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 8000/278639 [00:13<02:58, 1513.46 examples/s]
Tokenizing Prompts (num_proc=32):   3%|▎         | 9000/278639 [00:13<02:19, 1929.00 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▎         | 10000/278639 [00:14<01:55, 2326.08 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 1100
+2: 0/278639 [00:14<02:11, 2030.52 examples/s]
Tokenizing Prompts (num_proc=32):   4%|▍         | 12000/278639 [00:15<01:59, 2234.11 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▍         | 13000/278639 [00:15<01:50, 2394.14 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 14000/278639 [00:15<01:41, 2598.26 examples/s]
Tokenizing Prompts (num_proc=32):   5%|▌         | 15000/278639 [00:16<01:48, 2418.73 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▌         | 16000/278639 [00:16<01:38, 2653.17 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▌         | 17000/278639 [00:16<01:24, 3095.46 examples/s]
Tokenizing Prompts (num_proc=32):   6%|▋         | 18000/278639 [00:16<01:19, 3288.89 examples/s]
Tokenizing Prompts (num_proc=32):   7%|▋         | 19000/278639 [00:17<01:16, 3402.65 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 21000/278639 [00:17<00:59, 4324.57 examples/s]
Tokenizing Prompts (num_proc=32):   8%|▊         | 22000/278639 [00:17<01:09, 3
+2: 706.76 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▊         | 24000/278639 [00:18<00:55, 4608.69 examples/s]
Tokenizing Prompts (num_proc=32):   9%|▉         | 25000/278639 [00:18<01:06, 3821.31 examples/s]
Tokenizing Prompts (num_proc=32):  10%|▉         | 27000/278639 [00:19<01:15, 3322.69 examples/s]
Tokenizing Prompts (num_proc=32):  10%|█         | 28000/278639 [00:19<01:17, 3244.87 examples/s]
Tokenizing Prompts (num_proc=32):  10%|█         | 29000/278639 [00:20<01:23, 2973.47 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█         | 30000/278639 [00:20<01:21, 3061.73 examples/s]
Tokenizing Prompts (num_proc=32):  11%|█▏        | 32000/278639 [00:21<01:42, 2409.02 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 33000/278639 [00:23<03:22, 1214.85 examples/s]
Tokenizing Prompts (num_proc=32):  12%|█▏        | 34000/278639 [00:24<03:13, 1264.71 examples/s]
Tokenizing Prompts (num_proc=32):  13%|█▎        | 36000/278639 [00:24<02:06, 1910.78 examples/
+2: s]
Tokenizing Prompts (num_proc=32):  13%|█▎        | 37000/278639 [00:24<01:47, 2243.06 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▎        | 38000/278639 [00:25<01:38, 2452.22 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▍        | 39000/278639 [00:25<01:24, 2843.89 examples/s]
Tokenizing Prompts (num_proc=32):  14%|█▍        | 40000/278639 [00:25<01:11, 3351.97 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▍        | 41000/278639 [00:25<01:20, 2960.92 examples/s]
Tokenizing Prompts (num_proc=32):  15%|█▌        | 42000/278639 [00:26<01:33, 2518.72 examples/s]
Tokenizing Prompts (num_proc=32):  16%|█▌        | 44000/278639 [00:26<01:14, 3139.28 examples/s]
Tokenizing Prompts (num_proc=32):  16%|█▌        | 45000/278639 [00:27<01:21, 2859.71 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 47000/278639 [00:27<01:18, 2961.04 examples/s]
Tokenizing Prompts (num_proc=32):  17%|█▋        | 48000/278639 [00:28<01:34, 2438.57 examples/s]
T
+2: okenizing Prompts (num_proc=32):  18%|█▊        | 50000/278639 [00:28<01:09, 3309.22 examples/s]
Tokenizing Prompts (num_proc=32):  18%|█▊        | 51000/278639 [00:29<00:59, 3806.73 examples/s]
Tokenizing Prompts (num_proc=32):  19%|█▊        | 52000/278639 [00:29<00:53, 4218.10 examples/s]
Tokenizing Prompts (num_proc=32):  19%|█▉        | 54000/278639 [00:29<00:39, 5644.85 examples/s]
Tokenizing Prompts (num_proc=32):  20%|█▉        | 55000/278639 [00:29<00:46, 4830.34 examples/s]
Tokenizing Prompts (num_proc=32):  20%|██        | 56000/278639 [00:30<00:56, 3959.18 examples/s]
Tokenizing Prompts (num_proc=32):  20%|██        | 57000/278639 [00:30<01:05, 3368.74 examples/s]
Tokenizing Prompts (num_proc=32):  21%|██        | 59000/278639 [00:30<01:01, 3548.42 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 60000/278639 [00:31<00:53, 4081.24 examples/s]
Tokenizing Prompts (num_proc=32):  22%|██▏       | 61000/278639 [00:31<01:24, 2562.39 examples/s]
T
+2: okenizing Prompts (num_proc=32):  23%|██▎       | 63000/278639 [00:32<01:03, 3418.06 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 64000/278639 [00:32<01:12, 2970.62 examples/s]
Tokenizing Prompts (num_proc=32):  23%|██▎       | 65000/278639 [00:34<02:49, 1261.23 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▎       | 66000/278639 [00:36<03:17, 1076.12 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▍       | 67000/278639 [00:36<02:52, 1230.22 examples/s]
Tokenizing Prompts (num_proc=32):  24%|██▍       | 68000/278639 [00:36<02:17, 1530.75 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▌       | 70000/278639 [00:37<01:25, 2441.73 examples/s]
Tokenizing Prompts (num_proc=32):  25%|██▌       | 71000/278639 [00:37<01:13, 2808.24 examples/s]
Tokenizing Prompts (num_proc=32):  26%|██▌       | 72000/278639 [00:37<01:12, 2846.62 examples/s]
Tokenizing Prompts (num_proc=32):  26%|██▌       | 73000/278639 [00:38<01:14, 2763.
+2: 72 examples/s]
Tokenizing Prompts (num_proc=32):  27%|██▋       | 74000/278639 [00:38<01:43, 1975.76 examples/s]
Tokenizing Prompts (num_proc=32):  27%|██▋       | 75000/278639 [00:39<01:25, 2368.10 examples/s]
Tokenizing Prompts (num_proc=32):  27%|██▋       | 76000/278639 [00:39<01:07, 3005.35 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 78000/278639 [00:39<00:53, 3745.96 examples/s]
Tokenizing Prompts (num_proc=32):  28%|██▊       | 79000/278639 [00:40<01:13, 2731.05 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▊       | 80000/278639 [00:40<01:20, 2455.48 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▉       | 81000/278639 [00:41<01:06, 2974.48 examples/s]
Tokenizing Prompts (num_proc=32):  29%|██▉       | 82000/278639 [00:41<00:54, 3576.40 examples/s]
Tokenizing Prompts (num_proc=32):  30%|██▉       | 83000/278639 [00:41<00:51, 3803.08 examples/s]
Tokenizing Prompts (num_proc=32):  30%|███       | 84000/278639 [00
+2: :41<00:46, 4176.49 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███       | 85000/278639 [00:41<00:43, 4478.57 examples/s]
Tokenizing Prompts (num_proc=32):  31%|███       | 87000/278639 [00:41<00:28, 6643.69 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 88000/278639 [00:42<00:40, 4725.13 examples/s]
Tokenizing Prompts (num_proc=32):  32%|███▏      | 90000/278639 [00:42<00:48, 3852.97 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 91000/278639 [00:43<00:53, 3537.86 examples/s]
Tokenizing Prompts (num_proc=32):  33%|███▎      | 92000/278639 [00:43<00:50, 3678.96 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▎      | 94000/278639 [00:44<00:48, 3783.49 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▍      | 95000/278639 [00:44<00:57, 3201.24 examples/s]
Tokenizing Prompts (num_proc=32):  34%|███▍      | 96000/278639 [00:44<00:48, 3740.26 examples/s]
Tokenizing Prompts (num_proc=32):  35%|█�
+2: ��█▍      | 97000/278639 [00:46<02:00, 1508.90 examples/s]
Tokenizing Prompts (num_proc=32):  35%|███▌      | 98000/278639 [00:47<02:09, 1391.02 examples/s]
Tokenizing Prompts (num_proc=32):  36%|███▌      | 99000/278639 [00:48<02:31, 1182.88 examples/s]
Tokenizing Prompts (num_proc=32):  36%|███▌      | 100000/278639 [00:48<02:12, 1348.91 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 102000/278639 [00:49<01:39, 1772.22 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 103000/278639 [00:50<01:32, 1894.74 examples/s]
Tokenizing Prompts (num_proc=32):  37%|███▋      | 104000/278639 [00:50<01:34, 1839.04 examples/s]
Tokenizing Prompts (num_proc=32):  38%|███▊      | 106000/278639 [00:50<00:59, 2902.71 examples/s]
Tokenizing Prompts (num_proc=32):  38%|███▊      | 107000/278639 [00:51<01:00, 2815.10 examples/s]
Tokenizing Prompts (num_proc=32):  39%|███▉      | 108000/278639 [00:51<00:52, 3267.99 examples/s]
+2: Tokenizing Prompts (num_proc=32):  39%|███▉      | 109000/278639 [00:51<01:00, 2809.30 examples/s]
Tokenizing Prompts (num_proc=32):  40%|███▉      | 111000/278639 [00:52<00:47, 3500.06 examples/s]
Tokenizing Prompts (num_proc=32):  40%|████      | 112000/278639 [00:52<00:54, 3065.52 examples/s]
Tokenizing Prompts (num_proc=32):  41%|████      | 113000/278639 [00:53<00:56, 2920.23 examples/s]
Tokenizing Prompts (num_proc=32):  41%|████      | 114000/278639 [00:53<01:05, 2524.69 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 116000/278639 [00:53<00:42, 3822.12 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 117000/278639 [00:53<00:36, 4371.27 examples/s]
Tokenizing Prompts (num_proc=32):  42%|████▏     | 118000/278639 [00:54<00:44, 3617.98 examples/s]
Tokenizing Prompts (num_proc=32):  43%|████▎     | 119000/278639 [00:54<00:44, 3613.63 examples/s]
Tokenizing Prompts (num_proc=32):  43%|████�
+2: �     | 121000/278639 [00:54<00:37, 4179.78 examples/s]
Tokenizing Prompts (num_proc=32):  44%|████▍     | 122000/278639 [00:55<00:34, 4563.90 examples/s]
Tokenizing Prompts (num_proc=32):  44%|████▍     | 123000/278639 [00:55<00:50, 3054.00 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▍     | 124000/278639 [00:55<00:44, 3449.84 examples/s]
Tokenizing Prompts (num_proc=32):  45%|████▌     | 126000/278639 [00:56<00:31, 4855.57 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▌     | 127000/278639 [00:56<00:45, 3322.03 examples/s]
Tokenizing Prompts (num_proc=32):  46%|████▋     | 129000/278639 [00:57<00:50, 2984.15 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 130000/278639 [00:58<01:19, 1871.25 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 131000/278639 [00:59<01:33, 1583.36 examples/s]
Tokenizing Prompts (num_proc=32):  47%|████▋     | 132000/278639 [01:00<01:51, 1318.38
+2:  examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 133000/278639 [01:01<01:34, 1547.65 examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 134000/278639 [01:01<01:38, 1475.40 examples/s]
Tokenizing Prompts (num_proc=32):  48%|████▊     | 135000/278639 [01:02<01:30, 1589.72 examples/s]
Tokenizing Prompts (num_proc=32):  49%|████▉     | 137000/278639 [01:02<00:55, 2551.19 examples/s]
Tokenizing Prompts (num_proc=32):  50%|████▉     | 139000/278639 [01:03<00:44, 3111.75 examples/s]
Tokenizing Prompts (num_proc=32):  50%|█████     | 140000/278639 [01:03<00:54, 2542.03 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████     | 142000/278639 [01:04<00:45, 3036.14 examples/s]
Tokenizing Prompts (num_proc=32):  51%|█████▏    | 143000/278639 [01:04<00:39, 3460.36 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 144000/278639 [01:05<00:53, 2516.69 examples/s]
Tokenizing Prompts (num_pr
+2: oc=32):  52%|█████▏    | 145000/278639 [01:05<00:48, 2748.13 examples/s]
Tokenizing Prompts (num_proc=32):  52%|█████▏    | 146000/278639 [01:05<00:39, 3347.92 examples/s]
Tokenizing Prompts (num_proc=32):  53%|█████▎    | 148000/278639 [01:05<00:30, 4293.65 examples/s]
Tokenizing Prompts (num_proc=32):  53%|█████▎    | 149000/278639 [01:05<00:27, 4646.82 examples/s]
Tokenizing Prompts (num_proc=32):  54%|█████▍    | 150000/278639 [01:06<00:39, 3295.40 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▍    | 152000/278639 [01:06<00:27, 4531.98 examples/s]
Tokenizing Prompts (num_proc=32):  55%|█████▍    | 153000/278639 [01:06<00:28, 4364.07 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 155000/278639 [01:07<00:34, 3560.46 examples/s]
Tokenizing Prompts (num_proc=32):  56%|█████▌    | 156000/278639 [01:07<00:34, 3544.23 examples/s]
Tokenizing Prompts (num_proc=32):  56%|████
+2: █▋    | 157000/278639 [01:08<00:50, 2407.70 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 159000/278639 [01:08<00:32, 3701.44 examples/s]
Tokenizing Prompts (num_proc=32):  57%|█████▋    | 160000/278639 [01:09<00:37, 3168.89 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 161000/278639 [01:09<00:39, 2958.47 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 162000/278639 [01:10<00:55, 2090.35 examples/s]
Tokenizing Prompts (num_proc=32):  58%|█████▊    | 163000/278639 [01:12<01:39, 1167.33 examples/s]
Tokenizing Prompts (num_proc=32):  59%|█████▉    | 165000/278639 [01:13<01:12, 1576.76 examples/s]
Tokenizing Prompts (num_proc=32):  60%|█████▉    | 166000/278639 [01:14<01:24, 1340.12 examples/s]
Tokenizing Prompts (num_proc=32):  60%|██████    | 168000/278639 [01:14<00:52, 2100.18 examples/s]
Tokenizing Prompts (num_proc=32):  61%|██████    | 170000/278639
+2:  [01:15<00:48, 2258.51 examples/s]
Tokenizing Prompts (num_proc=32):  61%|██████▏   | 171000/278639 [01:15<00:42, 2524.26 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 172000/278639 [01:15<00:36, 2903.44 examples/s]
Tokenizing Prompts (num_proc=32):  62%|██████▏   | 174000/278639 [01:16<00:29, 3517.13 examples/s]
Tokenizing Prompts (num_proc=32):  63%|██████▎   | 175000/278639 [01:16<00:37, 2750.49 examples/s]
Tokenizing Prompts (num_proc=32):  64%|██████▎   | 177000/278639 [01:17<00:30, 3345.74 examples/s]
Tokenizing Prompts (num_proc=32):  64%|██████▍   | 178000/278639 [01:17<00:31, 3225.80 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▍   | 180000/278639 [01:17<00:29, 3346.24 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▍   | 181000/278639 [01:18<00:26, 3651.33 examples/s]
Tokenizing Prompts (num_proc=32):  65%|██████▌   | 182000/278639 [01:18
+2: <00:24, 3939.36 examples/s]
Tokenizing Prompts (num_proc=32):  66%|██████▋   | 185000/278639 [01:18<00:17, 5462.09 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 186000/278639 [01:19<00:19, 4777.02 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 187000/278639 [01:19<00:20, 4454.37 examples/s]
Tokenizing Prompts (num_proc=32):  67%|██████▋   | 188000/278639 [01:19<00:23, 3846.24 examples/s]
Tokenizing Prompts (num_proc=32):  68%|██████▊   | 189000/278639 [01:20<00:43, 2072.95 examples/s]
Tokenizing Prompts (num_proc=32):  68%|██████▊   | 190000/278639 [01:21<00:39, 2216.31 examples/s]
Tokenizing Prompts (num_proc=32):  69%|██████▊   | 191000/278639 [01:21<00:36, 2393.59 examples/s]
Tokenizing Prompts (num_proc=32):  69%|██████▉   | 193000/278639 [01:21<00:26, 3244.75 examples/s]
Tokenizing Prompts (num_proc=32):  70%|██████▉   | 194000/278639 [01:22<00:24,
+2:  3488.64 examples/s]
Tokenizing Prompts (num_proc=32):  70%|██████▉   | 195000/278639 [01:24<01:18, 1065.48 examples/s]
Tokenizing Prompts (num_proc=32):  70%|███████   | 196000/278639 [01:25<01:05, 1255.03 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████   | 197000/278639 [01:25<00:54, 1487.68 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████   | 198000/278639 [01:25<00:45, 1757.86 examples/s]
Tokenizing Prompts (num_proc=32):  71%|███████▏  | 199000/278639 [01:26<00:52, 1529.75 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 200000/278639 [01:26<00:39, 1982.12 examples/s]
Tokenizing Prompts (num_proc=32):  72%|███████▏  | 201000/278639 [01:27<00:38, 2014.74 examples/s]
Tokenizing Prompts (num_proc=32):  73%|███████▎  | 203000/278639 [01:27<00:27, 2782.45 examples/s]
Tokenizing Prompts (num_proc=32):  73%|███████▎  | 204000/278639 [01:28<00:
+2: 28, 2633.76 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▍  | 206000/278639 [01:28<00:17, 4051.46 examples/s]
Tokenizing Prompts (num_proc=32):  74%|███████▍  | 207000/278639 [01:28<00:19, 3734.33 examples/s]
Tokenizing Prompts (num_proc=32):  75%|███████▍  | 208000/278639 [01:29<00:24, 2843.07 examples/s]
Tokenizing Prompts (num_proc=32):  75%|███████▌  | 209000/278639 [01:29<00:23, 2915.87 examples/s]
Tokenizing Prompts (num_proc=32):  75%|███████▌  | 210000/278639 [01:30<00:24, 2763.21 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▌  | 212000/278639 [01:30<00:20, 3302.56 examples/s]
Tokenizing Prompts (num_proc=32):  76%|███████▋  | 213000/278639 [01:30<00:19, 3384.81 examples/s]
Tokenizing Prompts (num_proc=32):  77%|███████▋  | 215000/278639 [01:31<00:15, 4003.80 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 217000/278639
+2:  [01:31<00:14, 4340.34 examples/s]
Tokenizing Prompts (num_proc=32):  78%|███████▊  | 218000/278639 [01:31<00:13, 4415.43 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███████▉  | 220000/278639 [01:32<00:13, 4281.68 examples/s]
Tokenizing Prompts (num_proc=32):  79%|███████▉  | 221000/278639 [01:32<00:13, 4190.25 examples/s]
Tokenizing Prompts (num_proc=32):  80%|███████▉  | 222000/278639 [01:33<00:17, 3255.32 examples/s]
Tokenizing Prompts (num_proc=32):  80%|████████  | 223000/278639 [01:34<00:27, 2000.79 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████  | 225000/278639 [01:34<00:19, 2724.84 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████  | 226000/278639 [01:34<00:19, 2661.89 examples/s]
Tokenizing Prompts (num_proc=32):  81%|████████▏ | 227000/278639 [01:36<00:38, 1357.45 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ 
+2: | 228000/278639 [01:37<00:32, 1576.55 examples/s]
Tokenizing Prompts (num_proc=32):  82%|████████▏ | 229000/278639 [01:38<00:36, 1349.42 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 230000/278639 [01:38<00:35, 1363.21 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 231000/278639 [01:38<00:26, 1777.93 examples/s]
Tokenizing Prompts (num_proc=32):  83%|████████▎ | 232000/278639 [01:39<00:20, 2262.90 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▎ | 233000/278639 [01:39<00:15, 2870.23 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▍ | 234000/278639 [01:39<00:17, 2595.63 examples/s]
Tokenizing Prompts (num_proc=32):  84%|████████▍ | 235000/278639 [01:39<00:14, 2991.40 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▍ | 236000/278639 [01:40<00:12, 3461.94 examples/s]
Tokenizing Prompts (num_proc=32):  85%
+2: |████████▌ | 237000/278639 [01:40<00:12, 3335.29 examples/s]
Tokenizing Prompts (num_proc=32):  85%|████████▌ | 238000/278639 [01:40<00:10, 3813.68 examples/s]
Tokenizing Prompts (num_proc=32):  86%|████████▌ | 239000/278639 [01:40<00:09, 4132.52 examples/s]
Tokenizing Prompts (num_proc=32):  86%|████████▋ | 240416/278639 [01:41<00:12, 3120.76 examples/s]
Tokenizing Prompts (num_proc=32):  87%|████████▋ | 242416/278639 [01:41<00:10, 3438.86 examples/s]
Tokenizing Prompts (num_proc=32):  87%|████████▋ | 243416/278639 [01:42<00:09, 3829.62 examples/s]
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 244416/278639 [01:43<00:15, 2251.14 examples/s]
Tokenizing Prompts (num_proc=32):  88%|████████▊ | 245416/278639 [01:43<00:12, 2645.35 examples/s]
Tokenizing Prompts (num_proc=32):  89%|████████▉ | 247416/278639 [01:43<00:08, 3781.03 examples/s]
Tokenizin
+2: g Prompts (num_proc=32):  89%|████████▉ | 248416/278639 [01:43<00:07, 4152.83 examples/s]
Tokenizing Prompts (num_proc=32):  90%|████████▉ | 250416/278639 [01:43<00:04, 6084.50 examples/s]
Tokenizing Prompts (num_proc=32):  90%|█████████ | 251416/278639 [01:43<00:04, 6443.03 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 252416/278639 [01:44<00:04, 5521.15 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 253416/278639 [01:44<00:06, 3731.77 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████ | 254123/278639 [01:44<00:07, 3352.95 examples/s]
Tokenizing Prompts (num_proc=32):  91%|█████████▏| 254831/278639 [01:45<00:11, 2076.04 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 255831/278639 [01:46<00:09, 2429.48 examples/s]
Tokenizing Prompts (num_proc=32):  92%|█████████▏| 256831/278639 [01:46<0
+2: 0:07, 3079.39 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 258539/278639 [01:46<00:07, 2620.34 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 259247/278639 [01:47<00:06, 2811.66 examples/s]
Tokenizing Prompts (num_proc=32):  93%|█████████▎| 259954/278639 [01:47<00:06, 2866.41 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 261662/278639 [01:47<00:05, 3291.61 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 262370/278639 [01:47<00:04, 3294.91 examples/s]
Tokenizing Prompts (num_proc=32):  94%|█████████▍| 263078/278639 [01:48<00:05, 2679.21 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▍| 264494/278639 [01:48<00:04, 3456.28 examples/s]
Tokenizing Prompts (num_proc=32):  95%|█████████▌| 265201/278639 [01:48<00:04, 2971.01 examples/s]
Tokenizing Prompts (num_proc=32):  96%|██�
+2: ��██████▌| 266616/278639 [01:49<00:04, 2652.84 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▌| 267324/278639 [01:50<00:04, 2320.86 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▌| 268031/278639 [01:50<00:03, 2704.59 examples/s]
Tokenizing Prompts (num_proc=32):  96%|█████████▋| 268738/278639 [01:51<00:06, 1625.09 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 269446/278639 [01:51<00:04, 1969.19 examples/s]
Tokenizing Prompts (num_proc=32):  97%|█████████▋| 270153/278639 [01:51<00:04, 1775.86 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 272275/278639 [01:52<00:02, 2919.80 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 273690/278639 [01:52<00:01, 3952.77 examples/s]
Tokenizing Prompts (num_proc=32):  98%|█████████▊| 274397/278639 [01:52<00:01, 3112.73 examples/s]
+2: 
Tokenizing Prompts (num_proc=32):  99%|█████████▊| 275104/278639 [01:52<00:01, 3383.83 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 275811/278639 [01:53<00:01, 2327.93 examples/s]
Tokenizing Prompts (num_proc=32):  99%|█████████▉| 276518/278639 [01:53<00:01, 1917.47 examples/s]
Tokenizing Prompts (num_proc=32): 100%|█████████▉| 277932/278639 [01:55<00:00, 1541.23 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 278639/278639 [01:55<00:00, 1722.79 examples/s]
Tokenizing Prompts (num_proc=32): 100%|██████████| 278639/278639 [01:55<00:00, 2411.01 examples/s]
+2: 
Dropping Long Sequences (>16384) (num_proc=32):   0%|          | 0/557277 [00:00<?, ? examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   0%|          | 1000/557277 [00:00<05:10, 1789.00 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   2%|▏         | 9000/557277 [00:00<00:31, 17564.32 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   3%|▎         | 17000/557277 [00:00<00:17, 30485.42 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   5%|▍         | 26000/557277 [00:00<00:12, 42560.60 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   6%|▋         | 36000/557277 [00:01<00:09, 56255.84 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   8%|▊         | 44000/557277 [00:01<00:08, 57586.87 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):   9%|▉         | 52000/557277 [00:01<00:13, 37965.69 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  11%|█▏        | 64000/557277 [00:01<00:09, 49547.77 examples/s]
Dropping
+2:  Long Sequences (>16384) (num_proc=32):  15%|█▍        | 81000/557277 [00:01<00:06, 72009.60 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  17%|█▋        | 94000/557277 [00:01<00:05, 82752.52 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  19%|█▉        | 105000/557277 [00:01<00:05, 88071.49 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  21%|██        | 116000/557277 [00:02<00:05, 86115.20 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  23%|██▎       | 126000/557277 [00:02<00:05, 79984.76 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  25%|██▍       | 137000/557277 [00:02<00:04, 86364.39 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  27%|██▋       | 148000/557277 [00:02<00:04, 91588.40 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  29%|██▊       | 160000/557277 [00:02<00:04, 98327.37 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  31%|███       | 174000/5572
+2: 77 [00:02<00:03, 109147.47 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  34%|███▎      | 187000/557277 [00:02<00:03, 107049.50 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  36%|███▌      | 199000/557277 [00:02<00:03, 102808.68 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  38%|███▊      | 210000/557277 [00:03<00:03, 96100.64 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  39%|███▉      | 220000/557277 [00:03<00:03, 91287.20 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  41%|████▏     | 230000/557277 [00:03<00:03, 92769.75 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  43%|████▎     | 242000/557277 [00:03<00:03, 99103.41 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  46%|████▌     | 256000/557277 [00:03<00:02, 105534.12 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  49%|████▊     | 271000/557277 [00:03<00:02, 114584.42 examples
+2: /s]
Dropping Long Sequences (>16384) (num_proc=32):  51%|█████     | 283000/557277 [00:03<00:02, 101016.54 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  53%|█████▎    | 294000/557277 [00:03<00:02, 101394.75 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  55%|█████▍    | 305000/557277 [00:03<00:02, 96302.89 examples/s] 
Dropping Long Sequences (>16384) (num_proc=32):  57%|█████▋    | 315075/557277 [00:04<00:02, 91299.86 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  58%|█████▊    | 324395/557277 [00:04<00:02, 85868.99 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  60%|█████▉    | 333640/557277 [00:04<00:04, 55552.64 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  61%|██████    | 340640/557277 [00:04<00:04, 49709.14 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  62%|██████▏   | 346640/557277 [00:05<00:08, 26019.59 examples/s]
Dropping 
+2: Long Sequences (>16384) (num_proc=32):  63%|██████▎   | 352640/557277 [00:05<00:07, 28687.90 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  64%|██████▍   | 357640/557277 [00:05<00:06, 30991.24 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  65%|██████▌   | 362640/557277 [00:06<00:09, 19577.30 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  66%|██████▌   | 366640/557277 [00:06<00:08, 21472.69 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  67%|██████▋   | 372640/557277 [00:06<00:07, 24961.69 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  68%|██████▊   | 376640/557277 [00:06<00:09, 18496.03 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  68%|██████▊   | 379640/557277 [00:07<00:09, 17790.28 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  69%|██████▉   | 383640/557277 [00:07<00:08, 20499.73 examples/s]
Dropping 
+2: Long Sequences (>16384) (num_proc=32):  70%|██████▉   | 388640/557277 [00:07<00:07, 23658.68 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  70%|███████   | 391640/557277 [00:07<00:08, 19359.88 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  71%|███████   | 394640/557277 [00:07<00:10, 15691.44 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  71%|███████▏  | 397640/557277 [00:07<00:08, 17819.58 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  73%|███████▎  | 404640/557277 [00:08<00:06, 23883.45 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  73%|███████▎  | 407640/557277 [00:08<00:07, 20838.06 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  74%|███████▎  | 410640/557277 [00:08<00:09, 15836.38 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  74%|███████▍  | 413640/557277 [00:08<00:08, 17791.18 examples/s]
+2: 
Dropping Long Sequences (>16384) (num_proc=32):  75%|███████▌  | 419640/557277 [00:08<00:05, 25250.13 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  76%|███████▌  | 423640/557277 [00:09<00:06, 19899.78 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  77%|███████▋  | 426640/557277 [00:09<00:07, 16481.45 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  77%|███████▋  | 429640/557277 [00:09<00:07, 17802.33 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  78%|███████▊  | 435640/557277 [00:09<00:05, 23702.81 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  79%|███████▊  | 438640/557277 [00:10<00:06, 18544.97 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  79%|███████▉  | 441640/557277 [00:10<00:07, 16402.49 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  80%|███████▉  | 445640/557277 [00:10<00:06, 1788
+2: 3.40 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  81%|████████  | 451640/557277 [00:10<00:04, 23941.46 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  82%|████████▏ | 454640/557277 [00:10<00:05, 18890.17 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  82%|████████▏ | 457640/557277 [00:11<00:06, 15984.80 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  83%|████████▎ | 461640/557277 [00:11<00:05, 19090.74 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  83%|████████▎ | 464640/557277 [00:11<00:04, 20939.59 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  84%|████████▍ | 468640/557277 [00:11<00:03, 23009.39 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  85%|████████▍ | 471640/557277 [00:11<00:04, 18121.65 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  85%|████████▌ | 47
+2: 4640/557277 [00:12<00:05, 16217.02 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  86%|████████▌ | 477640/557277 [00:12<00:04, 17747.63 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  86%|████████▋ | 481640/557277 [00:12<00:03, 21616.92 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  87%|████████▋ | 484640/557277 [00:12<00:03, 23369.88 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  88%|████████▊ | 487640/557277 [00:12<00:04, 16781.47 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  88%|████████▊ | 490640/557277 [00:12<00:04, 16650.16 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  89%|████████▊ | 493640/557277 [00:12<00:03, 18552.48 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  89%|████████▉ | 496640/557277 [00:13<00:03, 19331.92 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  90%|
+2: ████████▉ | 500640/557277 [00:13<00:02, 22921.87 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  90%|█████████ | 503640/557277 [00:13<00:03, 17168.86 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  91%|█████████ | 506640/557277 [00:13<00:02, 17101.93 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  91%|█████████▏| 508640/557277 [00:13<00:02, 17331.03 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  92%|█████████▏| 510640/557277 [00:13<00:02, 17529.32 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  93%|█████████▎| 515640/557277 [00:14<00:01, 24375.40 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  93%|█████████▎| 518640/557277 [00:14<00:02, 17698.47 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  94%|█████████▎| 521640/557277 [00:14<00:01, 18166.50 examples/s]
Dropping Lo
+2: ng Sequences (>16384) (num_proc=32):  94%|█████████▍| 524640/557277 [00:14<00:01, 17556.95 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  95%|█████████▍| 526640/557277 [00:14<00:01, 17588.01 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  96%|█████████▌| 532640/557277 [00:14<00:00, 25624.74 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  96%|█████████▌| 535640/557277 [00:15<00:01, 19777.39 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  97%|█████████▋| 538640/557277 [00:15<00:00, 19240.06 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  97%|█████████▋| 541055/557277 [00:15<00:00, 17734.85 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  98%|█████████▊| 544470/557277 [00:15<00:00, 19553.35 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  99%|█████████▊| 549714/55727
+2: 7 [00:15<00:00, 23559.53 examples/s]
Dropping Long Sequences (>16384) (num_proc=32):  99%|█████████▉| 552958/557277 [00:15<00:00, 23149.39 examples/s]
Dropping Long Sequences (>16384) (num_proc=32): 100%|█████████▉| 556033/557277 [00:16<00:00, 24613.07 examples/s]
Dropping Long Sequences (>16384) (num_proc=32): 100%|██████████| 557277/557277 [00:16<00:00, 33865.07 examples/s]
+2: 
Drop Samples with Zero Trainable Tokens (num_proc=32):   0%|          | 0/553055 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   0%|          | 1000/553055 [00:00<04:52, 1885.78 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   3%|▎         | 15000/553055 [00:00<00:17, 30529.55 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   4%|▍         | 23000/553055 [00:00<00:13, 40549.50 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   6%|▌         | 34000/553055 [00:00<00:10, 49611.55 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):   9%|▊         | 48000/553055 [00:01<00:07, 68487.90 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  10%|█         | 58000/553055 [00:01<00:07, 70032.10 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  12%|█▏        | 69000/553055 [00:01<00:06, 77509.42 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  16%|█▌
+2:         | 89000/553055 [00:01<00:04, 107273.23 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  18%|█▊        | 102000/553055 [00:01<00:04, 105770.81 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  21%|██        | 114000/553055 [00:01<00:04, 89858.06 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  23%|██▎       | 127000/553055 [00:01<00:04, 97176.79 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  25%|██▍       | 138000/553055 [00:01<00:04, 92145.31 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  27%|██▋       | 148000/553055 [00:02<00:04, 88387.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  30%|██▉       | 165000/553055 [00:02<00:03, 108206.42 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  32%|███▏      | 179000/553055 [00:02<00:03, 115886.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  35%|███▍  
+2:     | 192000/553055 [00:02<00:03, 103015.67 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  37%|███▋      | 203000/553055 [00:02<00:03, 103538.28 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  39%|███▊      | 214000/553055 [00:02<00:03, 101771.55 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  41%|████      | 225000/553055 [00:02<00:03, 88997.74 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  43%|████▎     | 239000/553055 [00:02<00:03, 100810.29 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  46%|████▌     | 255000/553055 [00:03<00:02, 113720.57 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  49%|████▊     | 269000/553055 [00:03<00:02, 118511.39 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  51%|█████     | 282000/553055 [00:03<00:02, 104199.31 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=
+2: 32):  53%|█████▎    | 293000/553055 [00:03<00:02, 101424.46 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  55%|█████▍    | 304000/553055 [00:03<00:02, 88100.73 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  58%|█████▊    | 319698/553055 [00:03<00:02, 103404.28 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  60%|██████    | 332679/553055 [00:03<00:02, 109951.92 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  62%|██████▏   | 344528/553055 [00:04<00:04, 44734.40 examples/s] 
Drop Samples with Zero Trainable Tokens (num_proc=32):  64%|██████▍   | 353528/553055 [00:04<00:04, 46230.12 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  65%|██████▌   | 361528/553055 [00:05<00:06, 29115.44 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  66%|██████▋   | 367528/553055 [00:05<00:06, 30916.76 examples/
+2: s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  68%|██████▊   | 374528/553055 [00:05<00:05, 30773.44 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  69%|██████▊   | 379528/553055 [00:06<00:07, 23899.48 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  69%|██████▉   | 383528/553055 [00:06<00:06, 24503.63 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  71%|███████   | 390528/553055 [00:06<00:06, 26512.15 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  71%|███████▏  | 394528/553055 [00:06<00:07, 20032.13 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  72%|███████▏  | 398528/553055 [00:06<00:07, 21662.26 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  73%|███████▎  | 405528/553055 [00:07<00:05, 27708.82 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  74%|███�
+2: ��███▍  | 409528/553055 [00:07<00:07, 18191.18 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  75%|███████▍  | 412528/553055 [00:07<00:07, 19349.02 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  75%|███████▌  | 417528/553055 [00:07<00:05, 23833.15 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  76%|███████▋  | 422528/553055 [00:07<00:04, 26764.14 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  77%|███████▋  | 426528/553055 [00:08<00:06, 18266.92 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  78%|███████▊  | 429528/553055 [00:08<00:06, 19904.78 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  78%|███████▊  | 433528/553055 [00:08<00:05, 23145.10 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  79%|███████▉  | 438528/553055 [00:08<00:04, 25412.53 examples/s
+2: ]
Drop Samples with Zero Trainable Tokens (num_proc=32):  80%|████████  | 442528/553055 [00:09<00:06, 16968.04 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  81%|████████  | 447528/553055 [00:09<00:04, 21614.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  82%|████████▏ | 451528/553055 [00:09<00:04, 22927.34 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  82%|████████▏ | 455528/553055 [00:09<00:05, 17755.60 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  83%|████████▎ | 458528/553055 [00:09<00:05, 18337.60 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  84%|████████▍ | 463528/553055 [00:09<00:03, 23073.45 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  84%|████████▍ | 466528/553055 [00:10<00:03, 23836.25 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32)
+2: :  85%|████████▌ | 470528/553055 [00:10<00:03, 26948.22 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  86%|████████▌ | 474528/553055 [00:10<00:04, 16331.92 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  87%|████████▋ | 478528/553055 [00:10<00:03, 19561.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  87%|████████▋ | 482528/553055 [00:10<00:03, 22708.74 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  88%|████████▊ | 486528/553055 [00:10<00:02, 23956.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  89%|████████▊ | 489528/553055 [00:11<00:03, 16656.24 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  89%|████████▉ | 493528/553055 [00:11<00:02, 20318.63 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  90%|████████▉ | 496528/553055 
+2: [00:11<00:02, 20933.33 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  91%|█████████ | 501528/553055 [00:11<00:02, 23710.55 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  91%|█████████ | 504528/553055 [00:12<00:02, 16406.65 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  92%|█████████▏| 508528/553055 [00:12<00:02, 18955.55 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  93%|█████████▎| 512528/553055 [00:12<00:01, 22493.64 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  93%|█████████▎| 516528/553055 [00:12<00:01, 23921.87 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  94%|█████████▍| 519811/553055 [00:12<00:01, 18543.38 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  95%|█████████▍| 522811/553055 [00:12<00:01, 19174.25 examples/s]
Drop S
+2: amples with Zero Trainable Tokens (num_proc=32):  95%|█████████▌| 525811/553055 [00:13<00:01, 19761.51 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  96%|█████████▌| 530811/553055 [00:13<00:00, 25843.06 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  97%|█████████▋| 534811/553055 [00:13<00:01, 18105.22 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  97%|█████████▋| 537811/553055 [00:13<00:00, 18585.35 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  98%|█████████▊| 541094/553055 [00:13<00:00, 18792.72 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32):  99%|█████████▉| 547225/553055 [00:13<00:00, 26334.18 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=32): 100%|█████████▉| 550791/553055 [00:14<00:00, 26261.70 examples/s]
Drop Samples with Zero Trainable Tokens (nu
+2: m_proc=32): 100%|██████████| 553055/553055 [00:14<00:00, 38307.87 examples/s]
+2: 
Add position_id column (Sample Packing) (num_proc=32):   0%|          | 0/553055 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=32):   0%|          | 1000/553055 [00:00<05:55, 1552.68 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   3%|▎         | 18000/553055 [00:00<00:23, 22500.21 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   6%|▌         | 33000/553055 [00:01<00:12, 40577.46 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   7%|▋         | 41000/553055 [00:01<00:13, 38165.37 examples/s]
Add position_id column (Sample Packing) (num_proc=32):   9%|▉         | 51000/553055 [00:01<00:13, 36480.22 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  12%|█▏        | 67000/553055 [00:01<00:08, 55105.08 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  14%|█▍        | 77000/553055 [00:01<00:07, 61190.79 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  16%|█�
+2: ��        | 86000/553055 [00:01<00:07, 62149.38 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  18%|█▊        | 98000/553055 [00:02<00:08, 54691.74 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  20%|��▉        | 110000/553055 [00:02<00:06, 66189.04 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  22%|██▏       | 119000/553055 [00:02<00:08, 51629.26 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  24%|██▎       | 131000/553055 [00:02<00:08, 49468.43 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  26%|██▌       | 142000/553055 [00:03<00:06, 58928.65 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  27%|██▋       | 150000/553055 [00:03<00:07, 53395.00 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  30%|███       | 166000/553055 [00:03<00:05, 71219.79 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  32%|███▏      | 
+2: 178000/553055 [00:03<00:05, 63989.24 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  34%|███▍      | 190000/553055 [00:03<00:04, 74109.85 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  36%|███▌      | 199000/553055 [00:03<00:06, 56653.86 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  38%|███▊      | 210000/553055 [00:04<00:05, 59626.43 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  39%|███▉      | 218000/553055 [00:04<00:06, 55555.12 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  41%|████      | 227000/553055 [00:04<00:06, 48894.50 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  43%|████▎     | 238000/553055 [00:04<00:05, 58706.67 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  45%|████▌     | 251000/553055 [00:04<00:04, 71432.41 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  47%|██�
+2: �█▋     | 260000/553055 [00:04<00:04, 72380.65 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  49%|████▊     | 269000/553055 [00:04<00:03, 74613.81 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  50%|█████     | 278000/553055 [00:05<00:04, 57523.79 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  52%|█████▏    | 290000/553055 [00:05<00:04, 57380.59 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  54%|█████▎    | 297000/553055 [00:05<00:04, 52932.82 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  56%|█████▌    | 307000/553055 [00:05<00:04, 53661.70 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  57%|█████▋    | 313566/553055 [00:05<00:04, 52430.69 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  59%|█████▉    | 326830/553055 [00:06<00:03, 68866.92 examples/s]
Add position_id column (Sample Pac
+2: king) (num_proc=32):  61%|██████    | 335528/553055 [00:06<00:03, 62168.99 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  62%|██████▏   | 342528/553055 [00:06<00:06, 30359.98 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  63%|██████▎   | 348528/553055 [00:07<00:10, 19637.48 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  64%|██████▎   | 352528/553055 [00:07<00:09, 20550.66 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  65%|██████▍   | 357528/553055 [00:07<00:08, 23876.95 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  65%|██████▌   | 361528/553055 [00:08<00:16, 11299.99 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  66%|██████▌   | 365528/553055 [00:08<00:14, 13107.15 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  67%|██████▋   | 368528/553055 [00:09<00:12, 1
+2: 4547.58 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  67%|██████▋   | 371528/553055 [00:09<00:11, 16383.38 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  68%|██████▊   | 374528/553055 [00:09<00:13, 13010.59 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  68%|██████▊   | 377528/553055 [00:10<00:18, 9269.18 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  69%|██████▊   | 379528/553055 [00:10<00:17, 9904.41 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  69%|██████▉   | 383528/553055 [00:10<00:13, 12216.28 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  70%|███████   | 387528/553055 [00:10<00:10, 15286.87 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  71%|███████   | 390528/553055 [00:10<00:12, 13201.00 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  71%|
+2: ███████   | 392528/553055 [00:11<00:18, 8636.16 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  71%|███████▏  | 394528/553055 [00:11<00:16, 9417.68 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  72%|███████▏  | 399528/553055 [00:11<00:12, 12764.34 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  73%|███████▎  | 402528/553055 [00:11<00:10, 14130.34 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  74%|███████▎  | 406528/553055 [00:12<00:10, 14528.49 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  74%|███████▍  | 408528/553055 [00:12<00:16, 8734.32 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  74%|███████▍  | 410528/553055 [00:12<00:14, 9966.98 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  75%|███████▍  | 413528/553055 [00:13<00:11, 12403.99 exam
+2: ples/s]
Add position_id column (Sample Packing) (num_proc=32):  75%|███████▌  | 415528/553055 [00:13<00:10, 12953.43 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  75%|███████▌  | 417528/553055 [00:13<00:10, 12999.67 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  76%|███████▌  | 421528/553055 [00:13<00:07, 17442.18 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  77%|███████▋  | 424528/553055 [00:14<00:15, 8282.31 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  77%|███████▋  | 428528/553055 [00:14<00:11, 10825.83 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  78%|███████▊  | 431528/553055 [00:14<00:09, 12619.71 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  78%|███████▊  | 433528/553055 [00:14<00:09, 12057.25 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  7
+2: 9%|███████▉  | 437528/553055 [00:14<00:07, 15824.54 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  80%|███████▉  | 440528/553055 [00:15<00:12, 8656.73 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  80%|████████  | 443528/553055 [00:15<00:10, 10734.30 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  81%|████████  | 447528/553055 [00:15<00:08, 12959.29 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  81%|████████▏ | 449528/553055 [00:16<00:08, 12674.94 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  82%|████████▏ | 452528/553055 [00:16<00:06, 14984.16 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  82%|████████▏ | 454528/553055 [00:16<00:06, 15345.24 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  83%|████████▎ | 456528/553055 [00:16<00:11
+2: , 8258.67 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  83%|████████▎ | 458528/553055 [00:17<00:09, 9465.04 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  84%|████████▎ | 462528/553055 [00:17<00:06, 13509.02 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  84%|████████▍ | 465528/553055 [00:17<00:07, 11758.84 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  85%|████████▍ | 469528/553055 [00:17<00:05, 15780.04 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  85%|████████▌ | 472528/553055 [00:18<00:09, 8739.61 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  86%|████████▌ | 476528/553055 [00:18<00:06, 11215.45 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  87%|████████▋ | 478528/553055 [00:18<00:06, 12046.91 examples/s]
Add position_id column (Sampl
+2: e Packing) (num_proc=32):  87%|████████▋ | 481528/553055 [00:18<00:05, 11946.33 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  87%|████████▋ | 483528/553055 [00:18<00:05, 13112.65 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  88%|████████▊ | 486528/553055 [00:19<00:05, 12999.60 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  88%|████████▊ | 488528/553055 [00:19<00:07, 8736.02 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  89%|████████▊ | 490528/553055 [00:19<00:06, 9712.36 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  89%|████████▉ | 493528/553055 [00:19<00:05, 11106.14 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  90%|████████▉ | 497528/553055 [00:20<00:03, 13992.88 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  90%|███████
+2: ██ | 499528/553055 [00:20<00:03, 13592.24 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  91%|█████████ | 502528/553055 [00:20<00:03, 13309.38 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  91%|█████████ | 504528/553055 [00:21<00:05, 8697.70 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  92%|█████████▏| 506528/553055 [00:21<00:04, 9538.91 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  92%|█████████▏| 510528/553055 [00:21<00:03, 13373.83 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  93%|█████████▎| 512528/553055 [00:21<00:03, 12958.32 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  93%|█████████▎| 514528/553055 [00:21<00:03, 12238.18 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  94%|█████████▍| 518528/553055 [00:21<00:02, 12850
+2: .77 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  94%|█████████▍| 520811/553055 [00:22<00:03, 9134.74 examples/s] 
Add position_id column (Sample Packing) (num_proc=32):  95%|█████████▍| 522811/553055 [00:22<00:03, 9681.34 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  95%|█████████▌| 526811/553055 [00:22<00:01, 13802.14 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  96%|█████████▌| 528811/553055 [00:22<00:01, 13885.82 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  96%|█████████▌| 530811/553055 [00:23<00:01, 12412.71 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  97%|█████████▋| 533811/553055 [00:23<00:01, 11245.89 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  97%|█████████▋| 535811/553055 [00:23<00:02, 8462.82 examples/s] 
Add position_id column
+2:  (Sample Packing) (num_proc=32):  97%|█████████▋| 537811/553055 [00:23<00:01, 9732.22 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  98%|█████████▊| 542093/553055 [00:24<00:00, 14392.28 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  98%|█████████▊| 544376/553055 [00:24<00:00, 13923.84 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  99%|█████████▉| 546225/553055 [00:24<00:00, 14305.65 examples/s]
Add position_id column (Sample Packing) (num_proc=32):  99%|█████████▉| 548074/553055 [00:24<00:00, 14524.01 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|█████████▉| 550640/553055 [00:24<00:00, 15092.95 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|█████████▉| 552489/553055 [00:24<00:00, 11350.53 examples/s]
Add position_id column (Sample Packing) (num_proc=32): 100%|
+2: ██████████| 553055/553055 [00:25<00:00, 21814.21 examples/s]
+2: 
Saving the dataset (0/35 shards):   0%|          | 0/553055 [00:00<?, ? examples/s]
Saving the dataset (0/35 shards):   1%|          | 4000/553055 [00:00<00:54, 10044.67 examples/s]
Saving the dataset (0/35 shards):  24%|██▍       | 132000/553055 [00:00<00:01, 342845.14 examples/s]
Saving the dataset (1/35 shards):  32%|███▏      | 178802/553055 [00:00<00:01, 342845.14 examples/s]
Saving the dataset (2/35 shards):  41%|████      | 224604/553055 [00:00<00:00, 342845.14 examples/s]
Saving the dataset (3/35 shards):  46%|████▌     | 254208/553055 [00:00<00:00, 342845.14 examples/s]
Saving the dataset (4/35 shards):  46%|████▌     | 254208/553055 [00:00<00:00, 342845.14 examples/s]
Saving the dataset (4/35 shards):  47%|████▋     | 258208/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (5/35 shards):  50%|█████     | 277614/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (6/35 shards):  50%|█████     | 277614/5530
+2: 55 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (7/35 shards):  51%|█████     | 280416/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (8/35 shards):  52%|█████▏    | 286218/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (9/35 shards):  53%|█████▎    | 294822/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (10/35 shards):  53%|█████▎    | 294822/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (11/35 shards):  53%|█████▎    | 294822/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (12/35 shards):  55%|█████▌    | 305624/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (13/35 shards):  56%|█████▌    | 307426/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (14/35 shards):  57%|█████▋    | 316030/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (15/35 shards):  57%|█████▋    | 316030/553055 
+2: [00:00<00:00, 590161.41 examples/s]
Saving the dataset (16/35 shards):  58%|███���█▊    | 320832/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (17/35 shards):  59%|█████▊    | 323634/553055 [00:00<00:00, 590161.41 examples/s]
Saving the dataset (17/35 shards):  64%|██████▍   | 355634/553055 [00:00<00:00, 590181.02 examples/s]
Saving the dataset (18/35 shards):  72%|███████▏  | 399436/553055 [00:00<00:00, 590181.02 examples/s]
Saving the dataset (18/35 shards):  79%|███████▉  | 437436/553055 [00:01<00:00, 436421.35 examples/s]
Saving the dataset (18/35 shards):  90%|█████████ | 500436/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (19/35 shards):  94%|█████████▍| 520237/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (20/35 shards):  95%|█████████▌| 528038/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (21/35 shards):  96%|█�
+2: ��███████▌| 528839/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (22/35 shards):  96%|█████████▌| 529641/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (23/35 shards):  96%|█████████▌| 531442/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (24/35 shards):  97%|█████████▋| 534243/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (25/35 shards):  97%|█████████▋| 536045/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (26/35 shards):  97%|█████████▋| 536846/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (27/35 shards):  97%|█████████▋| 537647/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (28/35 shards):  97%|█████████▋| 538448/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (29/35 shards):  98%|█████████▊| 540249/553055 [00:01
+2: <00:00, 382276.81 examples/s]
Saving the dataset (30/35 shards):  98%|█████████▊| 542050/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (31/35 shards):  98%|█████████▊| 542851/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (32/35 shards):  99%|█████████▊| 545652/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (33/35 shards):  99%|█████████▉| 549453/553055 [00:01<00:00, 382276.81 examples/s]
Saving the dataset (33/35 shards): 100%|█████████▉| 551453/553055 [00:01<00:00, 303881.17 examples/s]
Saving the dataset (34/35 shards): 100%|█████████▉| 552254/553055 [00:01<00:00, 303881.17 examples/s]
Saving the dataset (35/35 shards): 100%|██████████| 553055/553055 [00:01<00:00, 303881.17 examples/s]
Saving the dataset (35/35 shards): 100%|██████████| 553055/553055 [00:01<00:00, 327995.30 examples/s]
+0: [2025-11-23 14:44:54,001] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:472] [PID:3958317] [RANK:0] Loading prepared dataset from disk at /lustre/fswork/projects/rech/dgo/udv55np/dataset_gemma/Nemotron-Super-49B-v1_5/split_0.5/d383a692316f2e59f08e99509bd12afa...
+0: jzxh251:3958317:3958317 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.233<0>
+0: jzxh251:3958317:3958317 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh251:3958317:3958317 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh251:3958317:3958317 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh251:3958317:3958317 [0] NCCL INFO cudaDriverVersion 12080
+0: NCCL version 2.21.5+cuda12.4
+0: jzxh251:3958317:3958317 [0] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958318:3958318 [1] NCCL INFO cudaDriverVersion 12080
+0: jzxh251:3958318:3958318 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.233<0>
+0: jzxh251:3958318:3958318 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh251:3958318:3958318 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh251:3958318:3958318 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh251:3958318:3958318 [1] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958319:3958319 [2] NCCL INFO cudaDriverVersion 12080
+0: jzxh251:3958320:3958320 [3] NCCL INFO cudaDriverVersion 12080
+2: jzxh253:1476125:1476125 [0] NCCL INFO cudaDriverVersion 12080
+0: jzxh251:3958320:3958320 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.233<0>
+0: jzxh251:3958319:3958319 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.233<0>
+0: jzxh251:3958320:3958320 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh251:3958320:3958320 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh251:3958320:3958320 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+0: jzxh251:3958319:3958319 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+0: jzxh251:3958319:3958319 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+0: jzxh251:3958319:3958319 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh252:2700705:2700705 [0] NCCL INFO cudaDriverVersion 12080
+1: jzxh252:2700705:2700705 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.237<0>
+0: jzxh251:3958320:3958320 [3] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958319:3958319 [2] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472009:1472009 [0] NCCL INFO cudaDriverVersion 12080
+3: jzxh254:1472009:1472009 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.245<0>
+2: jzxh253:1476125:1476125 [0] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.241<0>
+2: jzxh253:1476127:1476127 [2] NCCL INFO cudaDriverVersion 12080
+1: jzxh252:2700705:2700705 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh252:2700705:2700705 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh252:2700705:2700705 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh252:2700706:2700706 [1] NCCL INFO cudaDriverVersion 12080
+3: jzxh254:1472009:1472009 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh254:1472009:1472009 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh254:1472009:1472009 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476125:1476125 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh254:1472011:1472011 [2] NCCL INFO cudaDriverVersion 12080
+2: jzxh253:1476125:1476125 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476125:1476125 [0] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476127:1476127 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.241<0>
+2: jzxh253:1476127:1476127 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh253:1476127:1476127 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476127:1476127 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+1: jzxh252:2700708:2700708 [3] NCCL INFO cudaDriverVersion 12080
+1: jzxh252:2700706:2700706 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.237<0>
+1: jzxh252:2700705:2700705 [0] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700706:2700706 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh252:2700706:2700706 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh252:2700706:2700706 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh254:1472010:1472010 [1] NCCL INFO cudaDriverVersion 12080
+1: jzxh252:2700708:2700708 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.237<0>
+1: jzxh252:2700707:2700707 [2] NCCL INFO cudaDriverVersion 12080
+3: jzxh254:1472009:1472009 [0] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700708:2700708 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh252:2700708:2700708 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476128:1476128 [3] NCCL INFO cudaDriverVersion 12080
+1: jzxh252:2700708:2700708 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476125:1476125 [0] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472011:1472011 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.245<0>
+3: jzxh254:1472010:1472010 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.245<0>
+1: jzxh252:2700707:2700707 [2] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.237<0>
+2: jzxh253:1476126:1476126 [1] NCCL INFO cudaDriverVersion 12080
+2: jzxh253:1476127:1476127 [2] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700706:2700706 [1] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700707:2700707 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+1: jzxh252:2700707:2700707 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+1: jzxh252:2700707:2700707 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh254:1472012:1472012 [3] NCCL INFO cudaDriverVersion 12080
+3: jzxh254:1472011:1472011 [2] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh254:1472011:1472011 [2] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh254:1472011:1472011 [2] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh254:1472010:1472010 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh254:1472010:1472010 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+3: jzxh254:1472010:1472010 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476126:1476126 [1] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.241<0>
+2: jzxh253:1476128:1476128 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.241<0>
+1: jzxh252:2700708:2700708 [3] NCCL INFO Comm config Blocking set to 1
+2: jzxh253:1476126:1476126 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh253:1476126:1476126 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476126:1476126 [1] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476128:1476128 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+2: jzxh253:1476128:1476128 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476128:1476128 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+3: jzxh254:1472011:1472011 [2] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700707:2700707 [2] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472010:1472010 [1] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472012:1472012 [3] NCCL INFO Bootstrap : Using ibp24s0:10.100.7.245<0>
+3: jzxh254:1472012:1472012 [3] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
+3: jzxh254:1472012:1472012 [3] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
+2: jzxh253:1476126:1476126 [1] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472012:1472012 [3] NCCL INFO NET/Plugin: Using internal network plugin.
+2: jzxh253:1476128:1476128 [3] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472012:1472012 [3] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958318:3959113 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.233<0>
+0: jzxh251:3958318:3959113 [1] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958318:3959113 [1] NCCL INFO Using network IB
+0: jzxh251:3958317:3959112 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.233<0>
+0: jzxh251:3958320:3959114 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.233<0>
+0: jzxh251:3958317:3959112 [0] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958317:3959112 [0] NCCL INFO Using network IB
+0: jzxh251:3958319:3959115 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.233<0>
+0: jzxh251:3958320:3959114 [3] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958320:3959114 [3] NCCL INFO Using network IB
+0: jzxh251:3958319:3959115 [2] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958319:3959115 [2] NCCL INFO Using network IB
+2: jzxh253:1476127:1478099 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.241<0>
+2: jzxh253:1476125:1478098 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.241<0>
+2: jzxh253:1476128:1478101 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.241<0>
+2: jzxh253:1476126:1478100 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.241<0>
+2: jzxh253:1476127:1478099 [2] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476125:1478098 [0] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476127:1478099 [2] NCCL INFO Using network IB
+2: jzxh253:1476125:1478098 [0] NCCL INFO Using network IB
+2: jzxh253:1476128:1478101 [3] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476128:1478101 [3] NCCL INFO Using network IB
+2: jzxh253:1476126:1478100 [1] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476126:1478100 [1] NCCL INFO Using network IB
+1: jzxh252:2700707:2701505 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.237<0>
+1: jzxh252:2700705:2701502 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.237<0>
+1: jzxh252:2700706:2701503 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.237<0>
+1: jzxh252:2700707:2701505 [2] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700707:2701505 [2] NCCL INFO Using network IB
+1: jzxh252:2700705:2701502 [0] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700705:2701502 [0] NCCL INFO Using network IB
+1: jzxh252:2700708:2701504 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.237<0>
+1: jzxh252:2700706:2701503 [1] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700706:2701503 [1] NCCL INFO Using network IB
+1: jzxh252:2700708:2701504 [3] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700708:2701504 [3] NCCL INFO Using network IB
+3: jzxh254:1472009:1472810 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.245<0>
+3: jzxh254:1472010:1472812 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.245<0>
+3: jzxh254:1472011:1472811 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.245<0>
+3: jzxh254:1472009:1472810 [0] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472009:1472810 [0] NCCL INFO Using network IB
+3: jzxh254:1472010:1472812 [1] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472010:1472812 [1] NCCL INFO Using network IB
+3: jzxh254:1472012:1472813 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [2]mlx5_2:1/IB [3]mlx5_3:1/IB [RO]; OOB ibp24s0:10.100.7.245<0>
+3: jzxh254:1472011:1472811 [2] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472011:1472811 [2] NCCL INFO Using network IB
+3: jzxh254:1472012:1472813 [3] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472012:1472813 [3] NCCL INFO Using network IB
+0: jzxh251:3958317:3959112 [0] NCCL INFO ncclCommInitRank comm 0x55ad9c344130 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init START
+0: jzxh251:3958319:3959115 [2] NCCL INFO ncclCommInitRank comm 0x557a67ebf3e0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init START
+0: jzxh251:3958320:3959114 [3] NCCL INFO ncclCommInitRank comm 0x5627d69ed2e0 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init START
+0: jzxh251:3958318:3959113 [1] NCCL INFO ncclCommInitRank comm 0x561703831fa0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init START
+3: jzxh254:1472011:1472811 [2] NCCL INFO ncclCommInitRank comm 0x55ee68f2ae50 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init START
+3: jzxh254:1472012:1472813 [3] NCCL INFO ncclCommInitRank comm 0x55e3cd6150e0 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init START
+1: jzxh252:2700705:2701502 [0] NCCL INFO ncclCommInitRank comm 0x55691e321820 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init START
+1: jzxh252:2700706:2701503 [1] NCCL INFO ncclCommInitRank comm 0x55d9b6dce4a0 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init START
+3: jzxh254:1472010:1472812 [1] NCCL INFO ncclCommInitRank comm 0x558d1e103f20 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init START
+2: jzxh253:1476125:1478098 [0] NCCL INFO ncclCommInitRank comm 0x55705cd0ee30 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init START
+2: jzxh253:1476127:1478099 [2] NCCL INFO ncclCommInitRank comm 0x555b4b77d2b0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init START
+2: jzxh253:1476128:1478101 [3] NCCL INFO ncclCommInitRank comm 0x55c13cf8e7e0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init START
+2: jzxh253:1476126:1478100 [1] NCCL INFO ncclCommInitRank comm 0x559d74834b90 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init START
+3: jzxh254:1472009:1472810 [0] NCCL INFO ncclCommInitRank comm 0x5603bb4b7f00 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init START
+1: jzxh252:2700707:2701505 [2] NCCL INFO ncclCommInitRank comm 0x563d58040fd0 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init START
+1: jzxh252:2700708:2701504 [3] NCCL INFO ncclCommInitRank comm 0x55e552323b50 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init START
+3: jzxh254:1472009:1472810 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+3: jzxh254:1472009:1472810 [0] NCCL INFO NVLS multicast support is not available on dev 0
+1: jzxh252:2700705:2701502 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+1: jzxh252:2700705:2701502 [0] NCCL INFO NVLS multicast support is not available on dev 0
+0: jzxh251:3958317:3959112 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+0: jzxh251:3958317:3959112 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh253:1476128:1478101 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+2: jzxh253:1476128:1478101 [3] NCCL INFO NVLS multicast support is not available on dev 3
+3: jzxh254:1472011:1472811 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+3: jzxh254:1472011:1472811 [2] NCCL INFO NVLS multicast support is not available on dev 2
+1: jzxh252:2700707:2701505 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh252:2700707:2701505 [2] NCCL INFO NVLS multicast support is not available on dev 2
+3: jzxh254:1472012:1472813 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+3: jzxh254:1472012:1472813 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh251:3958319:3959115 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh252:2700708:2701504 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+1: jzxh252:2700708:2701504 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh251:3958319:3959115 [2] NCCL INFO NVLS multicast support is not available on dev 2
+3: jzxh254:1472010:1472812 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+3: jzxh254:1472010:1472812 [1] NCCL INFO NVLS multicast support is not available on dev 1
+1: jzxh252:2700706:2701503 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+1: jzxh252:2700706:2701503 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh251:3958318:3959113 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+0: jzxh251:3958318:3959113 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh251:3958320:3959114 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+0: jzxh251:3958320:3959114 [3] NCCL INFO NVLS multicast support is not available on dev 3
+2: jzxh253:1476127:1478099 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+2: jzxh253:1476127:1478099 [2] NCCL INFO NVLS multicast support is not available on dev 2
+2: jzxh253:1476125:1478098 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+2: jzxh253:1476125:1478098 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh253:1476126:1478100 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+2: jzxh253:1476126:1478100 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh251:3958317:3959112 [0] NCCL INFO comm 0x55ad9c344130 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+0: jzxh251:3958318:3959113 [1] NCCL INFO comm 0x561703831fa0 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 00/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 01/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 02/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 03/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 05/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 06/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 07/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958320:3959114 [3] NCCL INFO comm 0x5627d69ed2e0 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+0: jzxh251:3958319:3959115 [2] NCCL INFO comm 0x557a67ebf3e0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+0: jzxh251:3958318:3959113 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/9/-1->1->-1 [2] 3/-1/-1->1->0 [3] 0/-1/-1->1->3 [4] -1/-1/-1->1->2 [5] 3/9/-1->1->-1 [6] -1/-1/-1->1->3 [7] 0/-1/-1->1->2 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->5 [10] 3/-1/-1->1->0 [11] 0/-1/-1->1->3 [12] -1/-1/-1->1->2 [13] 3/-1/-1->1->5 [14] -1/-1/-1->1->3 [15] 0/-1/-1->1->2
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 08/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958318:3959113 [1] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 09/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 10/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 11/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 13/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 14/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958320:3959114 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 0/-1/-1->3->2 [2] -1/-1/-1->3->1 [3] 1/11/-1->3->-1 [4] 2/-1/-1->3->0 [5] 0/-1/-1->3->1 [6] 1/-1/-1->3->0 [7] 2/11/-1->3->-1 [8] -1/-1/-1->3->2 [9] 0/-1/-1->3->2 [10] -1/-1/-1->3->1 [11] 1/-1/-1->3->7 [12] 2/-1/-1->3->0 [13] 0/-1/-1->3->1 [14] 1/-1/-1->3->0 [15] 2/-1/-1->3->7
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 15/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958319:3959115 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 0/10/-1->2->-1 [3] -1/-1/-1->2->0 [4] 1/-1/-1->2->3 [5] -1/-1/-1->2->0 [6] 0/10/-1->2->-1 [7] 1/-1/-1->2->3 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 0/-1/-1->2->6 [11] -1/-1/-1->2->0 [12] 1/-1/-1->2->3 [13] -1/-1/-1->2->0 [14] 0/-1/-1->2->6 [15] 1/-1/-1->2->3
+0: jzxh251:3958320:3959114 [3] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958319:3959115 [2] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472012:1472813 [3] NCCL INFO comm 0x55e3cd6150e0 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+0: jzxh251:3958317:3959112 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->3 [2] 1/-1/-1->0->2 [3] 2/-1/-1->0->1 [4] 3/8/-1->0->-1 [5] 2/-1/-1->0->3 [6] 3/-1/-1->0->2 [7] -1/-1/-1->0->1 [8] 1/-1/-1->0->4 [9] -1/-1/-1->0->3 [10] 1/-1/-1->0->2 [11] 2/-1/-1->0->1 [12] 3/-1/-1->0->4 [13] 2/-1/-1->0->3 [14] 3/-1/-1->0->2 [15] -1/-1/-1->0->1
+0: jzxh251:3958317:3959112 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472012:1472813 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 12/-1/-1->15->14 [2] -1/-1/-1->15->13 [3] 13/-1/-1->15->11 [4] 14/-1/-1->15->12 [5] 12/-1/-1->15->13 [6] 13/-1/-1->15->12 [7] 14/-1/-1->15->11 [8] -1/-1/-1->15->14 [9] 12/-1/-1->15->14 [10] -1/-1/-1->15->13 [11] 13/7/-1->15->-1 [12] 14/-1/-1->15->12 [13] 12/-1/-1->15->13 [14] 13/-1/-1->15->12 [15] 14/7/-1->15->-1
+3: jzxh254:1472012:1472813 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472011:1472811 [2] NCCL INFO comm 0x55ee68f2ae50 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+1: jzxh252:2700705:2701502 [0] NCCL INFO comm 0x55691e321820 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+1: jzxh252:2700706:2701503 [1] NCCL INFO comm 0x55d9b6dce4a0 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+1: jzxh252:2700707:2701505 [2] NCCL INFO comm 0x563d58040fd0 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+1: jzxh252:2700708:2701504 [3] NCCL INFO comm 0x55e552323b50 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+3: jzxh254:1472011:1472811 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 12/-1/-1->14->10 [3] -1/-1/-1->14->12 [4] 13/-1/-1->14->15 [5] -1/-1/-1->14->12 [6] 12/-1/-1->14->10 [7] 13/-1/-1->14->15 [8] 15/-1/-1->14->13 [9] 15/-1/-1->14->13 [10] 12/6/-1->14->-1 [11] -1/-1/-1->14->12 [12] 13/-1/-1->14->15 [13] -1/-1/-1->14->12 [14] 12/6/-1->14->-1 [15] 13/-1/-1->14->15
+3: jzxh254:1472011:1472811 [2] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472010:1472812 [1] NCCL INFO comm 0x558d1e103f20 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+2: jzxh253:1476125:1478098 [0] NCCL INFO comm 0x55705cd0ee30 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+2: jzxh253:1476128:1478101 [3] NCCL INFO comm 0x55c13cf8e7e0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+2: jzxh253:1476127:1478099 [2] NCCL INFO comm 0x555b4b77d2b0 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+2: jzxh253:1476126:1478100 [1] NCCL INFO comm 0x559d74834b90 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+2: jzxh253:1476125:1478098 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->11 [2] 9/-1/-1->8->10 [3] 10/-1/-1->8->9 [4] 11/4/12->8->0 [5] 10/-1/-1->8->11 [6] 11/-1/-1->8->10 [7] -1/-1/-1->8->9 [8] 9/-1/-1->8->4 [9] -1/-1/-1->8->11 [10] 9/-1/-1->8->10 [11] 10/-1/-1->8->9 [12] 11/-1/-1->8->4 [13] 10/-1/-1->8->11 [14] 11/-1/-1->8->10 [15] -1/-1/-1->8->9
+2: jzxh253:1476125:1478098 [0] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476127:1478099 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 [2] 8/6/14->10->2 [3] -1/-1/-1->10->8 [4] 9/-1/-1->10->11 [5] -1/-1/-1->10->8 [6] 8/6/14->10->2 [7] 9/-1/-1->10->11 [8] 11/-1/-1->10->9 [9] 11/-1/-1->10->9 [10] 8/-1/-1->10->6 [11] -1/-1/-1->10->8 [12] 9/-1/-1->10->11 [13] -1/-1/-1->10->8 [14] 8/-1/-1->10->6 [15] 9/-1/-1->10->11
+2: jzxh253:1476128:1478101 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 8/-1/-1->11->10 [2] -1/-1/-1->11->9 [3] 9/7/15->11->3 [4] 10/-1/-1->11->8 [5] 8/-1/-1->11->9 [6] 9/-1/-1->11->8 [7] 10/7/15->11->3 [8] -1/-1/-1->11->10 [9] 8/-1/-1->11->10 [10] -1/-1/-1->11->9 [11] 9/-1/-1->11->7 [12] 10/-1/-1->11->8 [13] 8/-1/-1->11->9 [14] 9/-1/-1->11->8 [15] 10/-1/-1->11->7
+3: jzxh254:1472010:1472812 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->9 [2] 15/-1/-1->13->12 [3] 12/-1/-1->13->15 [4] -1/-1/-1->13->14 [5] 15/-1/-1->13->9 [6] -1/-1/-1->13->15 [7] 12/-1/-1->13->14 [8] 14/-1/-1->13->12 [9] 14/5/-1->13->-1 [10] 15/-1/-1->13->12 [11] 12/-1/-1->13->15 [12] -1/-1/-1->13->14 [13] 15/5/-1->13->-1 [14] -1/-1/-1->13->15 [15] 12/-1/-1->13->14
+3: jzxh254:1472010:1472812 [1] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472009:1472810 [0] NCCL INFO comm 0x5603bb4b7f00 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+3: jzxh254:1472009:1472810 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->15 [2] 13/-1/-1->12->14 [3] 14/-1/-1->12->13 [4] 15/-1/-1->12->8 [5] 14/-1/-1->12->15 [6] 15/-1/-1->12->14 [7] -1/-1/-1->12->13 [8] 13/4/-1->12->-1 [9] -1/-1/-1->12->15 [10] 13/-1/-1->12->14 [11] 14/-1/-1->12->13 [12] 15/4/-1->12->-1 [13] 14/-1/-1->12->15 [14] 15/-1/-1->12->14 [15] -1/-1/-1->12->13
+3: jzxh254:1472009:1472810 [0] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476126:1478100 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/5/13->9->1 [2] 11/-1/-1->9->8 [3] 8/-1/-1->9->11 [4] -1/-1/-1->9->10 [5] 11/5/13->9->1 [6] -1/-1/-1->9->11 [7] 8/-1/-1->9->10 [8] 10/-1/-1->9->8 [9] 10/-1/-1->9->5 [10] 11/-1/-1->9->8 [11] 8/-1/-1->9->11 [12] -1/-1/-1->9->10 [13] 11/-1/-1->9->5 [14] -1/-1/-1->9->11 [15] 8/-1/-1->9->10
+2: jzxh253:1476127:1478099 [2] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476128:1478101 [3] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476126:1478100 [1] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700705:2701502 [0] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->7 [2] 5/-1/-1->4->6 [3] 6/-1/-1->4->5 [4] 7/-1/-1->4->8 [5] 6/-1/-1->4->7 [6] 7/-1/-1->4->6 [7] -1/-1/-1->4->5 [8] 5/8/0->4->12 [9] -1/-1/-1->4->7 [10] 5/-1/-1->4->6 [11] 6/-1/-1->4->5 [12] 7/8/0->4->12 [13] 6/-1/-1->4->7 [14] 7/-1/-1->4->6 [15] -1/-1/-1->4->5
+1: jzxh252:2700705:2701502 [0] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700706:2701503 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->9 [2] 7/-1/-1->5->4 [3] 4/-1/-1->5->7 [4] -1/-1/-1->5->6 [5] 7/-1/-1->5->9 [6] -1/-1/-1->5->7 [7] 4/-1/-1->5->6 [8] 6/-1/-1->5->4 [9] 6/9/1->5->13 [10] 7/-1/-1->5->4 [11] 4/-1/-1->5->7 [12] -1/-1/-1->5->6 [13] 7/9/1->5->13 [14] -1/-1/-1->5->7 [15] 4/-1/-1->5->6
+1: jzxh252:2700707:2701505 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 4/-1/-1->6->10 [3] -1/-1/-1->6->4 [4] 5/-1/-1->6->7 [5] -1/-1/-1->6->4 [6] 4/-1/-1->6->10 [7] 5/-1/-1->6->7 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 4/10/2->6->14 [11] -1/-1/-1->6->4 [12] 5/-1/-1->6->7 [13] -1/-1/-1->6->4 [14] 4/10/2->6->14 [15] 5/-1/-1->6->7
+1: jzxh252:2700706:2701503 [1] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700708:2701504 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 4/-1/-1->7->6 [2] -1/-1/-1->7->5 [3] 5/-1/-1->7->11 [4] 6/-1/-1->7->4 [5] 4/-1/-1->7->5 [6] 5/-1/-1->7->4 [7] 6/-1/-1->7->11 [8] -1/-1/-1->7->6 [9] 4/-1/-1->7->6 [10] -1/-1/-1->7->5 [11] 5/11/3->7->15 [12] 6/-1/-1->7->4 [13] 4/-1/-1->7->5 [14] 5/-1/-1->7->4 [15] 6/11/3->7->15
+1: jzxh252:2700707:2701505 [2] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700708:2701504 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 00/0 : 14[2] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 04/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 00/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 08/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 12/0 : 14[2] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 04/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 08/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 00/0 : 5[1] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 12/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 03/0 : 5[1] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 04/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 00/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 07/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 03/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 08/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 08/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 04/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 11/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 07/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 12/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 08/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 12/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 08/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 11/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 15/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 11/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 12/0 : 9[1] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 12/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 15/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 15/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 00/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 03/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 04/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 07/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 08/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 11/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 12/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 15/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 00/0 : 4[0] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 03/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 04/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 07/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 01/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 02/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 11/0 : 4[0] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 05/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 15/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 06/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 09/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 08/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 10/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 01/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 01/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 13/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 11/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 02/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 02/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 14/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 05/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 12/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 05/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 06/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 15/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 06/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 09/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 10/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 09/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 13/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 14/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 10/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 02/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 13/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 06/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 14/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 09/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 10/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 13/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 14/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 01/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 02/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 05/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 06/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 13/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 10/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 01/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 02/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 14/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 09/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 05/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 06/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 10/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 09/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 13/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 10/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 13/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 14/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 14/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 03/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 07/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 15/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 11/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 11/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 03/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 07/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 15/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 15/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 02/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 01/0 : 15[3] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 06/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 10/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 01/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 05/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 05/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 09/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 14/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 09/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 02/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 09/0 : 7[3] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 10/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 13/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 14/0 : 10[2] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 14/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 13/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 13/0 : 7[3] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 06/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Connected all rings
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Connected all rings
+2: jzxh253:1476126:1478100 [1] NCCL INFO Connected all rings
+0: jzxh251:3958318:3959113 [1] NCCL INFO Connected all rings
+0: jzxh251:3958320:3959114 [3] NCCL INFO Connected all rings
+0: jzxh251:3958319:3959115 [2] NCCL INFO Connected all rings
+1: jzxh252:2700707:2701505 [2] NCCL INFO Connected all rings
+1: jzxh252:2700705:2701502 [0] NCCL INFO Connected all rings
+1: jzxh252:2700706:2701503 [1] NCCL INFO Connected all rings
+1: jzxh252:2700708:2701504 [3] NCCL INFO Connected all rings
+2: jzxh253:1476128:1478101 [3] NCCL INFO Connected all rings
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Connected all rings
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 02/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 10/0 : 8[0] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 10/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 01/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 09/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 01/0 : 6[2] -> 7[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 09/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Connected all rings
+3: jzxh254:1472011:1472811 [2] NCCL INFO Connected all rings
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Connected all rings
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 15/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 02/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 07/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 02/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Connected all rings
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 02/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 02/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 09/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 02/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 15/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 03/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 10/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 01/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 05/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 05/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 05/0 : 4[0] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 02/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 07/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 06/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 06/0 : 0[0] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 06/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 03/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 01/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 10/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 10/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 10/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 05/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 06/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 09/0 : 14[2] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 15/0 : 14[2] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 09/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 11/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 11/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 11/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 02/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 06/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 10/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 02/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 13/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 03/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 13/0 : 8[0] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 03/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 11/0 : 9[1] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 13/0 : 0[0] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 10/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 13/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 14/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 14/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 05/0 : 13[1] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 05/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 14/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 06/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 14/0 : 9[1] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 06/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 11/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 10/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 12/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 11/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 10/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 14/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 04/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 11/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 13/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 13/0 : 12[0] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 14/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 14/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 04/0 : 12[0] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 12/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958317:3959112 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472009:1472810 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2701502 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478098 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 01/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 02/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 01/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 02/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 02/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 01/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 04/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 03/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 04/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 04/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 05/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 03/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 05/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 05/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 06/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 05/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 05/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 06/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 05/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 06/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 06/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 10/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 06/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 09/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 06/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 10/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 12/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 11/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 10/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 09/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 09/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 11/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 12/0 : 11[3] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 13/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 11/0 : 6[2] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 12/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 12/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 13/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 14/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 13/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 13/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 13/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 13/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 13/0 : 14[2] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 14/0 : 15[3] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 14/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 14/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 02/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 14/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 14/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 14/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 02/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 03/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 02/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 02/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 05/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 03/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 05/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 06/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 05/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 06/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 05/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 10/0 : 15[3] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 10/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 11/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 06/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 13/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 10/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 14/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 11/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 11/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 13/0 : 11[3] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 14/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 13/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 00/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 06/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 14/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 04/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 10/0 : 7[3] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 11/0 : 7[3] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 13/0 : 7[3] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 14/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 07/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 00/0 : 7[3] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 04/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 08/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 07/0 : 7[3] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 08/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 12/0 : 15[3] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 12/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2701504 [3] NCCL INFO Channel 15/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1472813 [3] NCCL INFO Channel 15/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 00/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 01/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 04/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 07/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 00/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 08/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 00/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 03/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 08/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 09/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 03/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 12/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 07/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 12/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472011:1472811 [2] NCCL INFO Channel 15/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 07/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 08/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478101 [3] NCCL INFO Channel 15/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 11/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 08/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 08/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958318:3959113 [1] NCCL INFO Channel 15/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 09/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 00/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 11/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 12/0 : 10[2] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 12/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 01/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476127:1478099 [2] NCCL INFO Channel 15/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700706:2701503 [1] NCCL INFO Channel 15/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 04/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 07/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 08/0 : 6[2] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472010:1472812 [1] NCCL INFO Channel 15/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 08/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958319:3959115 [2] NCCL INFO Channel 15/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 11/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 09/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476126:1478100 [1] NCCL INFO Channel 15/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 12/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700707:2701505 [2] NCCL INFO Channel 15/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3959114 [3] NCCL INFO Connected all trees
+0: jzxh251:3958317:3959112 [0] NCCL INFO Connected all trees
+0: jzxh251:3958320:3959114 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958320:3959114 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958319:3959115 [2] NCCL INFO Connected all trees
+2: jzxh253:1476125:1478098 [0] NCCL INFO Connected all trees
+1: jzxh252:2700708:2701504 [3] NCCL INFO Connected all trees
+0: jzxh251:3958318:3959113 [1] NCCL INFO Connected all trees
+0: jzxh251:3958317:3959112 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958317:3959112 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958319:3959115 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958319:3959115 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958318:3959113 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958318:3959113 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476128:1478101 [3] NCCL INFO Connected all trees
+1: jzxh252:2700705:2701502 [0] NCCL INFO Connected all trees
+1: jzxh252:2700708:2701504 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476125:1478098 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476126:1478100 [1] NCCL INFO Connected all trees
+2: jzxh253:1476125:1478098 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700708:2701504 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476128:1478101 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476128:1478101 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700705:2701502 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700705:2701502 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476127:1478099 [2] NCCL INFO Connected all trees
+2: jzxh253:1476126:1478100 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476126:1478100 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700707:2701505 [2] NCCL INFO Connected all trees
+1: jzxh252:2700706:2701503 [1] NCCL INFO Connected all trees
+2: jzxh253:1476127:1478099 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476127:1478099 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700707:2701505 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700706:2701503 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700707:2701505 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700706:2701503 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472009:1472810 [0] NCCL INFO Connected all trees
+3: jzxh254:1472009:1472810 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472009:1472810 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472010:1472812 [1] NCCL INFO Connected all trees
+3: jzxh254:1472012:1472813 [3] NCCL INFO Connected all trees
+3: jzxh254:1472010:1472812 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472011:1472811 [2] NCCL INFO Connected all trees
+3: jzxh254:1472012:1472813 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472010:1472812 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472012:1472813 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472011:1472811 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472011:1472811 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476125:1478098 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh253:1476125:1478098 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh253:1476127:1478099 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh253:1476125:1478098 [0] NCCL INFO ncclCommInitRank comm 0x55705cd0ee30 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+2: jzxh253:1476126:1478100 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh253:1476128:1478101 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+2: jzxh253:1476127:1478099 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh253:1476126:1478100 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh253:1476128:1478101 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+2: jzxh253:1476127:1478099 [2] NCCL INFO ncclCommInitRank comm 0x555b4b77d2b0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+2: jzxh253:1476128:1478101 [3] NCCL INFO ncclCommInitRank comm 0x55c13cf8e7e0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+2: jzxh253:1476126:1478100 [1] NCCL INFO ncclCommInitRank comm 0x559d74834b90 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+1: jzxh252:2700707:2701505 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh252:2700707:2701505 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh252:2700707:2701505 [2] NCCL INFO ncclCommInitRank comm 0x563d58040fd0 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+1: jzxh252:2700706:2701503 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh252:2700705:2701502 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh252:2700706:2701503 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh252:2700705:2701502 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh252:2700706:2701503 [1] NCCL INFO ncclCommInitRank comm 0x55d9b6dce4a0 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+1: jzxh252:2700705:2701502 [0] NCCL INFO ncclCommInitRank comm 0x55691e321820 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+1: jzxh252:2700708:2701504 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+1: jzxh252:2700708:2701504 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+1: jzxh252:2700708:2701504 [3] NCCL INFO ncclCommInitRank comm 0x55e552323b50 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+0: jzxh251:3958318:3959113 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh251:3958318:3959113 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh251:3958318:3959113 [1] NCCL INFO ncclCommInitRank comm 0x561703831fa0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+0: jzxh251:3958320:3959114 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh251:3958320:3959114 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh251:3958320:3959114 [3] NCCL INFO ncclCommInitRank comm 0x5627d69ed2e0 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+0: jzxh251:3958317:3959112 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh251:3958317:3959112 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh251:3958317:3959112 [0] NCCL INFO ncclCommInitRank comm 0x55ad9c344130 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+0: jzxh251:3958319:3959115 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+0: jzxh251:3958319:3959115 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+0: jzxh251:3958319:3959115 [2] NCCL INFO ncclCommInitRank comm 0x557a67ebf3e0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+3: jzxh254:1472011:1472811 [2] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh254:1472011:1472811 [2] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh254:1472011:1472811 [2] NCCL INFO ncclCommInitRank comm 0x55ee68f2ae50 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+3: jzxh254:1472012:1472813 [3] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh254:1472012:1472813 [3] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh254:1472012:1472813 [3] NCCL INFO ncclCommInitRank comm 0x55e3cd6150e0 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+3: jzxh254:1472010:1472812 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh254:1472009:1472810 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
+3: jzxh254:1472010:1472812 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh254:1472009:1472810 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
+3: jzxh254:1472010:1472812 [1] NCCL INFO ncclCommInitRank comm 0x558d1e103f20 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+3: jzxh254:1472009:1472810 [0] NCCL INFO ncclCommInitRank comm 0x5603bb4b7f00 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0xfb9b4631e7b54d2e - Init COMPLETE
+2: jzxh253:1476126:1478130 [1] NCCL INFO Channel 04/1 : 9[1] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh253:1476126:1478130 [1] NCCL INFO Channel 05/1 : 9[1] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh253:1476125:1478131 [0] NCCL INFO Channel 04/1 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+2: jzxh253:1476125:1478131 [0] NCCL INFO Channel 05/1 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+2: jzxh253:1476128:1478132 [3] NCCL INFO Channel 04/1 : 11[3] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh253:1476128:1478132 [3] NCCL INFO Channel 05/1 : 11[3] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh253:1476127:1478133 [2] NCCL INFO Channel 04/1 : 10[2] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+2: jzxh253:1476127:1478133 [2] NCCL INFO Channel 05/1 : 10[2] -> 0[0] [send] via NET/IB/0(8)/GDRDMA/Shared
+1: jzxh252:2700707:2701534 [2] NCCL INFO Channel 12/1 : 6[2] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh252:2700707:2701534 [2] NCCL INFO Channel 13/1 : 6[2] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh252:2700705:2701535 [0] NCCL INFO Channel 12/1 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+1: jzxh252:2700706:2701536 [1] NCCL INFO Channel 12/1 : 5[1] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh252:2700705:2701535 [0] NCCL INFO Channel 13/1 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+1: jzxh252:2700706:2701536 [1] NCCL INFO Channel 13/1 : 5[1] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+0: jzxh251:3958319:3959146 [2] NCCL INFO Channel 00/1 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3959147 [1] NCCL INFO Channel 00/1 : 1[1] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700708:2701537 [3] NCCL INFO Channel 12/1 : 7[3] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+1: jzxh252:2700708:2701537 [3] NCCL INFO Channel 13/1 : 7[3] -> 0[0] [send] via NET/IB/0(4)/GDRDMA/Shared
+0: jzxh251:3958320:3959145 [3] NCCL INFO Channel 00/1 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 08/1 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 09/1 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958319:3959146 [2] NCCL INFO Channel 01/1 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3959147 [1] NCCL INFO Channel 01/1 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3959145 [3] NCCL INFO Channel 01/1 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1472842 [2] NCCL INFO Channel 08/1 : 14[2] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh254:1472010:1472843 [1] NCCL INFO Channel 08/1 : 13[1] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh254:1472011:1472842 [2] NCCL INFO Channel 09/1 : 14[2] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh254:1472010:1472843 [1] NCCL INFO Channel 09/1 : 13[1] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh254:1472012:1472844 [3] NCCL INFO Channel 08/1 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+3: jzxh254:1472009:1472845 [0] NCCL INFO Channel 08/1 : 12[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+3: jzxh254:1472009:1472845 [0] NCCL INFO Channel 09/1 : 12[0] -> 0[0] [send] via NET/IB/0/GDRDMA/Shared
+3: jzxh254:1472012:1472844 [3] NCCL INFO Channel 09/1 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 08/1 : 14[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 09/1 : 14[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 08/1 : 13[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 09/1 : 13[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 08/1 : 12[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 09/1 : 12[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 04/1 : 11[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 05/1 : 11[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 04/1 : 10[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 05/1 : 10[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 04/1 : 9[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 05/1 : 9[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 04/1 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 05/1 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 12/1 : 7[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 13/1 : 7[3] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 12/1 : 6[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 13/1 : 6[2] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 12/1 : 5[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 13/1 : 5[1] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 12/1 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: jzxh251:3958317:3959148 [0] NCCL INFO Channel 13/1 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA/Shared
+0: [2025-11-23 14:46:15,578] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:436] [PID:3958317] [RANK:0] gather_len_batches: [59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622]
+0: [2025-11-23 14:46:15,682] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:495] [PID:3958317] [RANK:0] sample_packing_eff_est across ranks: [0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812, 0.8715326189994812]
+0: [2025-11-23 14:46:15,699] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:127] [PID:3958317] [RANK:0] Maximum number of steps set at 2235
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+1: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+3: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+2: Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+0: [2025-11-23 14:46:22,541] [INFO] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:110] [PID:3958317] [RANK:0] Patched Trainer.evaluation_loop with nanmean loss calculation
+0: [2025-11-23 14:46:22,542] [INFO] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:164] [PID:3958317] [RANK:0] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
+3: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+1: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+0: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.79s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+2: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.72s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+1: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+3: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+2: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.90s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.33s/it]
+0: 
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  6.92s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.35s/it]
+0: [2025-11-23 14:46:43,232] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:3958317] [RANK:0] Converting modules to torch.bfloat16
+0: [2025-11-23 14:46:47,316] [INFO] [axolotl.train.save_initial_configs:416] [PID:3958317] [RANK:0] Pre-saving tokenizer to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5...
+0: [2025-11-23 14:46:48,113] [INFO] [axolotl.train.save_initial_configs:419] [PID:3958317] [RANK:0] Pre-saving model config to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5...
+0: [2025-11-23 14:46:48,124] [INFO] [axolotl.train.save_initial_configs:423] [PID:3958317] [RANK:0] Pre-saving processor to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5...
+0: [2025-11-23 14:46:51,062] [INFO] [axolotl.train.execute_training:203] [PID:3958317] [RANK:0] Starting trainer...
+0: [2025-11-23 14:50:55,455] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:436] [PID:3958317] [RANK:0] gather_len_batches: [59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622, 59622]
+0: Parameter Offload - Persistent parameters statistics: param_count = 479, numel = 768880
+0: jzxh251:3958317:3959998 [0] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958319:3960002 [2] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958320:3960011 [3] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958317:3960014 [0] NCCL INFO Using network IB
+0: jzxh251:3958318:3960003 [1] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958319:3960015 [2] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958319:3960015 [2] NCCL INFO Using network IB
+0: jzxh251:3958320:3960016 [3] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958320:3960016 [3] NCCL INFO Using network IB
+3: jzxh254:1472011:1473675 [2] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472012:1473680 [3] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472010:1473671 [1] NCCL INFO Comm config Blocking set to 1
+2: jzxh253:1476127:1478983 [2] NCCL INFO Comm config Blocking set to 1
+0: jzxh251:3958318:3960017 [1] NCCL INFO Using non-device net plugin version 0
+0: jzxh251:3958318:3960017 [1] NCCL INFO Using network IB
+2: jzxh253:1476128:1478980 [3] NCCL INFO Comm config Blocking set to 1
+2: jzxh253:1476126:1478972 [1] NCCL INFO Comm config Blocking set to 1
+2: jzxh253:1476125:1478970 [0] NCCL INFO Comm config Blocking set to 1
+2: jzxh253:1476127:1478985 [2] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476127:1478985 [2] NCCL INFO Using network IB
+2: jzxh253:1476126:1478987 [1] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476126:1478987 [1] NCCL INFO Using network IB
+2: jzxh253:1476125:1478988 [0] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476125:1478988 [0] NCCL INFO Using network IB
+3: jzxh254:1472009:1473667 [0] NCCL INFO Comm config Blocking set to 1
+3: jzxh254:1472011:1473682 [2] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472011:1473682 [2] NCCL INFO Using network IB
+3: jzxh254:1472010:1473684 [1] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472010:1473684 [1] NCCL INFO Using network IB
+3: jzxh254:1472012:1473683 [3] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472012:1473683 [3] NCCL INFO Using network IB
+2: jzxh253:1476128:1478986 [3] NCCL INFO Using non-device net plugin version 0
+2: jzxh253:1476128:1478986 [3] NCCL INFO Using network IB
+3: jzxh254:1472009:1473685 [0] NCCL INFO Using non-device net plugin version 0
+3: jzxh254:1472009:1473685 [0] NCCL INFO Using network IB
+1: jzxh252:2700705:2702382 [0] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700707:2702390 [2] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700708:2702393 [3] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700706:2702388 [1] NCCL INFO Comm config Blocking set to 1
+1: jzxh252:2700705:2702397 [0] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700705:2702397 [0] NCCL INFO Using network IB
+1: jzxh252:2700707:2702398 [2] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700707:2702398 [2] NCCL INFO Using network IB
+1: jzxh252:2700706:2702400 [1] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700706:2702400 [1] NCCL INFO Using network IB
+1: jzxh252:2700708:2702399 [3] NCCL INFO Using non-device net plugin version 0
+1: jzxh252:2700708:2702399 [3] NCCL INFO Using network IB
+2: jzxh253:1476126:1478987 [1] NCCL INFO ncclCommInitRank comm 0x1463a413d300 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init START
+2: jzxh253:1476128:1478986 [3] NCCL INFO ncclCommInitRank comm 0x151034113bc0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init START
+2: jzxh253:1476127:1478985 [2] NCCL INFO ncclCommInitRank comm 0x148618125040 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init START
+2: jzxh253:1476125:1478988 [0] NCCL INFO ncclCommInitRank comm 0x14687c132d80 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init START
+3: jzxh254:1472009:1473685 [0] NCCL INFO ncclCommInitRank comm 0x15272813ee40 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init START
+0: jzxh251:3958317:3960014 [0] NCCL INFO ncclCommInitRank comm 0x147c70132940 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init START
+0: jzxh251:3958318:3960017 [1] NCCL INFO ncclCommInitRank comm 0x14ceec116f00 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init START
+0: jzxh251:3958319:3960015 [2] NCCL INFO ncclCommInitRank comm 0x145828122fa0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init START
+0: jzxh251:3958320:3960016 [3] NCCL INFO ncclCommInitRank comm 0x14ef70130550 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init START
+3: jzxh254:1472010:1473684 [1] NCCL INFO ncclCommInitRank comm 0x15171c121dc0 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init START
+1: jzxh252:2700708:2702399 [3] NCCL INFO ncclCommInitRank comm 0x1460a413fc10 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init START
+3: jzxh254:1472011:1473682 [2] NCCL INFO ncclCommInitRank comm 0x145a0c1133c0 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init START
+3: jzxh254:1472012:1473683 [3] NCCL INFO ncclCommInitRank comm 0x14f628121240 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init START
+1: jzxh252:2700705:2702397 [0] NCCL INFO ncclCommInitRank comm 0x150c30122f80 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init START
+1: jzxh252:2700707:2702398 [2] NCCL INFO ncclCommInitRank comm 0x1528a013f400 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init START
+1: jzxh252:2700706:2702400 [1] NCCL INFO ncclCommInitRank comm 0x145ab8119b40 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init START
+2: jzxh253:1476126:1478987 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+2: jzxh253:1476126:1478987 [1] NCCL INFO NVLS multicast support is not available on dev 1
+3: jzxh254:1472009:1473685 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+3: jzxh254:1472009:1473685 [0] NCCL INFO NVLS multicast support is not available on dev 0
+0: jzxh251:3958317:3960014 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+0: jzxh251:3958317:3960014 [0] NCCL INFO NVLS multicast support is not available on dev 0
+1: jzxh252:2700708:2702399 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+1: jzxh252:2700708:2702399 [3] NCCL INFO NVLS multicast support is not available on dev 3
+2: jzxh253:1476125:1478988 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+2: jzxh253:1476125:1478988 [0] NCCL INFO NVLS multicast support is not available on dev 0
+2: jzxh253:1476128:1478986 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+2: jzxh253:1476128:1478986 [3] NCCL INFO NVLS multicast support is not available on dev 3
+3: jzxh254:1472011:1473682 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+3: jzxh254:1472011:1473682 [2] NCCL INFO NVLS multicast support is not available on dev 2
+2: jzxh253:1476127:1478985 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+2: jzxh253:1476127:1478985 [2] NCCL INFO NVLS multicast support is not available on dev 2
+1: jzxh252:2700705:2702397 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff
+3: jzxh254:1472010:1473684 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+1: jzxh252:2700705:2702397 [0] NCCL INFO NVLS multicast support is not available on dev 0
+3: jzxh254:1472010:1473684 [1] NCCL INFO NVLS multicast support is not available on dev 1
+3: jzxh254:1472012:1473683 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+3: jzxh254:1472012:1473683 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh251:3958319:3960015 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+0: jzxh251:3958319:3960015 [2] NCCL INFO NVLS multicast support is not available on dev 2
+1: jzxh252:2700706:2702400 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+1: jzxh252:2700706:2702400 [1] NCCL INFO NVLS multicast support is not available on dev 1
+1: jzxh252:2700707:2702398 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00000000,000000ff,ffff0000,00000000
+1: jzxh252:2700707:2702398 [2] NCCL INFO NVLS multicast support is not available on dev 2
+0: jzxh251:3958318:3960017 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ff000000,00000000,0000ffff,ff000000
+0: jzxh251:3958318:3960017 [1] NCCL INFO NVLS multicast support is not available on dev 1
+0: jzxh251:3958320:3960016 [3] NCCL INFO Setting affinity for GPU 3 to ffffff00,00000000,00000000,ffffff00,00000000,00000000
+0: jzxh251:3958320:3960016 [3] NCCL INFO NVLS multicast support is not available on dev 3
+0: jzxh251:3958320:3960016 [3] NCCL INFO comm 0x14ef70130550 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+0: jzxh251:3958319:3960015 [2] NCCL INFO comm 0x145828122fa0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+0: jzxh251:3958318:3960017 [1] NCCL INFO comm 0x14ceec116f00 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+0: jzxh251:3958317:3960014 [0] NCCL INFO comm 0x147c70132940 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+2: jzxh253:1476125:1478988 [0] NCCL INFO comm 0x14687c132d80 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+1: jzxh252:2700705:2702397 [0] NCCL INFO comm 0x150c30122f80 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+1: jzxh252:2700706:2702400 [1] NCCL INFO comm 0x145ab8119b40 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+1: jzxh252:2700708:2702399 [3] NCCL INFO comm 0x1460a413fc10 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+1: jzxh252:2700707:2702398 [2] NCCL INFO comm 0x1528a013f400 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+0: jzxh251:3958320:3960016 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 0/-1/-1->3->2 [2] -1/-1/-1->3->1 [3] 1/11/-1->3->-1 [4] 2/-1/-1->3->0 [5] 0/-1/-1->3->1 [6] 1/-1/-1->3->0 [7] 2/11/-1->3->-1 [8] -1/-1/-1->3->2 [9] 0/-1/-1->3->2 [10] -1/-1/-1->3->1 [11] 1/-1/-1->3->7 [12] 2/-1/-1->3->0 [13] 0/-1/-1->3->1 [14] 1/-1/-1->3->0 [15] 2/-1/-1->3->7
+0: jzxh251:3958319:3960015 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 0/10/-1->2->-1 [3] -1/-1/-1->2->0 [4] 1/-1/-1->2->3 [5] -1/-1/-1->2->0 [6] 0/10/-1->2->-1 [7] 1/-1/-1->2->3 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 0/-1/-1->2->6 [11] -1/-1/-1->2->0 [12] 1/-1/-1->2->3 [13] -1/-1/-1->2->0 [14] 0/-1/-1->2->6 [15] 1/-1/-1->2->3
+0: jzxh251:3958320:3960016 [3] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476126:1478987 [1] NCCL INFO comm 0x1463a413d300 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+2: jzxh253:1476128:1478986 [3] NCCL INFO comm 0x151034113bc0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+2: jzxh253:1476127:1478985 [2] NCCL INFO comm 0x148618125040 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+2: jzxh253:1476125:1478988 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->11 [2] 9/-1/-1->8->10 [3] 10/-1/-1->8->9 [4] 11/4/12->8->0 [5] 10/-1/-1->8->11 [6] 11/-1/-1->8->10 [7] -1/-1/-1->8->9 [8] 9/-1/-1->8->4 [9] -1/-1/-1->8->11 [10] 9/-1/-1->8->10 [11] 10/-1/-1->8->9 [12] 11/-1/-1->8->4 [13] 10/-1/-1->8->11 [14] 11/-1/-1->8->10 [15] -1/-1/-1->8->9
+2: jzxh253:1476125:1478988 [0] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700705:2702397 [0] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->7 [2] 5/-1/-1->4->6 [3] 6/-1/-1->4->5 [4] 7/-1/-1->4->8 [5] 6/-1/-1->4->7 [6] 7/-1/-1->4->6 [7] -1/-1/-1->4->5 [8] 5/8/0->4->12 [9] -1/-1/-1->4->7 [10] 5/-1/-1->4->6 [11] 6/-1/-1->4->5 [12] 7/8/0->4->12 [13] 6/-1/-1->4->7 [14] 7/-1/-1->4->6 [15] -1/-1/-1->4->5
+1: jzxh252:2700707:2702398 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 4/-1/-1->6->10 [3] -1/-1/-1->6->4 [4] 5/-1/-1->6->7 [5] -1/-1/-1->6->4 [6] 4/-1/-1->6->10 [7] 5/-1/-1->6->7 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 4/10/2->6->14 [11] -1/-1/-1->6->4 [12] 5/-1/-1->6->7 [13] -1/-1/-1->6->4 [14] 4/10/2->6->14 [15] 5/-1/-1->6->7
+0: jzxh251:3958318:3960017 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/9/-1->1->-1 [2] 3/-1/-1->1->0 [3] 0/-1/-1->1->3 [4] -1/-1/-1->1->2 [5] 3/9/-1->1->-1 [6] -1/-1/-1->1->3 [7] 0/-1/-1->1->2 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->5 [10] 3/-1/-1->1->0 [11] 0/-1/-1->1->3 [12] -1/-1/-1->1->2 [13] 3/-1/-1->1->5 [14] -1/-1/-1->1->3 [15] 0/-1/-1->1->2
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 00/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958319:3960015 [2] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958318:3960017 [1] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 01/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 02/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 03/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+2: jzxh253:1476126:1478987 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/5/13->9->1 [2] 11/-1/-1->9->8 [3] 8/-1/-1->9->11 [4] -1/-1/-1->9->10 [5] 11/5/13->9->1 [6] -1/-1/-1->9->11 [7] 8/-1/-1->9->10 [8] 10/-1/-1->9->8 [9] 10/-1/-1->9->5 [10] 11/-1/-1->9->8 [11] 8/-1/-1->9->11 [12] -1/-1/-1->9->10 [13] 11/-1/-1->9->5 [14] -1/-1/-1->9->11 [15] 8/-1/-1->9->10
+2: jzxh253:1476126:1478987 [1] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476128:1478986 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 8/-1/-1->11->10 [2] -1/-1/-1->11->9 [3] 9/7/15->11->3 [4] 10/-1/-1->11->8 [5] 8/-1/-1->11->9 [6] 9/-1/-1->11->8 [7] 10/7/15->11->3 [8] -1/-1/-1->11->10 [9] 8/-1/-1->11->10 [10] -1/-1/-1->11->9 [11] 9/-1/-1->11->7 [12] 10/-1/-1->11->8 [13] 8/-1/-1->11->9 [14] 9/-1/-1->11->8 [15] 10/-1/-1->11->7
+1: jzxh252:2700708:2702399 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 4/-1/-1->7->6 [2] -1/-1/-1->7->5 [3] 5/-1/-1->7->11 [4] 6/-1/-1->7->4 [5] 4/-1/-1->7->5 [6] 5/-1/-1->7->4 [7] 6/-1/-1->7->11 [8] -1/-1/-1->7->6 [9] 4/-1/-1->7->6 [10] -1/-1/-1->7->5 [11] 5/11/3->7->15 [12] 6/-1/-1->7->4 [13] 4/-1/-1->7->5 [14] 5/-1/-1->7->4 [15] 6/11/3->7->15
+1: jzxh252:2700706:2702400 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->9 [2] 7/-1/-1->5->4 [3] 4/-1/-1->5->7 [4] -1/-1/-1->5->6 [5] 7/-1/-1->5->9 [6] -1/-1/-1->5->7 [7] 4/-1/-1->5->6 [8] 6/-1/-1->5->4 [9] 6/9/1->5->13 [10] 7/-1/-1->5->4 [11] 4/-1/-1->5->7 [12] -1/-1/-1->5->6 [13] 7/9/1->5->13 [14] -1/-1/-1->5->7 [15] 4/-1/-1->5->6
+1: jzxh252:2700707:2702398 [2] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700705:2702397 [0] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700708:2702399 [3] NCCL INFO P2P Chunksize set to 131072
+1: jzxh252:2700706:2702400 [1] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+2: jzxh253:1476127:1478985 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 [2] 8/6/14->10->2 [3] -1/-1/-1->10->8 [4] 9/-1/-1->10->11 [5] -1/-1/-1->10->8 [6] 8/6/14->10->2 [7] 9/-1/-1->10->11 [8] 11/-1/-1->10->9 [9] 11/-1/-1->10->9 [10] 8/-1/-1->10->6 [11] -1/-1/-1->10->8 [12] 9/-1/-1->10->11 [13] -1/-1/-1->10->8 [14] 8/-1/-1->10->6 [15] 9/-1/-1->10->11
+2: jzxh253:1476128:1478986 [3] NCCL INFO P2P Chunksize set to 131072
+2: jzxh253:1476127:1478985 [2] NCCL INFO P2P Chunksize set to 131072
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 05/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 06/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 07/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 08/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 09/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 10/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 11/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/16 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 13/16 :    0   3   2   5   4   7   6   9   8  11  10  13  12  15  14   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 14/16 :    0   3   6   5   4   7  10   9   8  11  14  13  12  15   2   1
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 15/16 :    0   1   2   7   4   5   6  11   8   9  10  15  12  13  14   3
+0: jzxh251:3958317:3960014 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->3 [2] 1/-1/-1->0->2 [3] 2/-1/-1->0->1 [4] 3/8/-1->0->-1 [5] 2/-1/-1->0->3 [6] 3/-1/-1->0->2 [7] -1/-1/-1->0->1 [8] 1/-1/-1->0->4 [9] -1/-1/-1->0->3 [10] 1/-1/-1->0->2 [11] 2/-1/-1->0->1 [12] 3/-1/-1->0->4 [13] 2/-1/-1->0->3 [14] 3/-1/-1->0->2 [15] -1/-1/-1->0->1
+0: jzxh251:3958317:3960014 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472012:1473683 [3] NCCL INFO comm 0x14f628121240 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0
+3: jzxh254:1472010:1473684 [1] NCCL INFO comm 0x15171c121dc0 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0
+3: jzxh254:1472011:1473682 [2] NCCL INFO comm 0x145a0c1133c0 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0
+3: jzxh254:1472012:1473683 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 12/-1/-1->15->14 [2] -1/-1/-1->15->13 [3] 13/-1/-1->15->11 [4] 14/-1/-1->15->12 [5] 12/-1/-1->15->13 [6] 13/-1/-1->15->12 [7] 14/-1/-1->15->11 [8] -1/-1/-1->15->14 [9] 12/-1/-1->15->14 [10] -1/-1/-1->15->13 [11] 13/7/-1->15->-1 [12] 14/-1/-1->15->12 [13] 12/-1/-1->15->13 [14] 13/-1/-1->15->12 [15] 14/7/-1->15->-1
+3: jzxh254:1472011:1473682 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 12/-1/-1->14->10 [3] -1/-1/-1->14->12 [4] 13/-1/-1->14->15 [5] -1/-1/-1->14->12 [6] 12/-1/-1->14->10 [7] 13/-1/-1->14->15 [8] 15/-1/-1->14->13 [9] 15/-1/-1->14->13 [10] 12/6/-1->14->-1 [11] -1/-1/-1->14->12 [12] 13/-1/-1->14->15 [13] -1/-1/-1->14->12 [14] 12/6/-1->14->-1 [15] 13/-1/-1->14->15
+3: jzxh254:1472012:1473683 [3] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472010:1473684 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->9 [2] 15/-1/-1->13->12 [3] 12/-1/-1->13->15 [4] -1/-1/-1->13->14 [5] 15/-1/-1->13->9 [6] -1/-1/-1->13->15 [7] 12/-1/-1->13->14 [8] 14/-1/-1->13->12 [9] 14/5/-1->13->-1 [10] 15/-1/-1->13->12 [11] 12/-1/-1->13->15 [12] -1/-1/-1->13->14 [13] 15/5/-1->13->-1 [14] -1/-1/-1->13->15 [15] 12/-1/-1->13->14
+3: jzxh254:1472011:1473682 [2] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472010:1473684 [1] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472009:1473685 [0] NCCL INFO comm 0x15272813ee40 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0
+3: jzxh254:1472009:1473685 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->15 [2] 13/-1/-1->12->14 [3] 14/-1/-1->12->13 [4] 15/-1/-1->12->8 [5] 14/-1/-1->12->15 [6] 15/-1/-1->12->14 [7] -1/-1/-1->12->13 [8] 13/4/-1->12->-1 [9] -1/-1/-1->12->15 [10] 13/-1/-1->12->14 [11] 14/-1/-1->12->13 [12] 15/4/-1->12->-1 [13] 14/-1/-1->12->15 [14] 15/-1/-1->12->14 [15] -1/-1/-1->12->13
+3: jzxh254:1472009:1473685 [0] NCCL INFO P2P Chunksize set to 131072
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 00/0 : 14[2] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 00/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 00/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 03/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 04/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 04/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 04/0 : 13[1] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 08/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 08/0 : 6[2] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 07/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 08/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 12/0 : 14[2] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 08/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 12/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 08/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 12/0 : 10[2] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 00/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 11/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 11/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 03/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 12/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 12/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 04/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 15/0 : 13[1] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 15/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 07/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 08/0 : 5[1] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 11/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 12/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 15/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [send] via NET/IB/0(8)/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 00/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 04/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 08/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 12/0 : 11[3] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 00/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [send] via NET/IB/0(12)/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 03/0 : 12[0] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 08/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 04/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 11/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 07/0 : 12[0] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 12/0 : 8[0] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 15/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 08/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 00/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 11/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 12/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 15/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 04/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 08/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 12/0 : 7[3] -> 8[0] [send] via NET/IB/0(4)/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 00/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 02/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 03/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 01/0 : 12[0] -> 15[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 02/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 04/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 05/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 06/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 07/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 06/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 09/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 09/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 4[0] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 10/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 10/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 13/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 11/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 13/0 : 8[0] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 14/0 : 12[0] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 00/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 14/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 08/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 15/0 : 4[0] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/0 : 15[3] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[0] [send] via NET/IB/0(0)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 01/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 02/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [send] via NET/IB/2(10)/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 05/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 06/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 02/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 06/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 09/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 10/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 14/0 : 11[3] -> 14[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 10/0 : 4[0] -> 7[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 13/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 14/0 : 4[0] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 01/0 : 0[0] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 02/0 : 0[0] -> 3[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [send] via NET/IB/1(9)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 05/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 02/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 06/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 06/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 10/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 14/0 : 7[3] -> 10[2] [send] via NET/IB/2(6)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 09/0 : 0[0] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 10/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 01/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 13/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 05/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 09/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 14/0 : 0[0] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 13/0 : 10[2] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [send] via NET/IB/1(13)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [send] via NET/IB/2(14)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 01/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 02/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 02/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 02/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 06/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 06/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 10/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 10/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 14/0 : 15[3] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 14/0 : 3[3] -> 6[2] [send] via NET/IB/2(2)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 05/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 06/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 09/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 01/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 10/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 05/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 13/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 14/0 : 13[1] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 09/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 13/0 : 6[2] -> 9[1] [send] via NET/IB/1(5)/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 01/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 01/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 05/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 05/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 09/0 : 9[1] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 09/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 09/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 13/0 : 14[2] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 13/0 : 2[2] -> 5[1] [send] via NET/IB/1(1)/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 10/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 13/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 14/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [send] via NET/IB/3(7)/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 11/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [send] via NET/IB/3(11)/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 15/0 : 6[2] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 01/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 11/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 15/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 02/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 13/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 05/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 06/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 02/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 09/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 06/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 10/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 13/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 5[1] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 14/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 03/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 07/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 11/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 15/0 : 2[2] -> 7[3] [send] via NET/IB/3(3)/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 03/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 07/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 14/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 01/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 03/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 05/0 : 7[3] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 09/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 03/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 07/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 07/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 11/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 11/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 15/0 : 14[2] -> 3[3] [send] via NET/IB/3(15)/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 15/0 : 10[2] -> 15[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 13/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 03/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 07/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 15/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 11/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 15/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 02/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 01/0 : 15[3] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 06/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 10/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 05/0 : 15[3] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 14/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 09/0 : 15[3] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 13/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 10/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 14/0 : 10[2] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Connected all rings
+0: jzxh251:3958320:3960016 [3] NCCL INFO Connected all rings
+0: jzxh251:3958317:3960014 [0] NCCL INFO Connected all rings
+0: jzxh251:3958318:3960017 [1] NCCL INFO Connected all rings
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Connected all rings
+3: jzxh254:1472010:1473684 [1] NCCL INFO Connected all rings
+2: jzxh253:1476128:1478986 [3] NCCL INFO Connected all rings
+2: jzxh253:1476127:1478985 [2] NCCL INFO Connected all rings
+2: jzxh253:1476125:1478988 [0] NCCL INFO Connected all rings
+1: jzxh252:2700705:2702397 [0] NCCL INFO Connected all rings
+1: jzxh252:2700706:2702400 [1] NCCL INFO Connected all rings
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 02/0 : 4[0] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Connected all rings
+3: jzxh254:1472011:1473682 [2] NCCL INFO Connected all rings
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 02/0 : 12[0] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Connected all rings
+1: jzxh252:2700707:2702398 [2] NCCL INFO Connected all rings
+2: jzxh253:1476126:1478987 [1] NCCL INFO Connected all rings
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 10/0 : 12[0] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 10/0 : 4[0] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 10/0 : 8[0] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 01/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 01/0 : 6[2] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 01/0 : 13[1] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 07/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 09/0 : 13[1] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 09/0 : 9[1] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 09/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 02/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 07/0 : 14[2] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 15/0 : 6[2] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 09/0 : 14[2] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 09/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 02/0 : 12[0] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 15/0 : 14[2] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 02/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 05/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 15/0 : 10[2] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 03/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 01/0 : 5[1] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 02/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 05/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 05/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 5[1] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 02/0 : 9[1] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 06/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 06/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 03/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 02/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 06/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 10/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 02/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 10/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 10/0 : 12[0] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 03/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 05/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 03/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 11/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 11/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 05/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 11/0 : 12[0] -> 14[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 06/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 06/0 : 5[1] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 05/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 06/0 : 9[1] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 13/0 : 8[0] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 13/0 : 0[0] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 10/0 : 13[1] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 13/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 10/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 14/0 : 0[0] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 14/0 : 8[0] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 06/0 : 4[0] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 11/0 : 13[1] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 10/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 14/0 : 12[0] -> 14[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 11/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 13/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 10/0 : 4[0] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/0 : 0[0] -> 3[3] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 14/0 : 13[1] -> 15[3] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 5[1] -> 7[3] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 8[0] -> 11[3] via P2P/CUMEM
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/0 : 0[0] -> 3[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 11/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 12/0 : 8[0] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 13/0 : 4[0] -> 6[2] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 14/0 : 5[1] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 04/0 : 12[0] -> 15[3] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 06/0 : 10[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 12/0 : 12[0] -> 15[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 11/0 : 9[1] -> 11[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 13/0 : 9[1] -> 11[3] via P2P/CUMEM
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 14/0 : 4[0] -> 6[2] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 14/0 : 9[1] -> 11[3] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 1[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 01/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 05/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 9[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 5[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 09/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 04/0 : 4[0] -> 7[3] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 8[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 13[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 11[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 5[1] -> 13[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 4[0] -> 7[3] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 2[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 02/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 06/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [receive] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 3[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 0[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 6[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 03/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [send] via NET/IB/1/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 00/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 07/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 04/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 7[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 4[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 14[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 6[2] -> 14[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 15[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 12[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 7[3] -> 15[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472009:1473685 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [send] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 4[0] -> 12[0] [send] via NET/IB/0/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [receive] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+1: jzxh252:2700705:2702397 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 05/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 01/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 05/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 08/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 10/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+0: jzxh251:3958317:3960014 [0] NCCL INFO Channel 12/0 : 4[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 09/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 14/0 : 6[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 13/0 : 5[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 12[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 04/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 14[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 08/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 13[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
+2: jzxh253:1476125:1478988 [0] NCCL INFO Channel 12/0 : 8[0] -> 4[0] [send] via NET/IB/0/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 02/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 01/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 10/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 02/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 03/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 14/0 : 10[2] -> 6[2] [send] via NET/IB/2/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 05/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 09/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 13/0 : 9[1] -> 5[1] [send] via NET/IB/1/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 02/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 03/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 05/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 05/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 06/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 05/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 07/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 06/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 03/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 07/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 10/0 : 14[2] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 06/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 10/0 : 10[2] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 11/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 15/0 : 7[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 11/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 11/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 10/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 15[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 13/0 : 14[2] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 11/0 : 6[2] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 11/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 15/0 : 11[3] -> 7[3] [send] via NET/IB/3/GDRDMA
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 13/0 : 10[2] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 01/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 01/0 : 7[3] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 01/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 14/0 : 14[2] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 14/0 : 10[2] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 13/0 : 6[2] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 04/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 04/0 : 15[3] -> 12[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 04/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 12/0 : 3[3] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 05/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 14/0 : 6[2] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 05/0 : 15[3] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 13/0 : 3[3] -> 0[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 05/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 06/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 06/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 14/0 : 3[3] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 09/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 09/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 02/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 06/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 12/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 12/0 : 11[3] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 13/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 13/0 : 11[3] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 09/0 : 7[3] -> 4[0] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 12/0 : 7[3] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 13/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 14/0 : 15[3] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 14/0 : 11[3] -> 8[0] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 05/0 : 3[3] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 02/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 14/0 : 7[3] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 03/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 02/0 : 7[3] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 06/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 03/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 05/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 02/0 : 11[3] -> 9[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 10/0 : 3[3] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 05/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 06/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 06/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 10/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 05/0 : 11[3] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 10/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 11/0 : 15[3] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 06/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 13/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 11/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 10/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 14/0 : 15[3] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 13/0 : 7[3] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 11/0 : 11[3] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 14/0 : 7[3] -> 5[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 00/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 13/0 : 11[3] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 04/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 14/0 : 11[3] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 00/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 07/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 04/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 08/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 07/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 12/0 : 15[3] -> 14[2] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 11/0 : 3[3] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 13/0 : 3[3] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 08/0 : 7[3] -> 6[2] via P2P/CUMEM
+3: jzxh254:1472012:1473683 [3] NCCL INFO Channel 15/0 : 15[3] -> 14[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 14/0 : 3[3] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 08/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 12/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 12/0 : 2[2] -> 1[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 12/0 : 11[3] -> 10[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 00/0 : 14[2] -> 13[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM
+1: jzxh252:2700708:2702399 [3] NCCL INFO Channel 15/0 : 7[3] -> 6[2] via P2P/CUMEM
+0: jzxh251:3958319:3960015 [2] NCCL INFO Channel 15/0 : 2[2] -> 1[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 01/0 : 14[2] -> 13[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 04/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476128:1478986 [3] NCCL INFO Channel 15/0 : 11[3] -> 10[2] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 07/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 00/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 08/0 : 14[2] -> 13[1] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 01/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 09/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 04/0 : 6[2] -> 5[1] via P2P/CUMEM
+0: jzxh251:3958320:3960016 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 12/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 00/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/CUMEM
+3: jzxh254:1472011:1473682 [2] NCCL INFO Channel 15/0 : 14[2] -> 13[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 07/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 03/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 07/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 08/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 08/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 09/0 : 6[2] -> 5[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 08/0 : 5[1] -> 4[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 00/0 : 13[1] -> 12[0] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 08/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 11/0 : 5[1] -> 4[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 09/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 12/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 12/0 : 10[2] -> 9[1] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 11/0 : 9[1] -> 8[0] via P2P/CUMEM
+1: jzxh252:2700707:2702398 [2] NCCL INFO Channel 15/0 : 6[2] -> 5[1] via P2P/CUMEM
+2: jzxh253:1476126:1478987 [1] NCCL INFO Channel 15/0 : 9[1] -> 8[0] via P2P/CUMEM
+2: jzxh253:1476127:1478985 [2] NCCL INFO Channel 15/0 : 10[2] -> 9[1] via P2P/CUMEM
+1: jzxh252:2700706:2702400 [1] NCCL INFO Channel 15/0 : 5[1] -> 4[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 03/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 07/0 : 13[1] -> 12[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 08/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 11/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472010:1473684 [1] NCCL INFO Channel 15/0 : 13[1] -> 12[0] via P2P/CUMEM
+0: jzxh251:3958318:3960017 [1] NCCL INFO Channel 15/0 : 1[1] -> 0[0] via P2P/CUMEM
+3: jzxh254:1472009:1473685 [0] NCCL INFO Connected all trees
+3: jzxh254:1472009:1473685 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472012:1473683 [3] NCCL INFO Connected all trees
+3: jzxh254:1472009:1473685 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958317:3960014 [0] NCCL INFO Connected all trees
+3: jzxh254:1472012:1473683 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472012:1473683 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958320:3960016 [3] NCCL INFO Connected all trees
+3: jzxh254:1472010:1473684 [1] NCCL INFO Connected all trees
+3: jzxh254:1472011:1473682 [2] NCCL INFO Connected all trees
+0: jzxh251:3958320:3960016 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958320:3960016 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958318:3960017 [1] NCCL INFO Connected all trees
+0: jzxh251:3958319:3960015 [2] NCCL INFO Connected all trees
+3: jzxh254:1472010:1473684 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476125:1478988 [0] NCCL INFO Connected all trees
+0: jzxh251:3958318:3960017 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958319:3960015 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958318:3960017 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+0: jzxh251:3958319:3960015 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472010:1473684 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+3: jzxh254:1472011:1473682 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+3: jzxh254:1472011:1473682 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700705:2702397 [0] NCCL INFO Connected all trees
+1: jzxh252:2700708:2702399 [3] NCCL INFO Connected all trees
+0: jzxh251:3958317:3960014 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+0: jzxh251:3958317:3960014 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476125:1478988 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476125:1478988 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476128:1478986 [3] NCCL INFO Connected all trees
+2: jzxh253:1476128:1478986 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476128:1478986 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476127:1478985 [2] NCCL INFO Connected all trees
+1: jzxh252:2700708:2702399 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700705:2702397 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700708:2702399 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700705:2702397 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476126:1478987 [1] NCCL INFO Connected all trees
+1: jzxh252:2700707:2702398 [2] NCCL INFO Connected all trees
+1: jzxh252:2700706:2702400 [1] NCCL INFO Connected all trees
+1: jzxh252:2700707:2702398 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700707:2702398 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476127:1478985 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476126:1478987 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+2: jzxh253:1476127:1478985 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+1: jzxh252:2700706:2702400 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+1: jzxh252:2700706:2702400 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476126:1478987 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 2 p2p channels per peer
+2: jzxh253:1476127:1478985 [2] NCCL INFO ncclCommInitRank comm 0x148618125040 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init COMPLETE
+2: jzxh253:1476125:1478988 [0] NCCL INFO ncclCommInitRank comm 0x14687c132d80 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init COMPLETE
+3: jzxh254:1472012:1473683 [3] NCCL INFO ncclCommInitRank comm 0x14f628121240 rank 15 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init COMPLETE
+3: jzxh254:1472010:1473684 [1] NCCL INFO ncclCommInitRank comm 0x15171c121dc0 rank 13 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init COMPLETE
+3: jzxh254:1472011:1473682 [2] NCCL INFO ncclCommInitRank comm 0x145a0c1133c0 rank 14 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init COMPLETE
+3: jzxh254:1472009:1473685 [0] NCCL INFO ncclCommInitRank comm 0x15272813ee40 rank 12 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init COMPLETE
+2: jzxh253:1476128:1478986 [3] NCCL INFO ncclCommInitRank comm 0x151034113bc0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init COMPLETE
+2: jzxh253:1476126:1478987 [1] NCCL INFO ncclCommInitRank comm 0x1463a413d300 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init COMPLETE
+1: jzxh252:2700707:2702398 [2] NCCL INFO ncclCommInitRank comm 0x1528a013f400 rank 6 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init COMPLETE
+1: jzxh252:2700705:2702397 [0] NCCL INFO ncclCommInitRank comm 0x150c30122f80 rank 4 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init COMPLETE
+1: jzxh252:2700706:2702400 [1] NCCL INFO ncclCommInitRank comm 0x145ab8119b40 rank 5 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init COMPLETE
+1: jzxh252:2700708:2702399 [3] NCCL INFO ncclCommInitRank comm 0x1460a413fc10 rank 7 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init COMPLETE
+0: jzxh251:3958318:3960017 [1] NCCL INFO ncclCommInitRank comm 0x14ceec116f00 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 2c000 commId 0x8447d14ea06660bf - Init COMPLETE
+0: jzxh251:3958320:3960016 [3] NCCL INFO ncclCommInitRank comm 0x14ef70130550 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId ad000 commId 0x8447d14ea06660bf - Init COMPLETE
+0: jzxh251:3958317:3960014 [0] NCCL INFO ncclCommInitRank comm 0x147c70132940 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x8447d14ea06660bf - Init COMPLETE
+0: jzxh251:3958319:3960015 [2] NCCL INFO ncclCommInitRank comm 0x145828122fa0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 9d000 commId 0x8447d14ea06660bf - Init COMPLETE
+0: {'loss': 0.7349, 'grad_norm': 2.5536512675043928, 'learning_rate': 9.05e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.0}
+0: 
  0%|          | 0/2235 [00:00<?, ?it/s]
  0%|          | 1/2235 [00:43<27:00:03, 43.51s/it]
  0%|          | 2/2235 [00:56<15:45:24, 25.40s/it]
  0%|          | 3/2235 [01:06<11:32:30, 18.62s/it]
  0%|          | 4/2235 [01:17<9:32:55, 15.41s/it] 
  0%|          | 5/2235 [01:27<8:26:30, 13.63s/it]
  0%|          | 6/2235 [01:38<7:46:49, 12.57s/it]
  0%|          | 7/2235 [01:48<7:21:44, 11.90s/it]
  0%|          | 8/2235 [01:59<7:04:05, 11.43s/it]
  0%|          | 9/2235 [02:09<6:51:58, 11.10s/it]
  0%|          | 10/2235 [02:20<6:46:59, 10.97s/it]
                                                   

  0%|          | 10/2235 [02:20<6:46:59, 10.97s/it]
  0%|          | 11/2235 [02:30<6:40:00, 10.79s/it]
  1%|          | 12/2235 [02:41<6:35:18, 10.67s/it]
  1%|          | 13/2235 [02:51<6:31:10, 10.56s/it]
  1%|          | 14/2235 [03:01<6:27:59, 10.48s/it]
  1%|          | 15/2235 [03:11<6:26:09, 10.44s/it]
  1%|          | 16/2235 [03:22<6:25:09, 10.41s/it]
  1%|          | 17/2235 [03:32<6:24:51, 10.41s/it
+0: {'loss': 0.6665, 'grad_norm': 1.3114825660942901, 'learning_rate': 1.3550000000000002e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.01}
+0: {'loss': 0.6297, 'grad_norm': 0.9497310104582564, 'learning_rate': 1.805e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.01}
+0: ]
  1%|          | 18/2235 [03:43<6:23:58, 10.39s/it]
  1%|          | 19/2235 [03:53<6:24:17, 10.40s/it]
  1%|          | 20/2235 [04:03<6:24:56, 10.43s/it]
                                                   

  1%|          | 20/2235 [04:04<6:24:56, 10.43s/it]
  1%|          | 21/2235 [04:14<6:25:14, 10.44s/it]
  1%|          | 22/2235 [04:24<6:24:39, 10.43s/it]
  1%|          | 23/2235 [04:35<6:24:15, 10.42s/it]
  1%|          | 24/2235 [04:45<6:23:40, 10.41s/it]
  1%|          | 25/2235 [04:56<6:22:51, 10.39s/it]
  1%|          | 26/2235 [05:06<6:22:37, 10.39s/it]
  1%|          | 27/2235 [05:16<6:22:53, 10.40s/it]
  1%|▏         | 28/2235 [05:27<6:21:11, 10.36s/it]
  1%|▏         | 29/2235 [05:37<6:18:42, 10.30s/it]
  1%|▏         | 30/2235 [05:47<6:17:22, 10.27s/it]
                                                   

  1%|▏         | 30/2235 [05:47<6:17:22, 10.27s/it]
  1%|▏         | 31/2235 [05:57<6:16:13, 10.24s/it]
  1%|▏         | 32/2235 [06:07<6:15:29, 10.23s/it]
  1%|▏         | 3
+0: {'loss': 0.6001, 'grad_norm': 0.8126881300164254, 'learning_rate': 2.2550000000000004e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.01}
+0: 3/2235 [06:18<6:14:57, 10.22s/it]
  2%|▏         | 34/2235 [06:28<6:14:47, 10.22s/it]
  2%|▏         | 35/2235 [06:38<6:14:43, 10.22s/it]
  2%|▏         | 36/2235 [06:48<6:14:41, 10.22s/it]
  2%|▏         | 37/2235 [06:58<6:14:11, 10.21s/it]
  2%|▏         | 38/2235 [07:09<6:14:19, 10.22s/it]
  2%|▏         | 39/2235 [07:19<6:15:08, 10.25s/it]
  2%|▏         | 40/2235 [07:29<6:15:31, 10.27s/it]
                                                   

  2%|▏         | 40/2235 [07:29<6:15:31, 10.27s/it]
  2%|▏         | 41/2235 [07:39<6:14:56, 10.25s/it]
  2%|▏         | 42/2235 [07:50<6:14:20, 10.24s/it]
  2%|▏         | 43/2235 [08:00<6:13:58, 10.24s/it]
  2%|▏         | 44/2235 [08:10<6:13:57, 10.24s/it]
  2%|▏         | 45/2235 [08:20<6:13:07, 10.22s/it]
  2%|▏         | 46/2235 [08:31<6:12:29, 10.21s/it]
  2%|▏         | 47/2235 [08:41<6:12:09, 10.21s/it]
  2%|▏         | 48/2235 [08:51<6:12:02, 10.21s/it]
  2%|▏         | 49/2235 [09:01<6:11:58, 10.21s/it]
  2%|▏         | 
+0: {'loss': 0.5827, 'grad_norm': 0.9895357438740783, 'learning_rate': 2.7050000000000004e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.01}
+0: {'loss': 0.5647, 'grad_norm': 0.7485498869460085, 'learning_rate': 3.1550000000000003e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.02}
+0: 50/2235 [09:11<6:12:14, 10.22s/it]
                                                   

  2%|▏         | 50/2235 [09:11<6:12:14, 10.22s/it]
  2%|▏         | 51/2235 [09:22<6:11:57, 10.22s/it]
  2%|▏         | 52/2235 [09:32<6:11:51, 10.22s/it]
  2%|▏         | 53/2235 [09:42<6:12:53, 10.25s/it]
  2%|▏         | 54/2235 [09:52<6:12:55, 10.26s/it]
  2%|▏         | 55/2235 [10:03<6:12:50, 10.26s/it]
  3%|▎         | 56/2235 [10:13<6:11:17, 10.22s/it]
  3%|▎         | 57/2235 [10:23<6:09:47, 10.19s/it]
  3%|▎         | 58/2235 [10:33<6:09:04, 10.17s/it]
  3%|▎         | 59/2235 [10:43<6:08:56, 10.17s/it]
  3%|▎         | 60/2235 [10:53<6:07:50, 10.15s/it]
                                                   

  3%|▎         | 60/2235 [10:53<6:07:50, 10.15s/it]
  3%|▎         | 61/2235 [11:03<6:06:45, 10.12s/it]
  3%|▎         | 62/2235 [11:13<6:05:53, 10.10s/it]
  3%|▎         | 63/2235 [11:24<6:06:17, 10.12s/it]
  3%|▎         | 64/2235 [11:34<6:05:43, 10.11s/it]
  3%|▎         | 
+0: {'loss': 0.5511, 'grad_norm': 0.7540520406041363, 'learning_rate': 3.6050000000000002e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 65.68, 'epoch': 0.02}
+0: {'loss': 0.554, 'grad_norm': 0.7607703383821428, 'learning_rate': 4.055000000000001e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.02}
+0: 65/2235 [11:44<6:05:16, 10.10s/it]
  3%|▎         | 66/2235 [11:54<6:05:29, 10.11s/it]
  3%|▎         | 67/2235 [12:04<6:05:17, 10.11s/it]
  3%|▎         | 68/2235 [12:14<6:05:47, 10.13s/it]
  3%|▎         | 69/2235 [12:24<6:05:14, 10.12s/it]
  3%|▎         | 70/2235 [12:34<6:04:34, 10.10s/it]
                                                   

  3%|▎         | 70/2235 [12:34<6:04:34, 10.10s/it]
  3%|▎         | 71/2235 [12:44<6:04:03, 10.09s/it]
  3%|▎         | 72/2235 [12:54<6:03:02, 10.07s/it]
  3%|▎         | 73/2235 [13:04<6:01:36, 10.04s/it]
  3%|▎         | 74/2235 [13:15<6:03:46, 10.10s/it]
  3%|▎         | 75/2235 [13:25<6:02:21, 10.07s/it]
  3%|▎         | 76/2235 [13:35<6:01:01, 10.03s/it]
  3%|▎         | 77/2235 [13:45<6:00:22, 10.02s/it]
  3%|▎         | 78/2235 [13:55<5:59:17,  9.99s/it]
  4%|▎         | 79/2235 [14:05<6:01:24, 10.06s/it]
  4%|▎         | 80/2235 [14:15<6:00:09, 10.03s/it]
                                                   

  4%|▎         | 
+0: {'loss': 0.5503, 'grad_norm': 0.774396658020329, 'learning_rate': 4.505e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.02}
+0: 80/2235 [14:15<6:00:09, 10.03s/it]
  4%|▎         | 81/2235 [14:25<5:59:34, 10.02s/it]
  4%|▎         | 82/2235 [14:35<5:59:18, 10.01s/it]
  4%|▎         | 83/2235 [14:45<5:58:57, 10.01s/it]
  4%|▍         | 84/2235 [14:55<5:58:43, 10.01s/it]
  4%|▍         | 85/2235 [15:05<5:58:31, 10.01s/it]
  4%|▍         | 86/2235 [15:15<5:58:17, 10.00s/it]
  4%|▍         | 87/2235 [15:25<5:57:48,  9.99s/it]
  4%|▍         | 88/2235 [15:35<5:57:05,  9.98s/it]
  4%|▍         | 89/2235 [15:44<5:55:53,  9.95s/it]
  4%|▍         | 90/2235 [15:54<5:54:49,  9.92s/it]
                                                   

  4%|▍         | 90/2235 [15:54<5:54:49,  9.92s/it]
  4%|▍         | 91/2235 [16:04<5:54:27,  9.92s/it]
  4%|▍         | 92/2235 [16:14<5:54:31,  9.93s/it]
  4%|▍         | 93/2235 [16:24<5:54:43,  9.94s/it]
  4%|▍         | 94/2235 [16:34<5:54:39,  9.94s/it]
  4%|▍         | 95/2235 [16:44<5:55:10,  9.96s/it]
  4%|▍         | 96/2235 [16:54<5:55:22,  9.97s/it]
  4%|▍         |
+0: {'loss': 0.5479, 'grad_norm': 0.8465980431219081, 'learning_rate': 4.955e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0: {'loss': 0.5245, 'grad_norm': 0.789801377561324, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0:  97/2235 [17:04<5:55:12,  9.97s/it]
  4%|▍         | 98/2235 [17:14<5:54:43,  9.96s/it]
  4%|▍         | 99/2235 [17:24<5:54:29,  9.96s/it]
  4%|▍         | 100/2235 [17:34<5:54:01,  9.95s/it]
                                                    

  4%|▍         | 100/2235 [17:34<5:54:01,  9.95s/it]
  5%|▍         | 101/2235 [17:44<5:53:55,  9.95s/it]
  5%|▍         | 102/2235 [17:54<5:54:01,  9.96s/it]
  5%|▍         | 103/2235 [18:04<5:54:26,  9.97s/it]
  5%|▍         | 104/2235 [18:14<5:56:44, 10.04s/it]
  5%|▍         | 105/2235 [18:24<5:56:41, 10.05s/it]
  5%|▍         | 106/2235 [18:34<5:56:17, 10.04s/it]
  5%|▍         | 107/2235 [18:44<5:55:41, 10.03s/it]
  5%|▍         | 108/2235 [18:54<5:55:07, 10.02s/it]
  5%|▍         | 109/2235 [19:04<5:54:53, 10.02s/it]
  5%|▍         | 110/2235 [19:14<5:54:02, 10.00s/it]
                                                    

  5%|▍         | 110/2235 [19:14<5:54:02, 10.00s/it]
  5%|▍         | 111/2235 [19:24<5:53:40,  9.99s/it]
  
+0: {'loss': 0.5348, 'grad_norm': 0.7979964571477755, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0: 5%|▌         | 112/2235 [19:34<5:54:03, 10.01s/it]
  5%|▌         | 113/2235 [19:44<5:54:15, 10.02s/it]
  5%|▌         | 114/2235 [19:54<5:54:11, 10.02s/it]
  5%|▌         | 115/2235 [20:04<5:54:09, 10.02s/it]
  5%|▌         | 116/2235 [20:14<5:53:49, 10.02s/it]
  5%|▌         | 117/2235 [20:24<5:53:03, 10.00s/it]
  5%|▌         | 118/2235 [20:34<5:53:36, 10.02s/it]
  5%|▌         | 119/2235 [20:44<5:54:03, 10.04s/it]
  5%|▌         | 120/2235 [20:54<5:53:58, 10.04s/it]
                                                    

  5%|▌         | 120/2235 [20:54<5:53:58, 10.04s/it]
  5%|▌         | 121/2235 [21:04<5:54:09, 10.05s/it]
  5%|▌         | 122/2235 [21:14<5:53:55, 10.05s/it]
  6%|▌         | 123/2235 [21:24<5:53:24, 10.04s/it]
  6%|▌         | 124/2235 [21:34<5:52:32, 10.02s/it]
  6%|▌         | 125/2235 [21:45<5:52:43, 10.03s/it]
  6%|▌         | 126/2235 [21:55<5:52:44, 10.04s/it]
  6%|▌         | 127/2235 [22:05<5:52:25, 10.03s/it]
  6%|▌         | 128/2235 [22:15<5:
+0: {'loss': 0.5269, 'grad_norm': 0.8302598842073717, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.03}
+0: {'loss': 0.5235, 'grad_norm': 0.8692237001619731, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0: 51:37, 10.01s/it]
  6%|▌         | 129/2235 [22:25<5:51:26, 10.01s/it]
  6%|▌         | 130/2235 [22:35<5:51:40, 10.02s/it]
                                                    

  6%|▌         | 130/2235 [22:35<5:51:40, 10.02s/it]
  6%|▌         | 131/2235 [22:45<5:51:54, 10.04s/it]
  6%|▌         | 132/2235 [22:55<5:51:33, 10.03s/it]
  6%|▌         | 133/2235 [23:05<5:49:46,  9.98s/it]
  6%|▌         | 134/2235 [23:14<5:47:51,  9.93s/it]
  6%|▌         | 135/2235 [23:24<5:46:46,  9.91s/it]
  6%|▌         | 136/2235 [23:34<5:46:57,  9.92s/it]
  6%|▌         | 137/2235 [23:44<5:46:20,  9.90s/it]
  6%|▌         | 138/2235 [23:54<5:45:40,  9.89s/it]
  6%|▌         | 139/2235 [24:04<5:48:06,  9.96s/it]
  6%|▋         | 140/2235 [24:14<5:47:18,  9.95s/it]
                                                    

  6%|▋         | 140/2235 [24:14<5:47:18,  9.95s/it]
  6%|▋         | 141/2235 [24:24<5:47:11,  9.95s/it]
  6%|▋         | 142/2235 [24:34<5:46:24,  9.93s/it]
  6%|▋         |
+0: {'loss': 0.521, 'grad_norm': 0.8007905764251251, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0:  143/2235 [24:44<5:46:06,  9.93s/it]
  6%|▋         | 144/2235 [24:54<5:45:24,  9.91s/it]
  6%|▋         | 145/2235 [25:03<5:44:54,  9.90s/it]
  7%|▋         | 146/2235 [25:13<5:44:30,  9.89s/it]
  7%|▋         | 147/2235 [25:23<5:43:57,  9.88s/it]
  7%|▋         | 148/2235 [25:33<5:43:37,  9.88s/it]
  7%|▋         | 149/2235 [25:43<5:43:04,  9.87s/it]
  7%|▋         | 150/2235 [25:53<5:45:19,  9.94s/it]
                                                    

  7%|▋         | 150/2235 [25:53<5:45:19,  9.94s/it]
  7%|▋         | 151/2235 [26:03<5:45:22,  9.94s/it]
  7%|▋         | 152/2235 [26:13<5:45:57,  9.97s/it]
  7%|▋         | 153/2235 [26:23<5:46:53, 10.00s/it]
  7%|▋         | 154/2235 [26:33<5:47:23, 10.02s/it]
  7%|▋         | 155/2235 [26:43<5:45:46,  9.97s/it]
  7%|▋         | 156/2235 [26:53<5:44:04,  9.93s/it]
  7%|▋         | 157/2235 [27:03<5:42:51,  9.90s/it]
  7%|▋         | 158/2235 [27:13<5:42:50,  9.90s/it]
  7%|▋         | 159/2235 [27:23<5:43:55,  9.94s/it
+0: {'loss': 0.5112, 'grad_norm': 0.7676158590528094, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.04}
+0: {'loss': 0.5231, 'grad_norm': 0.7882747709260686, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0: ]
  7%|▋         | 160/2235 [27:33<5:44:31,  9.96s/it]
                                                    

  7%|▋         | 160/2235 [27:33<5:44:31,  9.96s/it]
  7%|▋         | 161/2235 [27:43<5:45:12,  9.99s/it]
  7%|▋         | 162/2235 [27:53<5:45:40, 10.00s/it]
  7%|▋         | 163/2235 [28:03<5:46:21, 10.03s/it]
  7%|▋         | 164/2235 [28:13<5:46:34, 10.04s/it]
  7%|▋         | 165/2235 [28:23<5:46:25, 10.04s/it]
  7%|▋         | 166/2235 [28:33<5:47:58, 10.09s/it]
  7%|▋         | 167/2235 [28:43<5:49:12, 10.13s/it]
  8%|▊         | 168/2235 [28:54<5:49:30, 10.15s/it]
  8%|▊         | 169/2235 [29:04<5:49:01, 10.14s/it]
  8%|▊         | 170/2235 [29:14<5:49:48, 10.16s/it]
                                                    

  8%|▊         | 170/2235 [29:14<5:49:48, 10.16s/it]
  8%|▊         | 171/2235 [29:24<5:50:34, 10.19s/it]
  8%|▊         | 172/2235 [29:34<5:50:30, 10.19s/it]
  8%|▊         | 173/2235 [29:44<5:48:42, 10.15s/it]
  8%|▊         | 174/2235 [29:54
+0: {'loss': 0.5075, 'grad_norm': 0.8355320759140278, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0: <5:47:04, 10.10s/it]
  8%|▊         | 175/2235 [30:05<5:48:58, 10.16s/it]
  8%|▊         | 176/2235 [30:15<5:48:45, 10.16s/it]
  8%|▊         | 177/2235 [30:25<5:48:12, 10.15s/it]
  8%|▊         | 178/2235 [30:35<5:48:07, 10.15s/it]
  8%|▊         | 179/2235 [30:45<5:48:18, 10.16s/it]
  8%|▊         | 180/2235 [30:56<5:49:10, 10.19s/it]
                                                    

  8%|▊         | 180/2235 [30:56<5:49:10, 10.19s/it]
  8%|▊         | 181/2235 [31:06<5:49:56, 10.22s/it]
  8%|▊         | 182/2235 [31:16<5:50:19, 10.24s/it]
  8%|▊         | 183/2235 [31:26<5:50:38, 10.25s/it]
  8%|▊         | 184/2235 [31:37<5:50:28, 10.25s/it]
  8%|▊         | 185/2235 [31:47<5:50:23, 10.26s/it]
  8%|▊         | 186/2235 [31:57<5:50:20, 10.26s/it]
  8%|▊         | 187/2235 [32:07<5:49:38, 10.24s/it]
  8%|▊         | 188/2235 [32:17<5:47:34, 10.19s/it]
  8%|▊         | 189/2235 [32:28<5:46:32, 10.16s/it]
  9%|▊         | 190/2235 [32:38<5:45:32, 10.14s/it]
              
+0: {'loss': 0.5106, 'grad_norm': 0.7863262485585292, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0: {'loss': 0.4984, 'grad_norm': 0.837203177284869, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.05}
+0:                                       

  9%|▊         | 190/2235 [32:38<5:45:32, 10.14s/it]
  9%|▊         | 191/2235 [32:48<5:45:29, 10.14s/it]
  9%|▊         | 192/2235 [32:58<5:44:45, 10.13s/it]
  9%|▊         | 193/2235 [33:08<5:43:20, 10.09s/it]
  9%|▊         | 194/2235 [33:18<5:41:51, 10.05s/it]
  9%|▊         | 195/2235 [33:28<5:41:20, 10.04s/it]
  9%|▉         | 196/2235 [33:38<5:40:40, 10.02s/it]
  9%|▉         | 197/2235 [33:48<5:40:30, 10.02s/it]
  9%|▉         | 198/2235 [33:58<5:39:57, 10.01s/it]
  9%|▉         | 199/2235 [34:08<5:43:03, 10.11s/it]
  9%|▉         | 200/2235 [34:18<5:42:45, 10.11s/it]
                                                    

  9%|▉         | 200/2235 [34:18<5:42:45, 10.11s/it]
  9%|▉         | 201/2235 [34:28<5:43:36, 10.14s/it]
  9%|▉         | 202/2235 [34:39<5:43:56, 10.15s/it]
  9%|▉         | 203/2235 [34:49<5:43:14, 10.13s/it]
  9%|▉         | 204/2235 [34:59<5:42:22, 10.11s/it]
  9%|▉         | 205/2235 [35:09<5:42:06, 10.11s
+0: {'loss': 0.5076, 'grad_norm': 0.8167973909471152, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: {'loss': 0.5085, 'grad_norm': 0.7739190539946252, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: /it]
  9%|▉         | 206/2235 [35:19<5:41:34, 10.10s/it]
  9%|▉         | 207/2235 [35:29<5:41:11, 10.09s/it]
  9%|▉         | 208/2235 [35:39<5:39:10, 10.04s/it]
  9%|▉         | 209/2235 [35:49<5:37:11,  9.99s/it]
  9%|▉         | 210/2235 [35:59<5:36:01,  9.96s/it]
                                                    

  9%|▉         | 210/2235 [35:59<5:36:01,  9.96s/it]
  9%|▉         | 211/2235 [36:09<5:35:22,  9.94s/it]
  9%|▉         | 212/2235 [36:19<5:36:40,  9.99s/it]
 10%|▉         | 213/2235 [36:29<5:36:52, 10.00s/it]
 10%|▉         | 214/2235 [36:39<5:37:13, 10.01s/it]
 10%|▉         | 215/2235 [36:49<5:37:01, 10.01s/it]
 10%|▉         | 216/2235 [36:59<5:36:38, 10.00s/it]
 10%|▉         | 217/2235 [37:09<5:37:20, 10.03s/it]
 10%|▉         | 218/2235 [37:19<5:37:13, 10.03s/it]
 10%|▉         | 219/2235 [37:29<5:38:59, 10.09s/it]
 10%|▉         | 220/2235 [37:39<5:38:29, 10.08s/it]
                                                    

 10%|▉         | 220/2235 [37
+0: {'loss': 0.4969, 'grad_norm': 0.7557351218480525, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: :39<5:38:29, 10.08s/it]
 10%|▉         | 221/2235 [37:49<5:38:15, 10.08s/it]
 10%|▉         | 222/2235 [37:59<5:37:34, 10.06s/it]
 10%|▉         | 223/2235 [38:09<5:37:24, 10.06s/it]
 10%|█         | 224/2235 [38:20<5:39:01, 10.11s/it]
 10%|█         | 225/2235 [38:30<5:38:23, 10.10s/it]
 10%|█         | 226/2235 [38:40<5:37:43, 10.09s/it]
 10%|█         | 227/2235 [38:50<5:36:34, 10.06s/it]
 10%|█         | 228/2235 [39:00<5:35:51, 10.04s/it]
 10%|█         | 229/2235 [39:10<5:35:02, 10.02s/it]
 10%|█         | 230/2235 [39:20<5:36:25, 10.07s/it]
                                                    

 10%|█         | 230/2235 [39:20<5:36:25, 10.07s/it]
 10%|█         | 231/2235 [39:30<5:36:57, 10.09s/it]
 10%|█         | 232/2235 [39:40<5:36:25, 10.08s/it]
 10%|█         | 233/2235 [39:50<5:36:12, 10.08s/it]
 10%|█         | 234/2235 [40:00<5:34:01, 10.02s/it]
 11%|█         | 235/2235 [40:10<5:32:43,  9.98s/it]
 11%|█         | 236/2235 [40:20<5:32:20,  9.98s/it]
 11%|█   
+0: {'loss': 0.4884, 'grad_norm': 0.7715216823091144, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.06}
+0: {'loss': 0.5022, 'grad_norm': 0.7639973506963044, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.07}
+0:       | 237/2235 [40:30<5:32:31,  9.99s/it]
 11%|█         | 238/2235 [40:40<5:31:30,  9.96s/it]
 11%|█         | 239/2235 [40:50<5:31:19,  9.96s/it]
 11%|█         | 240/2235 [41:00<5:32:01,  9.99s/it]
                                                    

 11%|█         | 240/2235 [41:00<5:32:01,  9.99s/it]
 11%|█         | 241/2235 [41:10<5:33:18, 10.03s/it]
 11%|█         | 242/2235 [41:20<5:33:32, 10.04s/it]
 11%|█         | 243/2235 [41:30<5:33:48, 10.05s/it]
 11%|█         | 244/2235 [41:40<5:33:43, 10.06s/it]
 11%|█         | 245/2235 [41:50<5:32:05, 10.01s/it]
 11%|█         | 246/2235 [42:00<5:30:57,  9.98s/it]
 11%|█         | 247/2235 [42:10<5:30:07,  9.96s/it]
 11%|█         | 248/2235 [42:20<5:29:27,  9.95s/it]
 11%|█         | 249/2235 [42:30<5:31:35, 10.02s/it]
 11%|█         | 250/2235 [42:40<5:30:29,  9.99s/it]
                                                    

 11%|█         | 250/2235 [42:40<5:30:29,  9.99s/it]
 11%|█         | 251/2235 [42:50<5:29:54,  9.
+0: {'loss': 0.4883, 'grad_norm': 0.9718858587421011, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.07}
+0: 98s/it]
 11%|█▏        | 252/2235 [43:00<5:29:51,  9.98s/it]
 11%|█▏        | 253/2235 [43:10<5:29:07,  9.96s/it]
 11%|█▏        | 254/2235 [43:20<5:29:01,  9.97s/it]
 11%|█▏        | 255/2235 [43:30<5:28:48,  9.96s/it]
 11%|█▏        | 256/2235 [43:40<5:28:01,  9.94s/it]
 11%|█▏        | 257/2235 [43:50<5:28:38,  9.97s/it]
 12%|█▏        | 258/2235 [44:00<5:28:34,  9.97s/it]
 12%|█▏        | 259/2235 [44:10<5:28:28,  9.97s/it]
 12%|█▏        | 260/2235 [44:20<5:28:37,  9.98s/it]
                                                    

 12%|█▏        | 260/2235 [44:20<5:28:37,  9.98s/it]
 12%|█▏        | 261/2235 [44:30<5:28:45,  9.99s/it]
 12%|█▏        | 262/2235 [44:40<5:28:05,  9.98s/it]
 12%|█▏        | 263/2235 [44:50<5:28:40, 10.00s/it]
 12%|█▏        | 264/2235 [45:00<5:29:14, 10.02s/it]
 12%|█▏        | 265/2235 [45:10<5:29:48, 10.05s/it]
 12%|█▏        | 266/2235 [45:20<5:29:31, 10.04s/it]
 12%|█▏        | 267/2235 [45:30<5:28:40, 10.0
+0: {'loss': 0.4926, 'grad_norm': 0.7496781606601971, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.07}
+0: {'loss': 0.4843, 'grad_norm': 2.2246249945609464, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: 2s/it]
 12%|█▏        | 268/2235 [45:40<5:26:50,  9.97s/it]
 12%|█▏        | 269/2235 [45:50<5:26:01,  9.95s/it]
 12%|█▏        | 270/2235 [46:00<5:27:03,  9.99s/it]
                                                    

 12%|█▏        | 270/2235 [46:00<5:27:03,  9.99s/it]
 12%|█▏        | 271/2235 [46:10<5:27:57, 10.02s/it]
 12%|█▏        | 272/2235 [46:20<5:28:46, 10.05s/it]
 12%|█▏        | 273/2235 [46:30<5:29:12, 10.07s/it]
 12%|█▏        | 274/2235 [46:40<5:29:18, 10.08s/it]
 12%|█▏        | 275/2235 [46:50<5:28:51, 10.07s/it]
 12%|█▏        | 276/2235 [47:00<5:26:46, 10.01s/it]
 12%|█▏        | 277/2235 [47:10<5:24:51,  9.95s/it]
 12%|█▏        | 278/2235 [47:20<5:23:36,  9.92s/it]
 12%|█▏        | 279/2235 [47:30<5:23:26,  9.92s/it]
 13%|█▎        | 280/2235 [47:39<5:22:59,  9.91s/it]
                                                    

 13%|█▎        | 280/2235 [47:39<5:22:59,  9.91s/it]
 13%|█▎        | 281/2235 [47:49<5:22:55,  9.92s/i
+0: {'loss': 0.4989, 'grad_norm': 0.785664478439844, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: t]
 13%|█▎        | 282/2235 [47:59<5:22:27,  9.91s/it]
 13%|█▎        | 283/2235 [48:09<5:23:06,  9.93s/it]
 13%|█▎        | 284/2235 [48:20<5:28:52, 10.11s/it]
 13%|█▎        | 285/2235 [48:30<5:29:02, 10.12s/it]
 13%|█▎        | 286/2235 [48:40<5:27:00, 10.07s/it]
 13%|█▎        | 287/2235 [48:50<5:25:51, 10.04s/it]
 13%|█▎        | 288/2235 [49:00<5:24:49, 10.01s/it]
 13%|█▎        | 289/2235 [49:10<5:27:20, 10.09s/it]
 13%|█▎        | 290/2235 [49:20<5:28:03, 10.12s/it]
                                                    

 13%|█▎        | 290/2235 [49:20<5:28:03, 10.12s/it]
 13%|█▎        | 291/2235 [49:31<5:31:20, 10.23s/it]
 13%|█▎        | 292/2235 [49:41<5:36:18, 10.39s/it]
 13%|█▎        | 293/2235 [49:52<5:35:39, 10.37s/it]
 13%|█▎        | 294/2235 [50:02<5:30:32, 10.22s/it]
 13%|█▎        | 295/2235 [50:12<5:27:09, 10.12s/it]
 13%|█▎        | 296/2235 [50:21<5:24:21, 10.04s/it]
 13%|█▎        | 297/2235 [50:32<5:25:07, 10.07s/it
+0: {'loss': 0.4889, 'grad_norm': 0.7980923324079882, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: {'loss': 0.487, 'grad_norm': 0.7348440208783213, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.08}
+0: ]
 13%|█▎        | 298/2235 [50:41<5:23:44, 10.03s/it]
 13%|█▎        | 299/2235 [50:51<5:22:14,  9.99s/it]
 13%|█▎        | 300/2235 [51:01<5:20:49,  9.95s/it]
                                                    

 13%|█▎        | 300/2235 [51:01<5:20:49,  9.95s/it]
 13%|█▎        | 301/2235 [51:11<5:20:51,  9.95s/it]
 14%|█▎        | 302/2235 [51:21<5:20:19,  9.94s/it]
 14%|█▎        | 303/2235 [51:31<5:19:23,  9.92s/it]
 14%|█▎        | 304/2235 [51:41<5:18:33,  9.90s/it]
 14%|█▎        | 305/2235 [51:51<5:18:27,  9.90s/it]
 14%|█▎        | 306/2235 [52:01<5:18:28,  9.91s/it]
 14%|█▎        | 307/2235 [52:11<5:18:55,  9.93s/it]
 14%|█▍        | 308/2235 [52:21<5:18:55,  9.93s/it]
 14%|█▍        | 309/2235 [52:30<5:18:30,  9.92s/it]
 14%|█▍        | 310/2235 [52:40<5:18:25,  9.92s/it]
                                                    

 14%|█▍        | 310/2235 [52:40<5:18:25,  9.92s/it]
 14%|█▍        | 311/2235 [52:50<5:19:52,  9.98s/it]
 1
+0: {'loss': 0.4994, 'grad_norm': 0.8227387562959629, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: 4%|█▍        | 312/2235 [53:01<5:20:45, 10.01s/it]
 14%|█▍        | 313/2235 [53:11<5:21:02, 10.02s/it]
 14%|█▍        | 314/2235 [53:21<5:21:50, 10.05s/it]
 14%|█▍        | 315/2235 [53:31<5:22:12, 10.07s/it]
 14%|█▍        | 316/2235 [53:41<5:21:19, 10.05s/it]
 14%|█▍        | 317/2235 [53:51<5:21:18, 10.05s/it]
 14%|█▍        | 318/2235 [54:01<5:21:05, 10.05s/it]
 14%|█▍        | 319/2235 [54:11<5:23:35, 10.13s/it]
 14%|█▍        | 320/2235 [54:21<5:22:22, 10.10s/it]
                                                    

 14%|█▍        | 320/2235 [54:21<5:22:22, 10.10s/it]
 14%|█▍        | 321/2235 [54:32<5:24:00, 10.16s/it]
 14%|█▍        | 322/2235 [54:45<5:57:55, 11.23s/it]
 14%|█▍        | 323/2235 [54:55<5:46:49, 10.88s/it]
 14%|█▍        | 324/2235 [55:05<5:37:46, 10.61s/it]
 15%|█▍        | 325/2235 [55:15<5:30:56, 10.40s/it]
 15%|█▍        | 326/2235 [55:25<5:27:32, 10.29s/it]
 15%|█▍        | 327/2235 [55:35<5:23:15, 10.17s/it]
 15
+0: {'loss': 0.4868, 'grad_norm': 0.742620790498849, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: {'loss': 0.4949, 'grad_norm': 0.7487961545747542, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: %|█▍        | 328/2235 [55:45<5:19:37, 10.06s/it]
 15%|█▍        | 329/2235 [55:55<5:17:35, 10.00s/it]
 15%|█▍        | 330/2235 [56:05<5:16:07,  9.96s/it]
                                                    

 15%|█▍        | 330/2235 [56:05<5:16:07,  9.96s/it]
 15%|█▍        | 331/2235 [56:15<5:15:13,  9.93s/it]
 15%|█▍        | 332/2235 [56:24<5:14:28,  9.92s/it]
 15%|█▍        | 333/2235 [56:34<5:13:24,  9.89s/it]
 15%|█▍        | 334/2235 [56:44<5:12:30,  9.86s/it]
 15%|█▍        | 335/2235 [56:54<5:12:34,  9.87s/it]
 15%|█▌        | 336/2235 [57:04<5:12:17,  9.87s/it]
 15%|█▌        | 337/2235 [57:14<5:11:51,  9.86s/it]
 15%|█▌        | 338/2235 [57:24<5:12:32,  9.89s/it]
 15%|█▌        | 339/2235 [57:34<5:12:28,  9.89s/it]
 15%|█▌        | 340/2235 [57:43<5:12:14,  9.89s/it]
                                                    

 15%|█▌        | 340/2235 [57:43<5:12:14,  9.89s/it]
 15%|█▌        | 341/2235 [57:53<5:12:03,  9.89s/it]
 15%|�
+0: {'loss': 0.4866, 'grad_norm': 0.7335532391555388, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.09}
+0: �▌        | 342/2235 [58:03<5:11:40,  9.88s/it]
 15%|█▌        | 343/2235 [58:13<5:11:22,  9.87s/it]
 15%|█▌        | 344/2235 [58:23<5:11:16,  9.88s/it]
 15%|█▌        | 345/2235 [58:33<5:11:05,  9.88s/it]
 15%|█▌        | 346/2235 [58:43<5:10:52,  9.87s/it]
 16%|█▌        | 347/2235 [58:53<5:10:50,  9.88s/it]
 16%|█▌        | 348/2235 [59:02<5:10:18,  9.87s/it]
 16%|█▌        | 349/2235 [59:12<5:10:28,  9.88s/it]
 16%|█▌        | 350/2235 [59:22<5:10:38,  9.89s/it]
                                                    

 16%|█▌        | 350/2235 [59:22<5:10:38,  9.89s/it]
 16%|█▌        | 351/2235 [59:32<5:11:03,  9.91s/it]
 16%|█▌        | 352/2235 [59:42<5:11:01,  9.91s/it]
 16%|█▌        | 353/2235 [59:52<5:12:37,  9.97s/it]
 16%|█▌        | 354/2235 [1:00:02<5:12:59,  9.98s/it]
 16%|█▌        | 355/2235 [1:00:12<5:13:21, 10.00s/it]
 16%|█▌        | 356/2235 [1:00:22<5:12:44,  9.99s/it]
 16%|█▌        | 357/2235 [1:00:32<5:12:34,  9.99s/it]
+0: {'loss': 0.4898, 'grad_norm': 0.7971789764125833, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0: {'loss': 0.4884, 'grad_norm': 0.7515580190199106, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0:  16%|█▌        | 358/2235 [1:00:42<5:13:01, 10.01s/it]
 16%|█▌        | 359/2235 [1:00:52<5:13:00, 10.01s/it]
 16%|█▌        | 360/2235 [1:01:02<5:11:36,  9.97s/it]
                                                      

 16%|█▌        | 360/2235 [1:01:02<5:11:36,  9.97s/it]
 16%|█▌        | 361/2235 [1:01:12<5:11:53,  9.99s/it]
 16%|█▌        | 362/2235 [1:01:22<5:11:39,  9.98s/it]
 16%|█▌        | 363/2235 [1:01:32<5:11:02,  9.97s/it]
 16%|█▋        | 364/2235 [1:01:42<5:10:50,  9.97s/it]
 16%|█▋        | 365/2235 [1:01:52<5:10:13,  9.95s/it]
 16%|█▋        | 366/2235 [1:02:02<5:11:14,  9.99s/it]
 16%|█▋        | 367/2235 [1:02:12<5:10:54,  9.99s/it]
 16%|█▋        | 368/2235 [1:02:22<5:09:25,  9.94s/it]
 17%|█▋        | 369/2235 [1:02:32<5:08:57,  9.93s/it]
 17%|█▋        | 370/2235 [1:02:42<5:09:12,  9.95s/it]
                                                      

 17%|█▋        | 370/2235 [1:02:42<5:09:12,  9.95s/it]
 17%|█▋        | 371/22
+0: {'loss': 0.4834, 'grad_norm': 2.047862283567717, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0: 35 [1:02:52<5:09:25,  9.96s/it]
 17%|█▋        | 372/2235 [1:03:02<5:09:21,  9.96s/it]
 17%|█▋        | 373/2235 [1:03:12<5:09:30,  9.97s/it]
 17%|█▋        | 374/2235 [1:03:22<5:11:07, 10.03s/it]
 17%|█▋        | 375/2235 [1:03:32<5:10:33, 10.02s/it]
 17%|█▋        | 376/2235 [1:03:42<5:09:56, 10.00s/it]
 17%|█▋        | 377/2235 [1:03:52<5:12:43, 10.10s/it]
 17%|█▋        | 378/2235 [1:04:02<5:12:28, 10.10s/it]
 17%|█▋        | 379/2235 [1:04:12<5:12:37, 10.11s/it]
 17%|█▋        | 380/2235 [1:04:22<5:12:21, 10.10s/it]
                                                      

 17%|█▋        | 380/2235 [1:04:22<5:12:21, 10.10s/it]
 17%|█▋        | 381/2235 [1:04:33<5:12:13, 10.10s/it]
 17%|█▋        | 382/2235 [1:04:43<5:15:15, 10.21s/it]
 17%|█▋        | 383/2235 [1:04:53<5:14:16, 10.18s/it]
 17%|█▋        | 384/2235 [1:05:03<5:14:34, 10.20s/it]
 17%|█▋        | 385/2235 [1:05:14<5:18:26, 10.33s/it]
 17%|█▋        | 386/2235 [1:05:25<5:25:57, 10.
+0: {'loss': 0.465, 'grad_norm': 1.0917398217081538, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.1}
+0: {'loss': 0.4728, 'grad_norm': 0.7981699343672782, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0: 58s/it]
 17%|█▋        | 387/2235 [1:05:36<5:28:59, 10.68s/it]
 17%|█▋        | 388/2235 [1:05:46<5:23:10, 10.50s/it]
 17%|█▋        | 389/2235 [1:05:56<5:19:39, 10.39s/it]
 17%|█▋        | 390/2235 [1:06:07<5:18:19, 10.35s/it]
                                                      

 17%|█▋        | 390/2235 [1:06:07<5:18:19, 10.35s/it]
 17%|█▋        | 391/2235 [1:06:17<5:16:17, 10.29s/it]
 18%|█▊        | 392/2235 [1:06:27<5:14:25, 10.24s/it]
 18%|█▊        | 393/2235 [1:06:37<5:13:24, 10.21s/it]
 18%|█▊        | 394/2235 [1:06:47<5:12:46, 10.19s/it]
 18%|█▊        | 395/2235 [1:06:57<5:12:22, 10.19s/it]
 18%|█▊        | 396/2235 [1:07:07<5:11:41, 10.17s/it]
 18%|█▊        | 397/2235 [1:07:17<5:10:46, 10.15s/it]
 18%|█▊        | 398/2235 [1:07:28<5:10:09, 10.13s/it]
 18%|█▊        | 399/2235 [1:07:38<5:09:39, 10.12s/it]
 18%|█▊        | 400/2235 [1:07:48<5:09:15, 10.11s/it]
                                                      

 18%|█▊        
+0: {'loss': 0.4798, 'grad_norm': 0.7363454967484379, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0: | 400/2235 [1:07:48<5:09:15, 10.11s/it]
 18%|█▊        | 401/2235 [1:07:58<5:08:45, 10.10s/it]
 18%|█▊        | 402/2235 [1:08:08<5:08:33, 10.10s/it]
 18%|█▊        | 403/2235 [1:08:18<5:12:34, 10.24s/it]
 18%|█▊        | 404/2235 [1:08:29<5:12:01, 10.22s/it]
 18%|█▊        | 405/2235 [1:08:39<5:11:22, 10.21s/it]
 18%|█▊        | 406/2235 [1:08:49<5:11:31, 10.22s/it]
 18%|█▊        | 407/2235 [1:08:59<5:11:21, 10.22s/it]
 18%|█▊        | 408/2235 [1:09:10<5:11:13, 10.22s/it]
 18%|█▊        | 409/2235 [1:09:20<5:11:39, 10.24s/it]
 18%|█▊        | 410/2235 [1:09:30<5:10:58, 10.22s/it]
                                                      

 18%|█▊        | 410/2235 [1:09:30<5:10:58, 10.22s/it]
 18%|█▊        | 411/2235 [1:09:40<5:10:48, 10.22s/it]
 18%|█▊        | 412/2235 [1:09:50<5:10:08, 10.21s/it]
 18%|█▊        | 413/2235 [1:10:00<5:08:35, 10.16s/it]
 19%|█▊        | 414/2235 [1:10:11<5:07:57, 10.15s/it]
 19%|█▊        | 415/2235 [1:10:21<5:07
+0: {'loss': 0.4696, 'grad_norm': 0.7452835192521371, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.11}
+0: :20, 10.13s/it]
 19%|█▊        | 416/2235 [1:10:31<5:06:51, 10.12s/it]
 19%|█▊        | 417/2235 [1:10:41<5:06:55, 10.13s/it]
 19%|█▊        | 418/2235 [1:10:51<5:07:39, 10.16s/it]
 19%|█▊        | 419/2235 [1:11:01<5:06:55, 10.14s/it]
 19%|█▉        | 420/2235 [1:11:11<5:05:39, 10.10s/it]
                                                      

 19%|█▉        | 420/2235 [1:11:11<5:05:39, 10.10s/it]
 19%|█▉        | 421/2235 [1:11:21<5:05:54, 10.12s/it]
 19%|█▉        | 422/2235 [1:11:32<5:05:33, 10.11s/it]
 19%|█▉        | 423/2235 [1:11:42<5:05:11, 10.11s/it]
 19%|█▉        | 424/2235 [1:11:52<5:04:16, 10.08s/it]
 19%|█▉        | 425/2235 [1:12:02<5:03:19, 10.06s/it]
 19%|█▉        | 426/2235 [1:12:12<5:02:11, 10.02s/it]
 19%|█▉        | 427/2235 [1:12:22<5:04:25, 10.10s/it]
 19%|█▉        | 428/2235 [1:12:32<5:04:07, 10.10s/it]
 19%|█▉        | 429/2235 [1:12:42<5:03:26, 10.08s/it]
 19%|█▉        | 430/2235 [1:12:52<5:03:41, 10.10s/it]
        
+0: {'loss': 0.468, 'grad_norm': 1.0802116773190777, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0: {'loss': 0.481, 'grad_norm': 0.7355753374256678, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0:                                               

 19%|█▉        | 430/2235 [1:12:52<5:03:41, 10.10s/it]
 19%|█▉        | 431/2235 [1:13:02<5:03:51, 10.11s/it]
 19%|█▉        | 432/2235 [1:13:12<5:03:53, 10.11s/it]
 19%|█▉        | 433/2235 [1:13:23<5:04:15, 10.13s/it]
 19%|█▉        | 434/2235 [1:13:33<5:03:31, 10.11s/it]
 19%|█▉        | 435/2235 [1:13:43<5:03:52, 10.13s/it]
 20%|█▉        | 436/2235 [1:13:53<5:03:34, 10.12s/it]
 20%|█▉        | 437/2235 [1:14:03<5:03:13, 10.12s/it]
 20%|█▉        | 438/2235 [1:14:13<5:02:42, 10.11s/it]
 20%|█▉        | 439/2235 [1:14:23<5:00:47, 10.05s/it]
 20%|█▉        | 440/2235 [1:14:33<4:59:41, 10.02s/it]
                                                      

 20%|█▉        | 440/2235 [1:14:33<4:59:41, 10.02s/it]
 20%|█▉        | 441/2235 [1:14:43<4:59:25, 10.01s/it]
 20%|█▉        | 442/2235 [1:14:53<5:00:02, 10.04s/it]
 20%|█▉        | 443/2235 [1:15:03<4:59:43, 10.04s/it]
 20%|█▉        | 444/2235 [1:15
+0: {'loss': 0.4749, 'grad_norm': 0.7355231373974477, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0: :13<4:59:43, 10.04s/it]
 20%|█▉        | 445/2235 [1:15:23<4:59:50, 10.05s/it]
 20%|█▉        | 446/2235 [1:15:33<4:59:19, 10.04s/it]
 20%|██        | 447/2235 [1:15:43<4:58:32, 10.02s/it]
 20%|██        | 448/2235 [1:15:53<4:58:25, 10.02s/it]
 20%|██        | 449/2235 [1:16:03<4:58:00, 10.01s/it]
 20%|██        | 450/2235 [1:16:13<4:57:50, 10.01s/it]
                                                      

 20%|██        | 450/2235 [1:16:13<4:57:50, 10.01s/it]
 20%|██        | 451/2235 [1:16:23<4:57:53, 10.02s/it]
 20%|██        | 452/2235 [1:16:33<4:57:40, 10.02s/it]
 20%|██        | 453/2235 [1:16:43<4:57:09, 10.01s/it]
 20%|██        | 454/2235 [1:16:53<4:56:23,  9.98s/it]
 20%|██        | 455/2235 [1:17:03<4:55:56,  9.98s/it]
 20%|██        | 456/2235 [1:17:13<4:56:00,  9.98s/it]
 20%|██        | 457/2235 [1:17:23<4:56:39, 10.01s/it]
 20%|██        | 458/2235 [1:17:33<4:57:19, 10.04s/it]
 21%|██        | 459/2235 [1:17:43<4:57:08, 10.04s/it]
+0: {'loss': 0.4638, 'grad_norm': 0.7412882722841846, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.12}
+0: {'loss': 0.4799, 'grad_norm': 0.7499087677827168, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0:  21%|██        | 460/2235 [1:17:53<4:56:30, 10.02s/it]
                                                      

 21%|██        | 460/2235 [1:17:53<4:56:30, 10.02s/it]
 21%|██        | 461/2235 [1:18:03<4:56:16, 10.02s/it]
 21%|██        | 462/2235 [1:18:13<4:55:52, 10.01s/it]
 21%|██        | 463/2235 [1:18:23<4:55:13, 10.00s/it]
 21%|██        | 464/2235 [1:18:33<4:54:22,  9.97s/it]
 21%|██        | 465/2235 [1:18:43<4:53:35,  9.95s/it]
 21%|██        | 466/2235 [1:18:53<4:53:10,  9.94s/it]
 21%|██        | 467/2235 [1:19:03<4:52:42,  9.93s/it]
 21%|██        | 468/2235 [1:19:13<4:52:38,  9.94s/it]
 21%|██        | 469/2235 [1:19:23<4:52:15,  9.93s/it]
 21%|██        | 470/2235 [1:19:33<4:52:38,  9.95s/it]
                                                      

 21%|██        | 470/2235 [1:19:33<4:52:38,  9.95s/it]
 21%|██        | 471/2235 [1:19:43<4:52:53,  9.96s/it]
 21%|██        | 472/2235 [1:19:53<4:55:15, 10.05s/it]
 21%|██        | 473/22
+0: {'loss': 0.4669, 'grad_norm': 0.8324456224375664, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: 35 [1:20:03<4:54:18, 10.02s/it]
 21%|██        | 474/2235 [1:20:13<4:53:38, 10.00s/it]
 21%|██▏       | 475/2235 [1:20:23<4:54:23, 10.04s/it]
 21%|██▏       | 476/2235 [1:20:33<4:55:12, 10.07s/it]
 21%|██▏       | 477/2235 [1:20:43<4:55:24, 10.08s/it]
 21%|██▏       | 478/2235 [1:20:53<4:55:29, 10.09s/it]
 21%|██▏       | 479/2235 [1:21:04<4:56:08, 10.12s/it]
 21%|██▏       | 480/2235 [1:21:14<4:57:07, 10.16s/it]
                                                      

 21%|██▏       | 480/2235 [1:21:14<4:57:07, 10.16s/it]
 22%|██▏       | 481/2235 [1:21:24<4:57:23, 10.17s/it]
 22%|██▏       | 482/2235 [1:21:34<4:58:14, 10.21s/it]
 22%|██▏       | 483/2235 [1:21:45<4:58:05, 10.21s/it]
 22%|██▏       | 484/2235 [1:21:55<4:57:12, 10.18s/it]
 22%|██▏       | 485/2235 [1:22:05<4:56:10, 10.15s/it]
 22%|██▏       | 486/2235 [1:22:15<4:55:56, 10.15s/it]
 22%|██▏       | 487/2235 [1:22:25<4:55:45, 10.15s/it]
 22%|██▏       | 
+0: {'loss': 0.4703, 'grad_norm': 0.7724096850916233, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: {'loss': 0.4725, 'grad_norm': 0.978084985988498, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.13}
+0: 488/2235 [1:22:35<4:54:09, 10.10s/it]
 22%|██▏       | 489/2235 [1:22:45<4:52:42, 10.06s/it]
 22%|██▏       | 490/2235 [1:22:55<4:50:46, 10.00s/it]
                                                      

 22%|██▏       | 490/2235 [1:22:55<4:50:46, 10.00s/it]
 22%|██▏       | 491/2235 [1:23:05<4:50:02,  9.98s/it]
 22%|██▏       | 492/2235 [1:23:15<4:49:36,  9.97s/it]
 22%|██▏       | 493/2235 [1:23:25<4:49:08,  9.96s/it]
 22%|██▏       | 494/2235 [1:23:35<4:48:57,  9.96s/it]
 22%|██▏       | 495/2235 [1:23:45<4:48:19,  9.94s/it]
 22%|██▏       | 496/2235 [1:23:54<4:47:24,  9.92s/it]
 22%|██▏       | 497/2235 [1:24:04<4:47:48,  9.94s/it]
 22%|██▏       | 498/2235 [1:24:14<4:48:10,  9.95s/it]
 22%|██▏       | 499/2235 [1:24:24<4:47:37,  9.94s/it]
 22%|██▏       | 500/2235 [1:24:34<4:47:45,  9.95s/it]
                                                      

 22%|██▏       | 500/2235 [1:24:34<4:47:45,  9.95s/it]
 22%|██▏      
+0: {'loss': 0.4694, 'grad_norm': 0.8434202596265841, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0:  | 501/2235 [1:24:44<4:48:01,  9.97s/it]
 22%|██▏       | 502/2235 [1:24:54<4:48:06,  9.97s/it]
 23%|██▎       | 503/2235 [1:25:04<4:48:12,  9.98s/it]
 23%|██▎       | 504/2235 [1:25:14<4:47:24,  9.96s/it]
 23%|██▎       | 505/2235 [1:25:24<4:46:54,  9.95s/it]
 23%|██▎       | 506/2235 [1:25:34<4:45:42,  9.91s/it]
 23%|██▎       | 507/2235 [1:25:44<4:44:48,  9.89s/it]
 23%|██▎       | 508/2235 [1:25:54<4:44:05,  9.87s/it]
 23%|██▎       | 509/2235 [1:26:03<4:43:05,  9.84s/it]
 23%|██▎       | 510/2235 [1:26:13<4:42:35,  9.83s/it]
                                                      

 23%|██▎       | 510/2235 [1:26:13<4:42:35,  9.83s/it]
 23%|██▎       | 511/2235 [1:26:23<4:42:21,  9.83s/it]
 23%|██▎       | 512/2235 [1:26:33<4:42:03,  9.82s/it]
 23%|██▎       | 513/2235 [1:26:43<4:41:22,  9.80s/it]
 23%|██▎       | 514/2235 [1:26:52<4:40:57,  9.80s/it]
 23%|██▎       | 515/2235 [1:27:02<4:40:30,  9.78s/it]
 23%|██�
+0: {'loss': 0.4753, 'grad_norm': 0.7437316238226022, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0: ��       | 516/2235 [1:27:12<4:39:45,  9.76s/it]
 23%|██▎       | 517/2235 [1:27:22<4:39:27,  9.76s/it]
 23%|██▎       | 518/2235 [1:27:31<4:38:39,  9.74s/it]
 23%|██▎       | 519/2235 [1:27:41<4:38:13,  9.73s/it]
 23%|██▎       | 520/2235 [1:27:51<4:38:31,  9.74s/it]
                                                      

 23%|██▎       | 520/2235 [1:27:51<4:38:31,  9.74s/it]
 23%|██▎       | 521/2235 [1:28:01<4:38:32,  9.75s/it]
 23%|██▎       | 522/2235 [1:28:10<4:38:26,  9.75s/it]
 23%|██▎       | 523/2235 [1:28:20<4:38:56,  9.78s/it]
 23%|██▎       | 524/2235 [1:28:30<4:39:24,  9.80s/it]
 23%|██▎       | 525/2235 [1:28:40<4:40:13,  9.83s/it]
 24%|██▎       | 526/2235 [1:28:50<4:40:16,  9.84s/it]
 24%|██▎       | 527/2235 [1:29:00<4:39:42,  9.83s/it]
 24%|██▎       | 528/2235 [1:29:09<4:39:34,  9.83s/it]
 24%|██▎       | 529/2235 [1:29:19<4:38:54,  9.81s/it]
 24%|██▎       | 530/2235 [1:29:29<4:38:30,  9.80s/it]
    
+0: {'loss': 0.4696, 'grad_norm': 0.7791190211644309, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0: {'loss': 0.4761, 'grad_norm': 0.7435828798501778, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.14}
+0:                                                   

 24%|██▎       | 530/2235 [1:29:29<4:38:30,  9.80s/it]
 24%|██▍       | 531/2235 [1:29:39<4:38:20,  9.80s/it]
 24%|██▍       | 532/2235 [1:29:49<4:38:35,  9.82s/it]
 24%|██▍       | 533/2235 [1:29:58<4:38:16,  9.81s/it]
 24%|██▍       | 534/2235 [1:30:08<4:38:04,  9.81s/it]
 24%|██▍       | 535/2235 [1:30:18<4:38:07,  9.82s/it]
 24%|██▍       | 536/2235 [1:30:28<4:38:02,  9.82s/it]
 24%|██▍       | 537/2235 [1:30:38<4:39:19,  9.87s/it]
 24%|██▍       | 538/2235 [1:30:48<4:40:22,  9.91s/it]
 24%|██▍       | 539/2235 [1:30:58<4:41:31,  9.96s/it]
 24%|██▍       | 540/2235 [1:31:08<4:42:30, 10.00s/it]
                                                      

 24%|██▍       | 540/2235 [1:31:08<4:42:30, 10.00s/it]
 24%|██▍       | 541/2235 [1:31:18<4:43:07, 10.03s/it]
 24%|██▍       | 542/2235 [1:31:28<4:41:17,  9.97s/it]
 24%|██▍       | 543/2235 [1:31:38<4:40:53,  9.96s/it]
 
+0: {'loss': 0.4675, 'grad_norm': 0.7371581478426831, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: 24%|██▍       | 544/2235 [1:31:48<4:40:46,  9.96s/it]
 24%|██▍       | 545/2235 [1:31:58<4:39:48,  9.93s/it]
 24%|██▍       | 546/2235 [1:32:08<4:38:56,  9.91s/it]
 24%|██▍       | 547/2235 [1:32:17<4:38:24,  9.90s/it]
 25%|██▍       | 548/2235 [1:32:27<4:37:45,  9.88s/it]
 25%|██▍       | 549/2235 [1:32:37<4:37:20,  9.87s/it]
 25%|██▍       | 550/2235 [1:32:47<4:37:30,  9.88s/it]
                                                      

 25%|██▍       | 550/2235 [1:32:47<4:37:30,  9.88s/it]
 25%|██▍       | 551/2235 [1:32:57<4:38:29,  9.92s/it]
 25%|██▍       | 552/2235 [1:33:07<4:39:27,  9.96s/it]
 25%|██▍       | 553/2235 [1:33:17<4:40:07,  9.99s/it]
 25%|██▍       | 554/2235 [1:33:27<4:39:59,  9.99s/it]
 25%|██▍       | 555/2235 [1:33:37<4:40:14, 10.01s/it]
 25%|██▍       | 556/2235 [1:33:47<4:40:45, 10.03s/it]
 25%|██▍       | 557/2235 [1:33:57<4:39:30,  9.99s/it]
 25%|██▍       | 558/2235 [1:34:07<4:38:27,  9.9
+0: {'loss': 0.455, 'grad_norm': 0.7582060364606189, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: {'loss': 0.4724, 'grad_norm': 0.7453577977248541, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.15}
+0: 6s/it]
 25%|██▌       | 559/2235 [1:34:17<4:38:23,  9.97s/it]
 25%|██▌       | 560/2235 [1:34:27<4:37:18,  9.93s/it]
                                                      

 25%|██▌       | 560/2235 [1:34:27<4:37:18,  9.93s/it]
 25%|██▌       | 561/2235 [1:34:37<4:37:33,  9.95s/it]
 25%|██▌       | 562/2235 [1:34:47<4:38:21,  9.98s/it]
 25%|██▌       | 563/2235 [1:34:57<4:39:25, 10.03s/it]
 25%|██▌       | 564/2235 [1:35:07<4:40:37, 10.08s/it]
 25%|██▌       | 565/2235 [1:35:17<4:40:44, 10.09s/it]
 25%|██▌       | 566/2235 [1:35:27<4:40:56, 10.10s/it]
 25%|██▌       | 567/2235 [1:35:37<4:39:17, 10.05s/it]
 25%|██▌       | 568/2235 [1:35:47<4:37:56, 10.00s/it]
 25%|██▌       | 569/2235 [1:35:57<4:36:56,  9.97s/it]
 26%|██▌       | 570/2235 [1:36:07<4:36:33,  9.97s/it]
                                                      

 26%|██▌       | 570/2235 [1:36:07<4:36:33,  9.97s/it]
 26%|██▌       | 571/2235 [1:36:17<4:36:14,  
+0: {'loss': 0.4659, 'grad_norm': 0.8072809534879143, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0: 9.96s/it]
 26%|██▌       | 572/2235 [1:36:27<4:35:09,  9.93s/it]
 26%|██▌       | 573/2235 [1:36:37<4:33:54,  9.89s/it]
 26%|██▌       | 574/2235 [1:36:47<4:32:37,  9.85s/it]
 26%|██▌       | 575/2235 [1:36:56<4:32:45,  9.86s/it]
 26%|██▌       | 576/2235 [1:37:06<4:32:20,  9.85s/it]
 26%|██▌       | 577/2235 [1:37:16<4:32:02,  9.84s/it]
 26%|██▌       | 578/2235 [1:37:26<4:31:33,  9.83s/it]
 26%|██▌       | 579/2235 [1:37:36<4:31:34,  9.84s/it]
 26%|██▌       | 580/2235 [1:37:46<4:31:52,  9.86s/it]
                                                      

 26%|██▌       | 580/2235 [1:37:46<4:31:52,  9.86s/it]
 26%|██▌       | 581/2235 [1:37:56<4:33:09,  9.91s/it]
 26%|██▌       | 582/2235 [1:38:06<4:34:33,  9.97s/it]
 26%|██▌       | 583/2235 [1:38:16<4:35:35, 10.01s/it]
 26%|██▌       | 584/2235 [1:38:26<4:36:28, 10.05s/it]
 26%|██▌       | 585/2235 [1:38:36<4:37:03, 10.08s/it]
 26%|██▌       | 586/2235 [1:38:46<4:
+0: {'loss': 0.4662, 'grad_norm': 0.8033390996170431, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0: 37:19, 10.09s/it]
 26%|██▋       | 587/2235 [1:38:56<4:37:42, 10.11s/it]
 26%|██▋       | 588/2235 [1:39:07<4:38:19, 10.14s/it]
 26%|██▋       | 589/2235 [1:39:17<4:38:21, 10.15s/it]
 26%|██▋       | 590/2235 [1:39:27<4:38:16, 10.15s/it]
                                                      

 26%|██▋       | 590/2235 [1:39:27<4:38:16, 10.15s/it]
 26%|██▋       | 591/2235 [1:39:37<4:38:00, 10.15s/it]
 26%|██▋       | 592/2235 [1:39:47<4:38:29, 10.17s/it]
 27%|██▋       | 593/2235 [1:39:58<4:38:37, 10.18s/it]
 27%|██▋       | 594/2235 [1:40:08<4:38:28, 10.18s/it]
 27%|██▋       | 595/2235 [1:40:18<4:38:25, 10.19s/it]
 27%|██▋       | 596/2235 [1:40:28<4:38:01, 10.18s/it]
 27%|██▋       | 597/2235 [1:40:38<4:37:34, 10.17s/it]
 27%|██▋       | 598/2235 [1:40:48<4:37:22, 10.17s/it]
 27%|██▋       | 599/2235 [1:40:59<4:37:25, 10.17s/it]
 27%|██▋       | 600/2235 [1:41:09<4:37:35, 10.19s/it]
                                   
+0: {'loss': 0.4441, 'grad_norm': 0.7354696364603601, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0: {'loss': 0.4824, 'grad_norm': 2.8611398428825816, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.16}
+0:                    

 27%|██▋       | 600/2235 [1:41:09<4:37:35, 10.19s/it]
 27%|██▋       | 601/2235 [1:41:19<4:37:22, 10.19s/it]
 27%|██▋       | 602/2235 [1:41:29<4:36:21, 10.15s/it]
 27%|██▋       | 603/2235 [1:41:39<4:36:02, 10.15s/it]
 27%|██▋       | 604/2235 [1:41:49<4:36:20, 10.17s/it]
 27%|██▋       | 605/2235 [1:42:00<4:36:04, 10.16s/it]
 27%|██▋       | 606/2235 [1:42:10<4:36:14, 10.17s/it]
 27%|██▋       | 607/2235 [1:42:20<4:35:42, 10.16s/it]
 27%|██▋       | 608/2235 [1:42:30<4:34:14, 10.11s/it]
 27%|██▋       | 609/2235 [1:42:40<4:33:29, 10.09s/it]
 27%|██▋       | 610/2235 [1:42:50<4:32:45, 10.07s/it]
                                                      

 27%|██▋       | 610/2235 [1:42:50<4:32:45, 10.07s/it]
 27%|██▋       | 611/2235 [1:43:00<4:32:17, 10.06s/it]
 27%|██▋       | 612/2235 [1:43:10<4:34:14, 10.14s/it]
 27%|██▋       | 613/2235 [1:43:21<4:35:52, 10.20s/it]
 27%|██▋       | 614/2235 
+0: {'loss': 0.7739, 'grad_norm': 1.3059045154110165, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: [1:43:31<4:37:09, 10.26s/it]
 28%|██▊       | 615/2235 [1:43:41<4:37:25, 10.28s/it]
 28%|██▊       | 616/2235 [1:43:52<4:37:24, 10.28s/it]
 28%|██▊       | 617/2235 [1:44:02<4:38:07, 10.31s/it]
 28%|██▊       | 618/2235 [1:44:12<4:38:35, 10.34s/it]
 28%|██▊       | 619/2235 [1:44:23<4:38:59, 10.36s/it]
 28%|██▊       | 620/2235 [1:44:33<4:37:44, 10.32s/it]
                                                      

 28%|██▊       | 620/2235 [1:44:33<4:37:44, 10.32s/it]
 28%|██▊       | 621/2235 [1:44:43<4:35:58, 10.26s/it]
 28%|██▊       | 622/2235 [1:44:53<4:34:16, 10.20s/it]
 28%|██▊       | 623/2235 [1:45:03<4:33:26, 10.18s/it]
 28%|██▊       | 624/2235 [1:45:13<4:31:48, 10.12s/it]
 28%|██▊       | 625/2235 [1:45:24<4:33:59, 10.21s/it]
 28%|██▊       | 626/2235 [1:45:34<4:36:46, 10.32s/it]
 28%|██▊       | 627/2235 [1:45:45<4:38:23, 10.39s/it]
 28%|██▊       | 628/2235 [1:45:55<4:38:12, 10.39s/it]
 28%|██▊       | 6
+0: {'loss': 0.6985, 'grad_norm': 1.0949784787064452, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: {'loss': 0.6732, 'grad_norm': 0.9461344634727675, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: 29/2235 [1:46:06<4:36:58, 10.35s/it]
 28%|██▊       | 630/2235 [1:46:16<4:36:16, 10.33s/it]
                                                      

 28%|██▊       | 630/2235 [1:46:16<4:36:16, 10.33s/it]
 28%|██▊       | 631/2235 [1:46:26<4:36:07, 10.33s/it]
 28%|██▊       | 632/2235 [1:46:36<4:35:22, 10.31s/it]
 28%|██▊       | 633/2235 [1:46:46<4:33:10, 10.23s/it]
 28%|██▊       | 634/2235 [1:46:57<4:32:20, 10.21s/it]
 28%|██▊       | 635/2235 [1:47:07<4:31:10, 10.17s/it]
 28%|██▊       | 636/2235 [1:47:17<4:30:24, 10.15s/it]
 29%|██▊       | 637/2235 [1:47:27<4:30:18, 10.15s/it]
 29%|██▊       | 638/2235 [1:47:37<4:31:06, 10.19s/it]
 29%|██▊       | 639/2235 [1:47:47<4:30:35, 10.17s/it]
 29%|██▊       | 640/2235 [1:47:57<4:29:54, 10.15s/it]
                                                      

 29%|██▊       | 640/2235 [1:47:57<4:29:54, 10.15s/it]
 29%|██▊       | 641/2235 [1:48:08<4:29:31, 10.15s/it]
 29%|██▊       
+0: {'loss': 0.6496, 'grad_norm': 1.0015485385359406, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.17}
+0: | 642/2235 [1:48:18<4:29:59, 10.17s/it]
 29%|██▉       | 643/2235 [1:48:28<4:29:49, 10.17s/it]
 29%|██▉       | 644/2235 [1:48:38<4:30:10, 10.19s/it]
 29%|██▉       | 645/2235 [1:48:49<4:30:50, 10.22s/it]
 29%|██▉       | 646/2235 [1:48:59<4:31:12, 10.24s/it]
 29%|██▉       | 647/2235 [1:49:09<4:31:39, 10.26s/it]
 29%|██▉       | 648/2235 [1:49:20<4:32:29, 10.30s/it]
 29%|██▉       | 649/2235 [1:49:30<4:32:01, 10.29s/it]
 29%|██▉       | 650/2235 [1:49:40<4:31:58, 10.30s/it]
                                                      

 29%|██▉       | 650/2235 [1:49:40<4:31:58, 10.30s/it]
 29%|██▉       | 651/2235 [1:49:50<4:30:32, 10.25s/it]
 29%|██▉       | 652/2235 [1:50:00<4:28:30, 10.18s/it]
 29%|██▉       | 653/2235 [1:50:10<4:27:14, 10.14s/it]
 29%|██▉       | 654/2235 [1:50:20<4:26:24, 10.11s/it]
 29%|██▉       | 655/2235 [1:50:30<4:25:52, 10.10s/it]
 29%|██▉       | 656/2235 [1:50:41<4:26:14, 10.12s/it]
 29%|██�
+0: {'loss': 0.6328, 'grad_norm': 1.0180271555668265, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.18}
+0: {'loss': 0.632, 'grad_norm': 1.026133025676264, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.18}
+0: �       | 657/2235 [1:50:51<4:26:26, 10.13s/it]
 29%|██▉       | 658/2235 [1:51:01<4:27:17, 10.17s/it]
 29%|██▉       | 659/2235 [1:51:11<4:28:08, 10.21s/it]
 30%|██▉       | 660/2235 [1:51:22<4:28:21, 10.22s/it]
                                                      

 30%|██▉       | 660/2235 [1:51:22<4:28:21, 10.22s/it]
 30%|██▉       | 661/2235 [1:51:32<4:28:38, 10.24s/it]
 30%|██▉       | 662/2235 [1:51:42<4:28:37, 10.25s/it]
 30%|██▉       | 663/2235 [1:51:52<4:26:30, 10.17s/it]
 30%|██▉       | 664/2235 [1:52:02<4:24:52, 10.12s/it]
 30%|██▉       | 665/2235 [1:52:12<4:24:01, 10.09s/it]
 30%|██▉       | 666/2235 [1:52:22<4:22:54, 10.05s/it]
 30%|██▉       | 667/2235 [1:52:32<4:22:07, 10.03s/it]
 30%|██▉       | 668/2235 [1:52:42<4:21:27, 10.01s/it]
 30%|██▉       | 669/2235 [1:52:52<4:20:46,  9.99s/it]
 30%|██▉       | 670/2235 [1:53:02<4:20:22,  9.98s/it]
                                                      

 30%|█�
+0: {'loss': 0.6237, 'grad_norm': 0.9424754366239566, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.18}
+0: �▉       | 670/2235 [1:53:02<4:20:22,  9.98s/it]
 30%|███       | 671/2235 [1:53:12<4:20:20,  9.99s/it]
 30%|███       | 672/2235 [1:53:22<4:21:35, 10.04s/it]
 30%|███       | 673/2235 [1:53:32<4:22:30, 10.08s/it]
 30%|███       | 674/2235 [1:53:43<4:26:31, 10.24s/it]
 30%|███       | 675/2235 [1:53:54<4:29:40, 10.37s/it]
 30%|███       | 676/2235 [1:54:04<4:31:26, 10.45s/it]
 30%|███       | 677/2235 [1:54:15<4:34:30, 10.57s/it]
 30%|███       | 678/2235 [1:54:26<4:34:10, 10.57s/it]
 30%|███       | 679/2235 [1:54:36<4:34:23, 10.58s/it]
 30%|███       | 680/2235 [1:54:47<4:35:12, 10.62s/it]
                                                      

 30%|███       | 680/2235 [1:54:47<4:35:12, 10.62s/it]
 30%|███       | 681/2235 [1:54:58<4:35:57, 10.65s/it]
 31%|███       | 682/2235 [1:55:08<4:35:46, 10.65s/it]
 31%|███       | 683/2235 [1:55:19<4:37:41, 10.74s/it]
 31%|███       | 684/2235 [1:55:29<4:33:03, 10.56s/it]
 3
+0: {'loss': 0.6237, 'grad_norm': 1.437494957139209, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0: 1%|███       | 685/2235 [1:55:39<4:27:51, 10.37s/it]
 31%|███       | 686/2235 [1:55:49<4:24:05, 10.23s/it]
 31%|███       | 687/2235 [1:55:59<4:21:12, 10.12s/it]
 31%|███       | 688/2235 [1:56:09<4:20:32, 10.11s/it]
 31%|███       | 689/2235 [1:56:21<4:30:26, 10.50s/it]
 31%|███       | 690/2235 [1:56:31<4:33:00, 10.60s/it]
                                                      

 31%|███       | 690/2235 [1:56:31<4:33:00, 10.60s/it]
 31%|███       | 691/2235 [1:56:41<4:28:39, 10.44s/it]
 31%|███       | 692/2235 [1:56:51<4:25:00, 10.31s/it]
 31%|███       | 693/2235 [1:57:01<4:22:34, 10.22s/it]
 31%|███       | 694/2235 [1:57:12<4:21:09, 10.17s/it]
 31%|███       | 695/2235 [1:57:22<4:19:40, 10.12s/it]
 31%|███       | 696/2235 [1:57:32<4:18:36, 10.08s/it]
 31%|███       | 697/2235 [1:57:42<4:18:11, 10.07s/it]
 31%|███       | 698/2235 [1:57:52<4:17:04, 10.04s/it]
 31%|███▏      | 699/2235 [1:58:01<4:16:29, 10.
+0: {'loss': 0.6179, 'grad_norm': 2.3792668965438475, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0: {'loss': 0.6117, 'grad_norm': 1.0079603593675321, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0: 02s/it]
 31%|███▏      | 700/2235 [1:58:12<4:16:30, 10.03s/it]
                                                      

 31%|███▏      | 700/2235 [1:58:12<4:16:30, 10.03s/it]
 31%|███▏      | 701/2235 [1:58:22<4:16:42, 10.04s/it]
 31%|███▏      | 702/2235 [1:58:32<4:16:30, 10.04s/it]
 31%|███▏      | 703/2235 [1:58:42<4:16:29, 10.05s/it]
 31%|███▏      | 704/2235 [1:58:52<4:15:50, 10.03s/it]
 32%|███▏      | 705/2235 [1:59:02<4:15:28, 10.02s/it]
 32%|███▏      | 706/2235 [1:59:12<4:15:15, 10.02s/it]
 32%|███▏      | 707/2235 [1:59:22<4:16:24, 10.07s/it]
 32%|███▏      | 708/2235 [1:59:32<4:20:17, 10.23s/it]
 32%|███▏      | 709/2235 [1:59:43<4:21:02, 10.26s/it]
 32%|███▏      | 710/2235 [1:59:53<4:19:56, 10.23s/it]
                                                      

 32%|███▏      | 710/2235 [1:59:53<4:19:56, 10.23s/it]
 32%|███▏      | 711/2235 [2:00:03<4:20:24, 10.25s/it]
 32%|███▏     
+0: {'loss': 0.604, 'grad_norm': 1.453074397723204, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.19}
+0:  | 712/2235 [2:00:14<4:20:31, 10.26s/it]
 32%|███▏      | 713/2235 [2:00:24<4:19:30, 10.23s/it]
 32%|███▏      | 714/2235 [2:00:34<4:18:22, 10.19s/it]
 32%|███▏      | 715/2235 [2:00:44<4:17:58, 10.18s/it]
 32%|███▏      | 716/2235 [2:00:54<4:18:05, 10.19s/it]
 32%|███▏      | 717/2235 [2:01:04<4:18:28, 10.22s/it]
 32%|███▏      | 718/2235 [2:01:15<4:18:34, 10.23s/it]
 32%|███▏      | 719/2235 [2:01:25<4:18:38, 10.24s/it]
 32%|███▏      | 720/2235 [2:01:35<4:18:39, 10.24s/it]
                                                      

 32%|███▏      | 720/2235 [2:01:35<4:18:39, 10.24s/it]
 32%|███▏      | 721/2235 [2:01:46<4:18:41, 10.25s/it]
 32%|███▏      | 722/2235 [2:01:56<4:18:25, 10.25s/it]
 32%|███▏      | 723/2235 [2:02:06<4:20:26, 10.34s/it]
 32%|███▏      | 724/2235 [2:02:16<4:18:51, 10.28s/it]
 32%|███▏      | 725/2235 [2:02:27<4:18:08, 10.26s/it]
 32%|███▏      | 726/2235 [2:02:37<4:
+0: {'loss': 0.6097, 'grad_norm': 1.0964301277740558, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0: 16:38, 10.20s/it]
 33%|███▎      | 727/2235 [2:02:47<4:15:20, 10.16s/it]
 33%|███▎      | 728/2235 [2:02:57<4:14:19, 10.13s/it]
 33%|███▎      | 729/2235 [2:03:07<4:13:39, 10.11s/it]
 33%|███▎      | 730/2235 [2:03:17<4:12:53, 10.08s/it]
                                                      

 33%|███▎      | 730/2235 [2:03:17<4:12:53, 10.08s/it]
 33%|███▎      | 731/2235 [2:03:27<4:12:55, 10.09s/it]
 33%|███▎      | 732/2235 [2:03:37<4:12:55, 10.10s/it]
 33%|███▎      | 733/2235 [2:03:47<4:12:36, 10.09s/it]
 33%|███▎      | 734/2235 [2:03:57<4:12:39, 10.10s/it]
 33%|███▎      | 735/2235 [2:04:08<4:12:58, 10.12s/it]
 33%|███▎      | 736/2235 [2:04:18<4:13:31, 10.15s/it]
 33%|███▎      | 737/2235 [2:04:28<4:15:17, 10.23s/it]
 33%|███▎      | 738/2235 [2:04:38<4:15:58, 10.26s/it]
 33%|███▎      | 739/2235 [2:04:49<4:14:36, 10.21s/it]
 33%|███▎      | 740/2235 [2:04:59<4:13:44, 10.18s/it]
     
+0: {'loss': 0.6051, 'grad_norm': 1.0387578817445569, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0: {'loss': 0.6055, 'grad_norm': 0.9906085179955888, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0:                                                  

 33%|███▎      | 740/2235 [2:04:59<4:13:44, 10.18s/it]
 33%|███▎      | 741/2235 [2:05:09<4:13:08, 10.17s/it]
 33%|███▎      | 742/2235 [2:05:19<4:14:00, 10.21s/it]
 33%|███▎      | 743/2235 [2:05:29<4:12:11, 10.14s/it]
 33%|███▎      | 744/2235 [2:05:39<4:09:34, 10.04s/it]
 33%|███▎      | 745/2235 [2:05:49<4:08:01,  9.99s/it]
 33%|███▎      | 746/2235 [2:05:59<4:07:46,  9.98s/it]
 33%|███▎      | 747/2235 [2:06:09<4:07:25,  9.98s/it]
 33%|███▎      | 748/2235 [2:06:19<4:06:22,  9.94s/it]
 34%|███▎      | 749/2235 [2:06:28<4:05:39,  9.92s/it]
 34%|███▎      | 750/2235 [2:06:38<4:04:52,  9.89s/it]
                                                      

 34%|███▎      | 750/2235 [2:06:38<4:04:52,  9.89s/it]
 34%|███▎      | 751/2235 [2:06:48<4:04:38,  9.89s/it]
 34%|███▎      | 752/2235 [2:06:58<4:04:26,  9.89s/it]
 34%|███▎      | 753/2235 [
+0: {'loss': 0.5946, 'grad_norm': 1.1700189998537633, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.2}
+0: 2:07:08<4:04:36,  9.90s/it]
 34%|███▎      | 754/2235 [2:07:18<4:04:11,  9.89s/it]
 34%|███▍      | 755/2235 [2:07:28<4:06:43, 10.00s/it]
 34%|███▍      | 756/2235 [2:07:39<4:09:46, 10.13s/it]
 34%|███▍      | 757/2235 [2:07:49<4:09:31, 10.13s/it]
 34%|███▍      | 758/2235 [2:07:59<4:09:00, 10.12s/it]
 34%|███▍      | 759/2235 [2:08:09<4:07:55, 10.08s/it]
 34%|███▍      | 760/2235 [2:08:19<4:08:50, 10.12s/it]
                                                      

 34%|███▍      | 760/2235 [2:08:19<4:08:50, 10.12s/it]
 34%|███▍      | 761/2235 [2:08:29<4:08:34, 10.12s/it]
 34%|███▍      | 762/2235 [2:08:39<4:08:41, 10.13s/it]
 34%|███▍      | 763/2235 [2:08:49<4:08:41, 10.14s/it]
 34%|███▍      | 764/2235 [2:09:00<4:08:33, 10.14s/it]
 34%|███▍      | 765/2235 [2:09:10<4:08:00, 10.12s/it]
 34%|███▍      | 766/2235 [2:09:20<4:07:07, 10.09s/it]
 34%|███▍      | 767/2235 [2:09:30<4:06:28, 10.07s
+0: {'loss': 0.6048, 'grad_norm': 1.012933989488154, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0: {'loss': 0.5822, 'grad_norm': 1.1060314808156524, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0: /it]
 34%|███▍      | 768/2235 [2:09:40<4:05:53, 10.06s/it]
 34%|███▍      | 769/2235 [2:09:50<4:04:58, 10.03s/it]
 34%|███▍      | 770/2235 [2:10:00<4:04:18, 10.01s/it]
                                                      

 34%|███▍      | 770/2235 [2:10:00<4:04:18, 10.01s/it]
 34%|███▍      | 771/2235 [2:10:10<4:03:58, 10.00s/it]
 35%|███▍      | 772/2235 [2:10:20<4:03:28,  9.99s/it]
 35%|███▍      | 773/2235 [2:10:30<4:03:47, 10.01s/it]
 35%|███▍      | 774/2235 [2:10:40<4:06:40, 10.13s/it]
 35%|███▍      | 775/2235 [2:10:50<4:05:09, 10.08s/it]
 35%|███▍      | 776/2235 [2:11:00<4:04:06, 10.04s/it]
 35%|███▍      | 777/2235 [2:11:10<4:04:30, 10.06s/it]
 35%|███▍      | 778/2235 [2:11:20<4:05:40, 10.12s/it]
 35%|███▍      | 779/2235 [2:11:31<4:06:17, 10.15s/it]
 35%|███▍      | 780/2235 [2:11:41<4:06:18, 10.16s/it]
                                                      

 35%|███▍      | 
+0: {'loss': 0.5939, 'grad_norm': 0.9257341017454825, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0: 780/2235 [2:11:41<4:06:18, 10.16s/it]
 35%|███▍      | 781/2235 [2:11:51<4:06:01, 10.15s/it]
 35%|███▍      | 782/2235 [2:12:01<4:05:47, 10.15s/it]
 35%|███▌      | 783/2235 [2:12:11<4:05:40, 10.15s/it]
 35%|███▌      | 784/2235 [2:12:21<4:03:37, 10.07s/it]
 35%|███▌      | 785/2235 [2:12:31<4:02:25, 10.03s/it]
 35%|███▌      | 786/2235 [2:12:41<4:01:19,  9.99s/it]
 35%|███▌      | 787/2235 [2:12:51<4:00:14,  9.95s/it]
 35%|███▌      | 788/2235 [2:13:01<3:59:24,  9.93s/it]
 35%|███▌      | 789/2235 [2:13:10<3:58:38,  9.90s/it]
 35%|███▌      | 790/2235 [2:13:20<3:59:16,  9.94s/it]
                                                      

 35%|███▌      | 790/2235 [2:13:20<3:59:16,  9.94s/it]
 35%|███▌      | 791/2235 [2:13:30<3:58:39,  9.92s/it]
 35%|███▌      | 792/2235 [2:13:40<3:58:25,  9.91s/it]
 35%|███▌      | 793/2235 [2:13:50<3:58:06,  9.91s/it]
 36%|███▌      | 794/2235 [2:14:00<3:58:
+0: {'loss': 0.5847, 'grad_norm': 0.9996238386406481, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.21}
+0: 28,  9.93s/it]
 36%|███▌      | 795/2235 [2:14:10<3:58:27,  9.94s/it]
 36%|███▌      | 796/2235 [2:14:20<3:58:21,  9.94s/it]
 36%|███▌      | 797/2235 [2:14:30<3:58:44,  9.96s/it]
 36%|███▌      | 798/2235 [2:14:40<3:59:24, 10.00s/it]
 36%|███▌      | 799/2235 [2:14:50<4:00:34, 10.05s/it]
 36%|███▌      | 800/2235 [2:15:00<4:00:57, 10.07s/it]
                                                      

 36%|███▌      | 800/2235 [2:15:00<4:00:57, 10.07s/it]
 36%|███▌      | 801/2235 [2:15:11<4:01:40, 10.11s/it]
 36%|███▌      | 802/2235 [2:15:21<4:02:42, 10.16s/it]
 36%|███▌      | 803/2235 [2:15:31<4:04:20, 10.24s/it]
 36%|███▌      | 804/2235 [2:15:42<4:05:05, 10.28s/it]
 36%|███▌      | 805/2235 [2:15:52<4:05:28, 10.30s/it]
 36%|███▌      | 806/2235 [2:16:02<4:05:55, 10.33s/it]
 36%|███▌      | 807/2235 [2:16:13<4:06:04, 10.34s/it]
 36%|███▌      | 808/2235 [2:16:23<4:05:29, 10.32s/it]
 36%|█
+0: {'loss': 0.5842, 'grad_norm': 1.1767104590709134, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.22}
+0: {'loss': 0.5765, 'grad_norm': 0.9581540478868384, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.22}
+0: ██▌      | 809/2235 [2:16:33<4:05:03, 10.31s/it]
 36%|███▌      | 810/2235 [2:16:44<4:09:23, 10.50s/it]
                                                      

 36%|███▌      | 810/2235 [2:16:44<4:09:23, 10.50s/it]
 36%|███▋      | 811/2235 [2:16:55<4:09:42, 10.52s/it]
 36%|███▋      | 812/2235 [2:17:06<4:13:25, 10.69s/it]
 36%|███▋      | 813/2235 [2:17:16<4:11:40, 10.62s/it]
 36%|███▋      | 814/2235 [2:17:27<4:14:35, 10.75s/it]
 36%|███▋      | 815/2235 [2:17:39<4:18:04, 10.90s/it]
 37%|███▋      | 816/2235 [2:17:49<4:16:57, 10.87s/it]
 37%|███▋      | 817/2235 [2:18:00<4:14:36, 10.77s/it]
 37%|███▋      | 818/2235 [2:18:11<4:12:49, 10.71s/it]
 37%|███▋      | 819/2235 [2:18:21<4:10:07, 10.60s/it]
 37%|███▋      | 820/2235 [2:18:31<4:07:34, 10.50s/it]
                                                      

 37%|███▋      | 820/2235 [2:18:31<4:07:34, 10.50s/it]
 37%|███▋      | 821/2235 [2:1
+0: {'loss': 0.5888, 'grad_norm': 2.1900811054808207, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.22}
+0: 8:42<4:06:16, 10.45s/it]
 37%|███▋      | 822/2235 [2:18:52<4:05:30, 10.42s/it]
 37%|███▋      | 823/2235 [2:19:02<4:04:36, 10.39s/it]
 37%|███▋      | 824/2235 [2:19:13<4:03:59, 10.38s/it]
 37%|███▋      | 825/2235 [2:19:23<4:02:58, 10.34s/it]
 37%|███▋      | 826/2235 [2:19:33<4:02:20, 10.32s/it]
 37%|███▋      | 827/2235 [2:19:43<4:01:11, 10.28s/it]
 37%|███▋      | 828/2235 [2:19:54<4:02:42, 10.35s/it]
 37%|███▋      | 829/2235 [2:20:04<4:01:04, 10.29s/it]
 37%|███▋      | 830/2235 [2:20:14<3:59:44, 10.24s/it]
                                                      

 37%|███▋      | 830/2235 [2:20:14<3:59:44, 10.24s/it]
 37%|███▋      | 831/2235 [2:20:24<3:59:08, 10.22s/it]
 37%|███▋      | 832/2235 [2:20:34<3:58:25, 10.20s/it]
 37%|███▋      | 833/2235 [2:20:44<3:57:31, 10.17s/it]
 37%|███▋      | 834/2235 [2:20:55<3:57:11, 10.16s/it]
 37%|███▋      | 835/2235 [2:21:05<3:58:11, 10.21s/it
+0: {'loss': 0.588, 'grad_norm': 0.9135261572967428, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0: ]
 37%|███▋      | 836/2235 [2:21:15<4:00:00, 10.29s/it]
 37%|███▋      | 837/2235 [2:21:26<4:00:59, 10.34s/it]
 37%|███▋      | 838/2235 [2:21:36<4:01:39, 10.38s/it]
 38%|███▊      | 839/2235 [2:21:47<4:01:44, 10.39s/it]
 38%|███▊      | 840/2235 [2:21:57<4:01:28, 10.39s/it]
                                                      

 38%|███▊      | 840/2235 [2:21:57<4:01:28, 10.39s/it]
 38%|███▊      | 841/2235 [2:22:08<4:01:28, 10.39s/it]
 38%|███▊      | 842/2235 [2:22:18<3:59:40, 10.32s/it]
 38%|███▊      | 843/2235 [2:22:28<3:59:01, 10.30s/it]
 38%|███▊      | 844/2235 [2:22:38<3:58:56, 10.31s/it]
 38%|███▊      | 845/2235 [2:22:48<3:58:11, 10.28s/it]
 38%|███▊      | 846/2235 [2:22:59<3:57:13, 10.25s/it]
 38%|███▊      | 847/2235 [2:23:09<3:56:17, 10.21s/it]
 38%|███▊      | 848/2235 [2:23:19<3:55:50, 10.20s/it]
 38%|███▊      | 849/2235 [2:23:29<3:55:04, 10.18s/it]
 38%|███▊    
+0: {'loss': 0.5698, 'grad_norm': 0.9605577774047425, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0: {'loss': 0.5768, 'grad_norm': 1.1413515856503087, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0:   | 850/2235 [2:23:39<3:54:35, 10.16s/it]
                                                      

 38%|███▊      | 850/2235 [2:23:39<3:54:35, 10.16s/it]
 38%|███▊      | 851/2235 [2:23:49<3:54:37, 10.17s/it]
 38%|███▊      | 852/2235 [2:24:00<3:54:09, 10.16s/it]
 38%|███▊      | 853/2235 [2:24:10<3:53:44, 10.15s/it]
 38%|███▊      | 854/2235 [2:24:20<3:53:15, 10.13s/it]
 38%|███▊      | 855/2235 [2:24:30<3:52:58, 10.13s/it]
 38%|███▊      | 856/2235 [2:24:40<3:53:04, 10.14s/it]
 38%|███▊      | 857/2235 [2:24:50<3:53:04, 10.15s/it]
 38%|███▊      | 858/2235 [2:25:00<3:53:09, 10.16s/it]
 38%|███▊      | 859/2235 [2:25:10<3:52:35, 10.14s/it]
 38%|███▊      | 860/2235 [2:25:21<3:52:33, 10.15s/it]
                                                      

 38%|███▊      | 860/2235 [2:25:21<3:52:33, 10.15s/it]
 39%|███▊      | 861/2235 [2:25:31<3:52:50, 10.17s/it]
 39%|███▊      | 862/2235 [2:25:41<3:52:01,
+0: {'loss': 0.5776, 'grad_norm': 1.1113052817115814, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.23}
+0:  10.14s/it]
 39%|███▊      | 863/2235 [2:25:51<3:51:33, 10.13s/it]
 39%|███▊      | 864/2235 [2:26:01<3:51:13, 10.12s/it]
 39%|███▊      | 865/2235 [2:26:11<3:51:19, 10.13s/it]
 39%|███▊      | 866/2235 [2:26:21<3:51:14, 10.13s/it]
 39%|███▉      | 867/2235 [2:26:32<3:53:28, 10.24s/it]
 39%|███▉      | 868/2235 [2:26:42<3:53:30, 10.25s/it]
 39%|███▉      | 869/2235 [2:26:52<3:52:58, 10.23s/it]
 39%|███▉      | 870/2235 [2:27:03<3:52:36, 10.22s/it]
                                                      

 39%|███▉      | 870/2235 [2:27:03<3:52:36, 10.22s/it]
 39%|███▉      | 871/2235 [2:27:13<3:52:11, 10.21s/it]
 39%|███▉      | 872/2235 [2:27:23<3:51:46, 10.20s/it]
 39%|███▉      | 873/2235 [2:27:33<3:51:20, 10.19s/it]
 39%|███▉      | 874/2235 [2:27:43<3:50:38, 10.17s/it]
 39%|███▉      | 875/2235 [2:27:54<3:51:52, 10.23s/it]
 39%|███▉      | 876/2235 [2:28:04<3:51:07, 10.20s/it]
 39%|██
+0: {'loss': 0.5809, 'grad_norm': 1.1769020341595884, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0: █▉      | 877/2235 [2:28:14<3:50:28, 10.18s/it]
 39%|███▉      | 878/2235 [2:28:24<3:50:29, 10.19s/it]
 39%|███▉      | 879/2235 [2:28:34<3:50:27, 10.20s/it]
 39%|███▉      | 880/2235 [2:28:44<3:49:52, 10.18s/it]
                                                      

 39%|███▉      | 880/2235 [2:28:44<3:49:52, 10.18s/it]
 39%|███▉      | 881/2235 [2:28:54<3:48:45, 10.14s/it]
 39%|███▉      | 882/2235 [2:29:05<3:48:05, 10.11s/it]
 40%|███▉      | 883/2235 [2:29:15<3:47:39, 10.10s/it]
 40%|███▉      | 884/2235 [2:29:25<3:46:57, 10.08s/it]
 40%|███▉      | 885/2235 [2:29:35<3:46:25, 10.06s/it]
 40%|███▉      | 886/2235 [2:29:45<3:46:00, 10.05s/it]
 40%|███▉      | 887/2235 [2:29:55<3:45:30, 10.04s/it]
 40%|███▉      | 888/2235 [2:30:05<3:45:22, 10.04s/it]
 40%|███▉      | 889/2235 [2:30:15<3:45:29, 10.05s/it]
 40%|███▉      | 890/2235 [2:30:25<3:45:18, 10.05s/it]
                                  
+0: {'loss': 0.5652, 'grad_norm': 0.9801241366150467, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0: {'loss': 0.5761, 'grad_norm': 1.0517798927359199, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0:                     

 40%|███▉      | 890/2235 [2:30:25<3:45:18, 10.05s/it]
 40%|███▉      | 891/2235 [2:30:35<3:44:51, 10.04s/it]
 40%|███▉      | 892/2235 [2:30:45<3:44:00, 10.01s/it]
 40%|███▉      | 893/2235 [2:30:55<3:44:00, 10.02s/it]
 40%|████      | 894/2235 [2:31:05<3:43:35, 10.00s/it]
 40%|████      | 895/2235 [2:31:15<3:42:52,  9.98s/it]
 40%|████      | 896/2235 [2:31:25<3:42:57,  9.99s/it]
 40%|████      | 897/2235 [2:31:35<3:43:00, 10.00s/it]
 40%|████      | 898/2235 [2:31:45<3:43:42, 10.04s/it]
 40%|████      | 899/2235 [2:31:55<3:44:03, 10.06s/it]
 40%|████      | 900/2235 [2:32:05<3:43:23, 10.04s/it]
                                                      

 40%|████      | 900/2235 [2:32:05<3:43:23, 10.04s/it]
 40%|████      | 901/2235 [2:32:15<3:44:05, 10.08s/it]
 40%|████      | 902/2235 [2:32:25<3:44:55, 10.12s/it]
 40%|████      | 903/2235 [2:32:36<3:45:48, 10.17s/it]
 
+0: {'loss': 0.5602, 'grad_norm': 0.9867240262908773, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.24}
+0: 40%|████      | 904/2235 [2:32:46<3:46:11, 10.20s/it]
 40%|████      | 905/2235 [2:32:56<3:46:36, 10.22s/it]
 41%|████      | 906/2235 [2:33:07<3:46:37, 10.23s/it]
 41%|████      | 907/2235 [2:33:17<3:46:27, 10.23s/it]
 41%|████      | 908/2235 [2:33:27<3:46:14, 10.23s/it]
 41%|████      | 909/2235 [2:33:37<3:46:11, 10.24s/it]
 41%|████      | 910/2235 [2:33:47<3:46:14, 10.24s/it]
                                                      

 41%|████      | 910/2235 [2:33:47<3:46:14, 10.24s/it]
 41%|████      | 911/2235 [2:33:58<3:45:53, 10.24s/it]
 41%|████      | 912/2235 [2:34:08<3:46:11, 10.26s/it]
 41%|████      | 913/2235 [2:34:18<3:46:57, 10.30s/it]
 41%|████      | 914/2235 [2:34:29<3:48:07, 10.36s/it]
 41%|████      | 915/2235 [2:34:39<3:47:31, 10.34s/it]
 41%|████      | 916/2235 [2:34:50<3:47:10, 10.33s/it]
 41%|████      | 917/2235 [2:35:00<3:46:50, 10.33s/it]
 41%|████      |
+0: {'loss': 0.5675, 'grad_norm': 1.0356304444066784, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: {'loss': 0.5494, 'grad_norm': 1.430867249860078, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0:  918/2235 [2:35:10<3:46:34, 10.32s/it]
 41%|████      | 919/2235 [2:35:20<3:46:05, 10.31s/it]
 41%|████      | 920/2235 [2:35:31<3:45:09, 10.27s/it]
                                                      

 41%|████      | 920/2235 [2:35:31<3:45:09, 10.27s/it]
 41%|████      | 921/2235 [2:35:41<3:44:02, 10.23s/it]
 41%|████▏     | 922/2235 [2:35:51<3:43:19, 10.21s/it]
 41%|████▏     | 923/2235 [2:36:01<3:42:57, 10.20s/it]
 41%|████▏     | 924/2235 [2:36:11<3:42:36, 10.19s/it]
 41%|████▏     | 925/2235 [2:36:21<3:42:25, 10.19s/it]
 41%|████▏     | 926/2235 [2:36:32<3:42:51, 10.22s/it]
 41%|████▏     | 927/2235 [2:36:42<3:43:14, 10.24s/it]
 42%|████▏     | 928/2235 [2:36:52<3:43:17, 10.25s/it]
 42%|████▏     | 929/2235 [2:37:02<3:41:50, 10.19s/it]
 42%|████▏     | 930/2235 [2:37:12<3:41:02, 10.16s/it]
                                                      

 42%|████▏     | 930/2235 
+0: {'loss': 0.5696, 'grad_norm': 1.2629445275126256, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: [2:37:12<3:41:02, 10.16s/it]
 42%|████▏     | 931/2235 [2:37:22<3:39:59, 10.12s/it]
 42%|████▏     | 932/2235 [2:37:32<3:38:43, 10.07s/it]
 42%|████▏     | 933/2235 [2:37:43<3:39:25, 10.11s/it]
 42%|████▏     | 934/2235 [2:37:53<3:41:02, 10.19s/it]
 42%|████▏     | 935/2235 [2:38:03<3:40:49, 10.19s/it]
 42%|████▏     | 936/2235 [2:38:13<3:39:59, 10.16s/it]
 42%|████▏     | 937/2235 [2:38:23<3:39:07, 10.13s/it]
 42%|████▏     | 938/2235 [2:38:33<3:37:54, 10.08s/it]
 42%|████▏     | 939/2235 [2:38:43<3:37:03, 10.05s/it]
 42%|████▏     | 940/2235 [2:38:53<3:36:58, 10.05s/it]
                                                      

 42%|████▏     | 940/2235 [2:38:53<3:36:58, 10.05s/it]
 42%|████▏     | 941/2235 [2:39:03<3:36:42, 10.05s/it]
 42%|████▏     | 942/2235 [2:39:13<3:35:49, 10.01s/it]
 42%|████▏     | 943/2235 [2:39:23<3:35:26, 10.01s/it]
 42%|████▏     | 94
+0: {'loss': 0.5799, 'grad_norm': 1.040039421373885, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.25}
+0: 4/2235 [2:39:33<3:34:53,  9.99s/it]
 42%|████▏     | 945/2235 [2:39:43<3:34:33,  9.98s/it]
 42%|████▏     | 946/2235 [2:39:53<3:34:01,  9.96s/it]
 42%|████▏     | 947/2235 [2:40:03<3:34:08,  9.98s/it]
 42%|████▏     | 948/2235 [2:40:13<3:34:51, 10.02s/it]
 42%|████▏     | 949/2235 [2:40:23<3:35:19, 10.05s/it]
 43%|████▎     | 950/2235 [2:40:33<3:35:34, 10.07s/it]
                                                      

 43%|████▎     | 950/2235 [2:40:33<3:35:34, 10.07s/it]
 43%|████▎     | 951/2235 [2:40:43<3:34:36, 10.03s/it]
 43%|████▎     | 952/2235 [2:40:53<3:33:27,  9.98s/it]
 43%|████▎     | 953/2235 [2:41:03<3:32:44,  9.96s/it]
 43%|████▎     | 954/2235 [2:41:13<3:32:18,  9.94s/it]
 43%|████▎     | 955/2235 [2:41:23<3:31:59,  9.94s/it]
 43%|████▎     | 956/2235 [2:41:33<3:31:45,  9.93s/it]
 43%|████▎     | 957/2235 [2:41:43<3:31:32,  9.93s/it]
 43%|████▎  
+0: {'loss': 0.564, 'grad_norm': 0.9685008479294198, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.26}
+0: {'loss': 0.5608, 'grad_norm': 0.988700573744174, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.26}
+0:    | 958/2235 [2:41:53<3:31:18,  9.93s/it]
 43%|████▎     | 959/2235 [2:42:03<3:30:56,  9.92s/it]
 43%|████▎     | 960/2235 [2:42:13<3:30:57,  9.93s/it]
                                                      

 43%|████▎     | 960/2235 [2:42:13<3:30:57,  9.93s/it]
 43%|████▎     | 961/2235 [2:42:23<3:30:48,  9.93s/it]
 43%|████▎     | 962/2235 [2:42:32<3:30:13,  9.91s/it]
 43%|████▎     | 963/2235 [2:42:42<3:29:42,  9.89s/it]
 43%|████▎     | 964/2235 [2:42:52<3:28:50,  9.86s/it]
 43%|████▎     | 965/2235 [2:43:02<3:28:10,  9.83s/it]
 43%|████▎     | 966/2235 [2:43:12<3:28:32,  9.86s/it]
 43%|████▎     | 967/2235 [2:43:22<3:28:33,  9.87s/it]
 43%|████▎     | 968/2235 [2:43:32<3:28:49,  9.89s/it]
 43%|████▎     | 969/2235 [2:43:41<3:28:26,  9.88s/it]
 43%|████▎     | 970/2235 [2:43:51<3:28:12,  9.88s/it]
                                                      

 43%|████▎    
+0: {'loss': 0.5583, 'grad_norm': 1.0730697916008645, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.26}
+0:  | 970/2235 [2:43:51<3:28:12,  9.88s/it]
 43%|████▎     | 971/2235 [2:44:01<3:28:16,  9.89s/it]
 43%|████▎     | 972/2235 [2:44:11<3:29:09,  9.94s/it]
 44%|████▎     | 973/2235 [2:44:21<3:29:52,  9.98s/it]
 44%|████▎     | 974/2235 [2:44:31<3:29:59,  9.99s/it]
 44%|████▎     | 975/2235 [2:44:41<3:30:14, 10.01s/it]
 44%|████▎     | 976/2235 [2:44:51<3:30:19, 10.02s/it]
 44%|████▎     | 977/2235 [2:45:02<3:31:00, 10.06s/it]
 44%|████▍     | 978/2235 [2:45:12<3:31:36, 10.10s/it]
 44%|████▍     | 979/2235 [2:45:22<3:31:59, 10.13s/it]
 44%|████▍     | 980/2235 [2:45:32<3:30:55, 10.08s/it]
                                                      

 44%|████▍     | 980/2235 [2:45:32<3:30:55, 10.08s/it]
 44%|████▍     | 981/2235 [2:45:42<3:31:40, 10.13s/it]
 44%|████▍     | 982/2235 [2:45:52<3:30:56, 10.10s/it]
 44%|████▍     | 983/2235 [2:46:02<3:30:43, 10.10s/it]
 44%|████
+0: {'loss': 0.5576, 'grad_norm': 1.1658462154517546, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: ▍     | 984/2235 [2:46:12<3:30:42, 10.11s/it]
 44%|████▍     | 985/2235 [2:46:23<3:30:29, 10.10s/it]
 44%|████▍     | 986/2235 [2:46:33<3:30:21, 10.10s/it]
 44%|████▍     | 987/2235 [2:46:43<3:30:41, 10.13s/it]
 44%|████▍     | 988/2235 [2:46:53<3:30:21, 10.12s/it]
 44%|████▍     | 989/2235 [2:47:03<3:30:36, 10.14s/it]
 44%|████▍     | 990/2235 [2:47:13<3:30:26, 10.14s/it]
                                                      

 44%|████▍     | 990/2235 [2:47:13<3:30:26, 10.14s/it]
 44%|████▍     | 991/2235 [2:47:23<3:30:27, 10.15s/it]
 44%|████▍     | 992/2235 [2:47:34<3:30:18, 10.15s/it]
 44%|████▍     | 993/2235 [2:47:44<3:30:09, 10.15s/it]
 44%|████▍     | 994/2235 [2:47:54<3:29:39, 10.14s/it]
 45%|████▍     | 995/2235 [2:48:04<3:28:52, 10.11s/it]
 45%|████▍     | 996/2235 [2:48:14<3:28:19, 10.09s/it]
 45%|████▍     | 997/2235 [2:48:24<3:28:26, 10.10s/it]
 45%|█�
+0: {'loss': 0.561, 'grad_norm': 2.9648576282473504, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: �██▍     | 998/2235 [2:48:34<3:29:22, 10.16s/it]
 45%|████▍     | 999/2235 [2:48:45<3:30:17, 10.21s/it]
 45%|████▍     | 1000/2235 [2:48:55<3:29:32, 10.18s/it]
                                                       

 45%|████▍     | 1000/2235 [2:48:55<3:29:32, 10.18s/it]
 45%|████▍     | 1001/2235 [2:49:05<3:30:30, 10.24s/it]
 45%|████▍     | 1002/2235 [2:49:15<3:29:24, 10.19s/it]
 45%|████▍     | 1003/2235 [2:49:25<3:28:40, 10.16s/it]
 45%|████▍     | 1004/2235 [2:49:35<3:27:24, 10.11s/it]
 45%|████▍     | 1005/2235 [2:49:45<3:26:25, 10.07s/it]
 45%|████▌     | 1006/2235 [2:49:55<3:25:56, 10.05s/it]
 45%|████▌     | 1007/2235 [2:50:05<3:25:18, 10.03s/it]
 45%|████▌     | 1008/2235 [2:50:15<3:24:18,  9.99s/it]
 45%|████▌     | 1009/2235 [2:50:25<3:23:34,  9.96s/it]
 45%|████▌     | 1010/2235 [2:50:35<3:23:14,  9.95s/it]
                                                       
+0: {'loss': 0.5487, 'grad_norm': 1.5946801942764173, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: {'loss': 0.5615, 'grad_norm': 1.69618129723528, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.27}
+0: 

 45%|████▌     | 1010/2235 [2:50:35<3:23:14,  9.95s/it]
 45%|████▌     | 1011/2235 [2:50:45<3:23:05,  9.96s/it]
 45%|████▌     | 1012/2235 [2:50:55<3:23:05,  9.96s/it]
 45%|████▌     | 1013/2235 [2:51:05<3:23:09,  9.97s/it]
 45%|████▌     | 1014/2235 [2:51:15<3:23:18,  9.99s/it]
 45%|████▌     | 1015/2235 [2:51:25<3:23:28, 10.01s/it]
 45%|████▌     | 1016/2235 [2:51:35<3:23:32, 10.02s/it]
 46%|████▌     | 1017/2235 [2:51:45<3:23:33, 10.03s/it]
 46%|████▌     | 1018/2235 [2:51:55<3:23:18, 10.02s/it]
 46%|████▌     | 1019/2235 [2:52:05<3:23:36, 10.05s/it]
 46%|████▌     | 1020/2235 [2:52:15<3:23:09, 10.03s/it]
                                                       

 46%|████▌     | 1020/2235 [2:52:15<3:23:09, 10.03s/it]
 46%|████▌     | 1021/2235 [2:52:25<3:22:55, 10.03s/it]
 46%|████▌     | 1022/2235 [2:52:35<3:23:08, 10.05s/it]
 46%|█���██▌     | 1023/2235 [2:5
+0: {'loss': 0.5579, 'grad_norm': 1.02384013975649, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: 2:46<3:24:23, 10.12s/it]
 46%|████▌     | 1024/2235 [2:52:56<3:26:43, 10.24s/it]
 46%|████▌     | 1025/2235 [2:53:06<3:26:28, 10.24s/it]
 46%|████▌     | 1026/2235 [2:53:17<3:26:12, 10.23s/it]
 46%|████▌     | 1027/2235 [2:53:27<3:25:21, 10.20s/it]
 46%|████▌     | 1028/2235 [2:53:37<3:24:57, 10.19s/it]
 46%|████▌     | 1029/2235 [2:53:47<3:24:02, 10.15s/it]
 46%|████▌     | 1030/2235 [2:53:57<3:22:52, 10.10s/it]
                                                       

 46%|████▌     | 1030/2235 [2:53:57<3:22:52, 10.10s/it]
 46%|████▌     | 1031/2235 [2:54:07<3:22:00, 10.07s/it]
 46%|████▌     | 1032/2235 [2:54:17<3:20:58, 10.02s/it]
 46%|████▌     | 1033/2235 [2:54:27<3:20:02,  9.99s/it]
 46%|████▋     | 1034/2235 [2:54:37<3:19:03,  9.94s/it]
 46%|████▋     | 1035/2235 [2:54:47<3:19:44,  9.99s/it]
 46%|████▋     | 1036/2235 [2:54:57<3:20:39, 10.04s/it]
 46%|████�
+0: {'loss': 0.5644, 'grad_norm': 1.2936925233292902, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: ��     | 1037/2235 [2:55:07<3:21:29, 10.09s/it]
 46%|████▋     | 1038/2235 [2:55:17<3:21:53, 10.12s/it]
 46%|████▋     | 1039/2235 [2:55:27<3:21:51, 10.13s/it]
 47%|████▋     | 1040/2235 [2:55:37<3:20:52, 10.09s/it]
                                                       

 47%|████▋     | 1040/2235 [2:55:37<3:20:52, 10.09s/it]
 47%|████▋     | 1041/2235 [2:55:47<3:20:11, 10.06s/it]
 47%|████▋     | 1042/2235 [2:55:57<3:19:49, 10.05s/it]
 47%|████▋     | 1043/2235 [2:56:07<3:19:18, 10.03s/it]
 47%|████▋     | 1044/2235 [2:56:17<3:18:13,  9.99s/it]
 47%|████▋     | 1045/2235 [2:56:27<3:17:21,  9.95s/it]
 47%|████▋     | 1046/2235 [2:56:37<3:17:54,  9.99s/it]
 47%|████▋     | 1047/2235 [2:56:47<3:18:46, 10.04s/it]
 47%|████▋     | 1048/2235 [2:56:58<3:19:44, 10.10s/it]
 47%|████▋     | 1049/2235 [2:57:08<3:20:06, 10.12s/it]
 47%|████▋     | 1050/2235 [2:57:18<3:20:55, 10.17s
+0: {'loss': 0.5674, 'grad_norm': 2.6172991847663667, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: {'loss': 0.5496, 'grad_norm': 1.1380294945986047, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.28}
+0: /it]
                                                       

 47%|████▋     | 1050/2235 [2:57:18<3:20:55, 10.17s/it]
 47%|████▋     | 1051/2235 [2:57:29<3:22:06, 10.24s/it]
 47%|████▋     | 1052/2235 [2:57:39<3:22:50, 10.29s/it]
 47%|████▋     | 1053/2235 [2:57:49<3:23:05, 10.31s/it]
 47%|████▋     | 1054/2235 [2:58:00<3:22:34, 10.29s/it]
 47%|████▋     | 1055/2235 [2:58:10<3:21:24, 10.24s/it]
 47%|████▋     | 1056/2235 [2:58:20<3:20:27, 10.20s/it]
 47%|████▋     | 1057/2235 [2:58:30<3:19:38, 10.17s/it]
 47%|████▋     | 1058/2235 [2:58:40<3:19:43, 10.18s/it]
 47%|████▋     | 1059/2235 [2:58:50<3:20:58, 10.25s/it]
 47%|████▋     | 1060/2235 [2:59:01<3:21:57, 10.31s/it]
                                                       

 47%|████▋     | 1060/2235 [2:59:01<3:21:57, 10.31s/it]
 47%|████▋     | 1061/2235 [2:59:11<3:22:44, 10.36s/it]
 48%|████▊     | 1062/2235 [2:59:22<3
+0: {'loss': 0.5558, 'grad_norm': 1.2053455989524502, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0: :23:38, 10.42s/it]
 48%|████▊     | 1063/2235 [2:59:32<3:23:26, 10.41s/it]
 48%|████▊     | 1064/2235 [2:59:43<3:23:09, 10.41s/it]
 48%|████▊     | 1065/2235 [2:59:53<3:24:13, 10.47s/it]
 48%|████▊     | 1066/2235 [3:00:04<3:23:46, 10.46s/it]
 48%|████▊     | 1067/2235 [3:00:14<3:23:12, 10.44s/it]
 48%|████▊     | 1068/2235 [3:00:25<3:22:39, 10.42s/it]
 48%|████▊     | 1069/2235 [3:00:35<3:22:17, 10.41s/it]
 48%|████▊     | 1070/2235 [3:00:45<3:21:20, 10.37s/it]
                                                       

 48%|████▊     | 1070/2235 [3:00:45<3:21:20, 10.37s/it]
 48%|████▊     | 1071/2235 [3:00:55<3:19:40, 10.29s/it]
 48%|████▊     | 1072/2235 [3:01:05<3:18:16, 10.23s/it]
 48%|████▊     | 1073/2235 [3:01:16<3:17:20, 10.19s/it]
 48%|████▊     | 1074/2235 [3:01:26<3:16:34, 10.16s/it]
 48%|████▊     | 1075/2235 [3:01:36<3:16:10, 10.15s/it]
 48%|████▊    
+0: {'loss': 0.547, 'grad_norm': 1.1835908240731505, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0:  | 1076/2235 [3:01:46<3:15:40, 10.13s/it]
 48%|████▊     | 1077/2235 [3:01:56<3:15:45, 10.14s/it]
 48%|████▊     | 1078/2235 [3:02:06<3:15:48, 10.15s/it]
 48%|████▊     | 1079/2235 [3:02:16<3:16:17, 10.19s/it]
 48%|████▊     | 1080/2235 [3:02:27<3:15:50, 10.17s/it]
                                                       

 48%|████▊     | 1080/2235 [3:02:27<3:15:50, 10.17s/it]
 48%|████▊     | 1081/2235 [3:02:37<3:15:26, 10.16s/it]
 48%|████▊     | 1082/2235 [3:02:47<3:14:49, 10.14s/it]
 48%|████▊     | 1083/2235 [3:02:57<3:14:12, 10.12s/it]
 49%|████▊     | 1084/2235 [3:03:07<3:12:53, 10.06s/it]
 49%|████▊     | 1085/2235 [3:03:17<3:12:02, 10.02s/it]
 49%|████▊     | 1086/2235 [3:03:27<3:11:16,  9.99s/it]
 49%|████▊     | 1087/2235 [3:03:37<3:10:44,  9.97s/it]
 49%|████▊     | 1088/2235 [3:03:47<3:10:52,  9.98s/it]
 49%|████▊     | 1089/2235 [3:03:56<3:09:58,  9.95s/it]
 
+0: {'loss': 0.5436, 'grad_norm': 1.3291747837205672, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.29}
+0: {'loss': 0.5424, 'grad_norm': 1.641531627856187, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: 49%|████▉     | 1090/2235 [3:04:06<3:09:30,  9.93s/it]
                                                       

 49%|████▉     | 1090/2235 [3:04:06<3:09:30,  9.93s/it]
 49%|████▉     | 1091/2235 [3:04:16<3:09:22,  9.93s/it]
 49%|████▉     | 1092/2235 [3:04:27<3:11:37, 10.06s/it]
 49%|████▉     | 1093/2235 [3:04:37<3:13:22, 10.16s/it]
 49%|████▉     | 1094/2235 [3:04:47<3:14:16, 10.22s/it]
 49%|████▉     | 1095/2235 [3:04:58<3:14:21, 10.23s/it]
 49%|████▉     | 1096/2235 [3:05:08<3:12:45, 10.15s/it]
 49%|████▉     | 1097/2235 [3:05:18<3:11:49, 10.11s/it]
 49%|████▉     | 1098/2235 [3:05:28<3:11:10, 10.09s/it]
 49%|████▉     | 1099/2235 [3:05:38<3:10:28, 10.06s/it]
 49%|████▉     | 1100/2235 [3:05:48<3:09:59, 10.04s/it]
                                                       

 49%|████▉     | 1100/2235 [3:05:48<3:09:59, 10.04s/it]
 49%|████▉     | 1101/2235 [3:05:58<3:09:37
+0: {'loss': 0.5568, 'grad_norm': 1.306063497356509, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: , 10.03s/it]
 49%|████▉     | 1102/2235 [3:06:08<3:09:20, 10.03s/it]
 49%|████▉     | 1103/2235 [3:06:18<3:08:57, 10.02s/it]
 49%|████▉     | 1104/2235 [3:06:28<3:08:30, 10.00s/it]
 49%|████▉     | 1105/2235 [3:06:38<3:08:04,  9.99s/it]
 49%|████▉     | 1106/2235 [3:06:48<3:07:35,  9.97s/it]
 50%|████▉     | 1107/2235 [3:06:57<3:07:22,  9.97s/it]
 50%|████▉     | 1108/2235 [3:07:08<3:08:06, 10.01s/it]
 50%|████▉     | 1109/2235 [3:07:18<3:11:03, 10.18s/it]
 50%|████▉     | 1110/2235 [3:07:29<3:11:52, 10.23s/it]
                                                       

 50%|████▉     | 1110/2235 [3:07:29<3:11:52, 10.23s/it]
 50%|████▉     | 1111/2235 [3:07:39<3:12:12, 10.26s/it]
 50%|████▉     | 1112/2235 [3:07:49<3:12:04, 10.26s/it]
 50%|████▉     | 1113/2235 [3:07:59<3:12:06, 10.27s/it]
 50%|████▉     | 1114/2235 [3:08:10<3:11:56, 10.27s/it]
 50%|████▉     | 111
+0: {'loss': 0.5419, 'grad_norm': 1.1830578694387712, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: 5/2235 [3:08:20<3:11:44, 10.27s/it]
 50%|████▉     | 1116/2235 [3:08:30<3:11:38, 10.28s/it]
 50%|████▉     | 1117/2235 [3:08:41<3:11:29, 10.28s/it]
 50%|█████     | 1118/2235 [3:08:51<3:11:20, 10.28s/it]
 50%|█████     | 1119/2235 [3:09:01<3:11:07, 10.28s/it]
 50%|█████     | 1120/2235 [3:09:11<3:10:00, 10.22s/it]
                                                       

 50%|█████     | 1120/2235 [3:09:11<3:10:00, 10.22s/it]
 50%|█████     | 1121/2235 [3:09:21<3:09:03, 10.18s/it]
 50%|█████     | 1122/2235 [3:09:31<3:08:23, 10.16s/it]
 50%|█████     | 1123/2235 [3:09:41<3:07:47, 10.13s/it]
 50%|█████     | 1124/2235 [3:09:51<3:07:12, 10.11s/it]
 50%|█████     | 1125/2235 [3:10:01<3:06:22, 10.07s/it]
 50%|█████     | 1126/2235 [3:10:11<3:05:25, 10.03s/it]
 50%|█████     | 1127/2235 [3:10:21<3:04:54, 10.01s/it]
 50%|█████     | 1128/2235 [3:10:31<3:04:08,  9.98s/it]
 51%|�
+0: {'loss': 0.5501, 'grad_norm': 1.0199308181345577, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.3}
+0: {'loss': 0.5596, 'grad_norm': 0.9729345023167901, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: �████     | 1129/2235 [3:10:41<3:03:28,  9.95s/it]
 51%|█████     | 1130/2235 [3:10:51<3:03:54,  9.99s/it]
                                                       

 51%|█████     | 1130/2235 [3:10:51<3:03:54,  9.99s/it]
 51%|█████     | 1131/2235 [3:11:01<3:03:56, 10.00s/it]
 51%|█████     | 1132/2235 [3:11:11<3:03:54, 10.00s/it]
 51%|█████     | 1133/2235 [3:11:21<3:03:42, 10.00s/it]
 51%|█████     | 1134/2235 [3:11:31<3:03:35, 10.00s/it]
 51%|█████     | 1135/2235 [3:11:41<3:03:31, 10.01s/it]
 51%|█████     | 1136/2235 [3:11:51<3:03:41, 10.03s/it]
 51%|█████     | 1137/2235 [3:12:01<3:03:50, 10.05s/it]
 51%|█████     | 1138/2235 [3:12:11<3:03:33, 10.04s/it]
 51%|█████     | 1139/2235 [3:12:22<3:03:21, 10.04s/it]
 51%|█████     | 1140/2235 [3:12:32<3:03:13, 10.04s/it]
                                                       

 51%|█████     | 1140/2235 [3:12:32<3:03:13, 10.0
+0: {'loss': 0.5468, 'grad_norm': 0.9280199424158943, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: 4s/it]
 51%|█████     | 1141/2235 [3:12:42<3:03:00, 10.04s/it]
 51%|█████     | 1142/2235 [3:12:52<3:03:06, 10.05s/it]
 51%|█████     | 1143/2235 [3:13:02<3:03:17, 10.07s/it]
 51%|█████     | 1144/2235 [3:13:12<3:03:12, 10.08s/it]
 51%|█████     | 1145/2235 [3:13:22<3:02:59, 10.07s/it]
 51%|█████▏    | 1146/2235 [3:13:32<3:03:06, 10.09s/it]
 51%|█████▏    | 1147/2235 [3:13:42<3:03:16, 10.11s/it]
 51%|█████▏    | 1148/2235 [3:13:52<3:03:37, 10.14s/it]
 51%|█████▏    | 1149/2235 [3:14:03<3:03:55, 10.16s/it]
 51%|█████▏    | 1150/2235 [3:14:13<3:03:35, 10.15s/it]
                                                       

 51%|█████▏    | 1150/2235 [3:14:13<3:03:35, 10.15s/it]
 51%|█████▏    | 1151/2235 [3:14:23<3:03:33, 10.16s/it]
 52%|█████▏    | 1152/2235 [3:14:33<3:03:24, 10.16s/it]
 52%|█████▏    | 1153/2235 [3:14:43<3:02:52, 10.14s/it]
 52%|████�
+0: {'loss': 0.549, 'grad_norm': 1.0889691485325224, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: ��▏    | 1154/2235 [3:14:53<3:02:04, 10.11s/it]
 52%|█████▏    | 1155/2235 [3:15:03<3:01:45, 10.10s/it]
 52%|█████▏    | 1156/2235 [3:15:13<3:01:35, 10.10s/it]
 52%|█████▏    | 1157/2235 [3:15:24<3:02:25, 10.15s/it]
 52%|█████▏    | 1158/2235 [3:15:34<3:02:09, 10.15s/it]
 52%|█████▏    | 1159/2235 [3:15:44<3:01:56, 10.15s/it]
 52%|█████▏    | 1160/2235 [3:15:54<3:01:22, 10.12s/it]
                                                       

 52%|█████▏    | 1160/2235 [3:15:54<3:01:22, 10.12s/it]
 52%|█████▏    | 1161/2235 [3:16:04<3:01:09, 10.12s/it]
 52%|█████▏    | 1162/2235 [3:16:14<3:01:03, 10.12s/it]
 52%|█████▏    | 1163/2235 [3:16:24<3:01:10, 10.14s/it]
 52%|█████▏    | 1164/2235 [3:16:35<3:00:44, 10.13s/it]
 52%|█████▏    | 1165/2235 [3:16:45<3:00:22, 10.11s/it]
 52%|█████▏    | 1166/2235 [3:16:55<3:00:30, 10.13s/it]
 52%|█████▏    | 1167
+0: {'loss': 0.5547, 'grad_norm': 1.1029791889593616, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.31}
+0: /2235 [3:17:05<3:01:50, 10.22s/it]
 52%|█████▏    | 1168/2235 [3:17:15<3:01:44, 10.22s/it]
 52%|█████▏    | 1169/2235 [3:17:26<3:01:44, 10.23s/it]
 52%|█████▏    | 1170/2235 [3:17:36<3:01:46, 10.24s/it]
                                                       

 52%|█████▏    | 1170/2235 [3:17:36<3:01:46, 10.24s/it]
 52%|█████▏    | 1171/2235 [3:17:46<3:01:39, 10.24s/it]
 52%|█████▏    | 1172/2235 [3:17:56<3:01:32, 10.25s/it]
 52%|█████▏    | 1173/2235 [3:18:07<3:01:06, 10.23s/it]
 53%|█████▎    | 1174/2235 [3:18:17<3:00:48, 10.22s/it]
 53%|█████▎    | 1175/2235 [3:18:27<3:00:55, 10.24s/it]
 53%|█████▎    | 1176/2235 [3:18:37<2:59:51, 10.19s/it]
 53%|█████▎    | 1177/2235 [3:18:47<2:59:23, 10.17s/it]
 53%|█████▎    | 1178/2235 [3:18:58<2:59:13, 10.17s/it]
 53%|█████▎    | 1179/2235 [3:19:08<2:59:05, 10.18s/it]
 53%|█████▎    | 1180/2235 [3:19:18<
+0: {'loss': 0.5537, 'grad_norm': 0.9787953644123929, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: {'loss': 0.5438, 'grad_norm': 0.9519861902844139, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: 2:59:14, 10.19s/it]
                                                       

 53%|█████▎    | 1180/2235 [3:19:18<2:59:14, 10.19s/it]
 53%|█████▎    | 1181/2235 [3:19:28<2:59:31, 10.22s/it]
 53%|█████▎    | 1182/2235 [3:19:39<2:59:50, 10.25s/it]
 53%|█████▎    | 1183/2235 [3:19:49<2:59:57, 10.26s/it]
 53%|█████▎    | 1184/2235 [3:19:59<2:59:35, 10.25s/it]
 53%|█████▎    | 1185/2235 [3:20:09<2:59:21, 10.25s/it]
 53%|█████▎    | 1186/2235 [3:20:19<2:58:35, 10.21s/it]
 53%|█████▎    | 1187/2235 [3:20:30<2:58:00, 10.19s/it]
 53%|█████▎    | 1188/2235 [3:20:40<2:57:22, 10.16s/it]
 53%|█████▎    | 1189/2235 [3:20:50<2:57:35, 10.19s/it]
 53%|█████▎    | 1190/2235 [3:21:00<2:57:37, 10.20s/it]
                                                       

 53%|█████▎    | 1190/2235 [3:21:00<2:57:37, 10.20s/it]
 53%|█████▎    | 1191/2235 [3:21:10<2:57:44, 10.22s/it]
 53%|�
+0: {'loss': 0.5373, 'grad_norm': 1.200803748119201, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: ��████▎    | 1192/2235 [3:21:21<2:57:26, 10.21s/it]
 53%|█████▎    | 1193/2235 [3:21:31<2:57:10, 10.20s/it]
 53%|█████▎    | 1194/2235 [3:21:41<2:56:09, 10.15s/it]
 53%|█████▎    | 1195/2235 [3:21:51<2:55:01, 10.10s/it]
 54%|█████▎    | 1196/2235 [3:22:01<2:55:08, 10.11s/it]
 54%|█████▎    | 1197/2235 [3:22:11<2:54:59, 10.12s/it]
 54%|█████▎    | 1198/2235 [3:22:21<2:53:57, 10.06s/it]
 54%|█████▎    | 1199/2235 [3:22:31<2:53:25, 10.04s/it]
 54%|█████▎    | 1200/2235 [3:22:41<2:55:06, 10.15s/it]
                                                       

 54%|█████▎    | 1200/2235 [3:22:41<2:55:06, 10.15s/it]
 54%|█████▎    | 1201/2235 [3:22:52<2:55:01, 10.16s/it]
 54%|█████▍    | 1202/2235 [3:23:02<2:55:13, 10.18s/it]
 54%|█████▍    | 1203/2235 [3:23:12<2:55:22, 10.20s/it]
 54%|█████▍    | 1204/2235 [3:23:22<2:55:35, 10.22s/it]
 54%|█████�
+0: {'loss': 0.5491, 'grad_norm': 1.4101646106873496, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.32}
+0: ��    | 1205/2235 [3:23:33<2:55:37, 10.23s/it]
 54%|█████▍    | 1206/2235 [3:23:43<2:54:48, 10.19s/it]
 54%|█████▍    | 1207/2235 [3:23:53<2:53:28, 10.12s/it]
 54%|█████▍    | 1208/2235 [3:24:03<2:52:41, 10.09s/it]
 54%|█████▍    | 1209/2235 [3:24:13<2:51:53, 10.05s/it]
 54%|█████▍    | 1210/2235 [3:24:23<2:51:05, 10.02s/it]
                                                       

 54%|█████▍    | 1210/2235 [3:24:23<2:51:05, 10.02s/it]
 54%|█████▍    | 1211/2235 [3:24:33<2:50:29,  9.99s/it]
 54%|█████▍    | 1212/2235 [3:24:43<2:50:48, 10.02s/it]
 54%|█████▍    | 1213/2235 [3:24:53<2:50:53, 10.03s/it]
 54%|█████▍    | 1214/2235 [3:25:03<2:51:04, 10.05s/it]
 54%|█████▍    | 1215/2235 [3:25:13<2:51:20, 10.08s/it]
 54%|█████▍    | 1216/2235 [3:25:23<2:51:54, 10.12s/it]
 54%|█████▍    | 1217/2235 [3:25:34<2:53:44, 10.24s/it]
 54%|█████▍    | 1218/22
+0: {'loss': 0.5488, 'grad_norm': 1.018792469658303, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.33}
+0: 35 [3:25:44<2:52:22, 10.17s/it]
 55%|█████▍    | 1219/2235 [3:25:54<2:51:06, 10.10s/it]
 55%|█████▍    | 1220/2235 [3:26:04<2:50:22, 10.07s/it]
                                                       

 55%|█████▍    | 1220/2235 [3:26:04<2:50:22, 10.07s/it]
 55%|█████▍    | 1221/2235 [3:26:14<2:51:05, 10.12s/it]
 55%|█████▍    | 1222/2235 [3:26:24<2:51:23, 10.15s/it]
 55%|█████▍    | 1223/2235 [3:26:34<2:50:57, 10.14s/it]
 55%|█████▍    | 1224/2235 [3:26:44<2:50:26, 10.12s/it]
 55%|█████▍    | 1225/2235 [3:26:54<2:49:46, 10.09s/it]
 55%|█████▍    | 1226/2235 [3:27:04<2:50:11, 10.12s/it]
 55%|█████▍    | 1227/2235 [3:27:15<2:50:18, 10.14s/it]
 55%|█████▍    | 1228/2235 [3:27:25<2:49:51, 10.12s/it]
 55%|█████▍    | 1229/2235 [3:27:35<2:50:17, 10.16s/it]
 55%|█████▌    | 1230/2235 [3:27:45<2:50:59, 10.21s/it]
                                                   
+0: {'loss': 0.5429, 'grad_norm': 1.3825573885679603, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.33}
+0: {'loss': 0.5424, 'grad_norm': 1.159397754016814, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.33}
+0:     

 55%|█████▌    | 1230/2235 [3:27:45<2:50:59, 10.21s/it]
 55%|█████▌    | 1231/2235 [3:27:56<2:51:20, 10.24s/it]
 55%|█████▌    | 1232/2235 [3:28:06<2:51:30, 10.26s/it]
 55%|█████▌    | 1233/2235 [3:28:16<2:50:46, 10.23s/it]
 55%|█████▌    | 1234/2235 [3:28:26<2:50:18, 10.21s/it]
 55%|█████▌    | 1235/2235 [3:28:36<2:49:43, 10.18s/it]
 55%|█████▌    | 1236/2235 [3:28:47<2:52:10, 10.34s/it]
 55%|█████▌    | 1237/2235 [3:28:57<2:51:17, 10.30s/it]
 55%|█████▌    | 1238/2235 [3:29:08<2:54:39, 10.51s/it]
 55%|█████▌    | 1239/2235 [3:29:19<2:53:21, 10.44s/it]
 55%|█████▌    | 1240/2235 [3:29:29<2:55:18, 10.57s/it]
                                                       

 55%|█████▌    | 1240/2235 [3:29:29<2:55:18, 10.57s/it]
 56%|█████▌    | 1241/2235 [3:29:41<2:57:59, 10.74s/it]
 56%|█████▌    | 1242/2235 [3:29:51<2:56:05, 10.64s/it]
 56%|█�
+0: {'loss': 0.5411, 'grad_norm': 1.148759878927006, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: ��███▌    | 1243/2235 [3:30:01<2:54:16, 10.54s/it]
 56%|█████▌    | 1244/2235 [3:30:11<2:51:30, 10.38s/it]
 56%|█████▌    | 1245/2235 [3:30:22<2:52:42, 10.47s/it]
 56%|█████▌    | 1246/2235 [3:30:32<2:49:45, 10.30s/it]
 56%|█████▌    | 1247/2235 [3:30:42<2:47:45, 10.19s/it]
 56%|█████▌    | 1248/2235 [3:30:52<2:46:17, 10.11s/it]
 56%|█████▌    | 1249/2235 [3:31:02<2:45:26, 10.07s/it]
 56%|█████▌    | 1250/2235 [3:31:12<2:44:48, 10.04s/it]
                                                       

 56%|█████▌    | 1250/2235 [3:31:12<2:44:48, 10.04s/it]
 56%|█████▌    | 1251/2235 [3:31:22<2:44:15, 10.02s/it]
 56%|█████▌    | 1252/2235 [3:31:32<2:44:35, 10.05s/it]
 56%|█████▌    | 1253/2235 [3:31:42<2:45:03, 10.09s/it]
 56%|█████▌    | 1254/2235 [3:31:52<2:45:08, 10.10s/it]
 56%|█████▌    | 1255/2235 [3:32:02<2:46:31, 10.20s/it]
 56%|█████▌ 
+0: {'loss': 0.5361, 'grad_norm': 1.0307059720030702, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0:    | 1256/2235 [3:32:12<2:45:18, 10.13s/it]
 56%|█████▌    | 1257/2235 [3:32:22<2:44:07, 10.07s/it]
 56%|█████▋    | 1258/2235 [3:32:33<2:44:25, 10.10s/it]
 56%|█████▋    | 1259/2235 [3:32:42<2:43:22, 10.04s/it]
 56%|█████▋    | 1260/2235 [3:32:52<2:43:21, 10.05s/it]
                                                       

 56%|█████▋    | 1260/2235 [3:32:53<2:43:21, 10.05s/it]
 56%|█████▋    | 1261/2235 [3:33:03<2:43:04, 10.05s/it]
 56%|█████▋    | 1262/2235 [3:33:13<2:43:06, 10.06s/it]
 57%|█████▋    | 1263/2235 [3:33:23<2:43:12, 10.07s/it]
 57%|█████▋    | 1264/2235 [3:33:33<2:43:42, 10.12s/it]
 57%|█████▋    | 1265/2235 [3:33:43<2:43:59, 10.14s/it]
 57%|█████▋    | 1266/2235 [3:33:53<2:44:19, 10.17s/it]
 57%|█████▋    | 1267/2235 [3:34:04<2:44:06, 10.17s/it]
 57%|█████▋    | 1268/2235 [3:34:14<2:44:13, 10.19s/it]
 57%|█████▋    | 1269/2235 
+0: {'loss': 0.5452, 'grad_norm': 1.2223570292734718, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: {'loss': 0.5355, 'grad_norm': 1.277729351874439, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.34}
+0: [3:34:24<2:43:24, 10.15s/it]
 57%|█████▋    | 1270/2235 [3:34:34<2:42:43, 10.12s/it]
                                                       

 57%|█████▋    | 1270/2235 [3:34:34<2:42:43, 10.12s/it]
 57%|█████▋    | 1271/2235 [3:34:44<2:42:09, 10.09s/it]
 57%|█████▋    | 1272/2235 [3:34:54<2:41:38, 10.07s/it]
 57%|█████▋    | 1273/2235 [3:35:04<2:41:04, 10.05s/it]
 57%|█████▋    | 1274/2235 [3:35:14<2:41:22, 10.08s/it]
 57%|█████���    | 1275/2235 [3:35:24<2:41:39, 10.10s/it]
 57%|█████▋    | 1276/2235 [3:35:34<2:41:17, 10.09s/it]
 57%|█████▋    | 1277/2235 [3:35:44<2:40:56, 10.08s/it]
 57%|█████▋    | 1278/2235 [3:35:54<2:39:58, 10.03s/it]
 57%|█████▋    | 1279/2235 [3:36:04<2:39:06,  9.99s/it]
 57%|█████▋    | 1280/2235 [3:36:14<2:38:36,  9.96s/it]
                                                       

 57%|█████▋    | 1280/2235 [3:36:14<2:38:36,  9.96s/i
+0: {'loss': 0.5433, 'grad_norm': 0.9931440903435789, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: t]
 57%|█████▋    | 1281/2235 [3:36:24<2:38:14,  9.95s/it]
 57%|█████▋    | 1282/2235 [3:36:34<2:37:56,  9.94s/it]
 57%|█████▋    | 1283/2235 [3:36:44<2:38:01,  9.96s/it]
 57%|█████▋    | 1284/2235 [3:36:54<2:38:08,  9.98s/it]
 57%|█████▋    | 1285/2235 [3:37:04<2:38:01,  9.98s/it]
 58%|█████▊    | 1286/2235 [3:37:14<2:37:58,  9.99s/it]
 58%|█████▊    | 1287/2235 [3:37:24<2:37:54,  9.99s/it]
 58%|█████▊    | 1288/2235 [3:37:34<2:37:22,  9.97s/it]
 58%|█████▊    | 1289/2235 [3:37:44<2:36:55,  9.95s/it]
 58%|█████▊    | 1290/2235 [3:37:54<2:36:26,  9.93s/it]
                                                       

 58%|█████▊    | 1290/2235 [3:37:54<2:36:26,  9.93s/it]
 58%|█████▊    | 1291/2235 [3:38:04<2:36:03,  9.92s/it]
 58%|█████▊    | 1292/2235 [3:38:13<2:35:50,  9.92s/it]
 58%|█████▊    | 1293/2235 [3:38:23<2:35:27,  9.90s/it]
 58%|██�
+0: {'loss': 0.5392, 'grad_norm': 0.9178488084224611, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: ��██▊    | 1294/2235 [3:38:33<2:35:12,  9.90s/it]
 58%|█████▊    | 1295/2235 [3:38:43<2:34:57,  9.89s/it]
 58%|█████▊    | 1296/2235 [3:38:53<2:34:45,  9.89s/it]
 58%|█████▊    | 1297/2235 [3:39:03<2:34:37,  9.89s/it]
 58%|█████▊    | 1298/2235 [3:39:13<2:34:17,  9.88s/it]
 58%|█████▊    | 1299/2235 [3:39:23<2:34:07,  9.88s/it]
 58%|█████▊    | 1300/2235 [3:39:32<2:34:00,  9.88s/it]
                                                       

 58%|█████▊    | 1300/2235 [3:39:32<2:34:00,  9.88s/it]
 58%|█████▊    | 1301/2235 [3:39:42<2:34:19,  9.91s/it]
 58%|█████▊    | 1302/2235 [3:39:52<2:34:11,  9.92s/it]
 58%|█████▊    | 1303/2235 [3:40:02<2:33:51,  9.91s/it]
 58%|█████▊    | 1304/2235 [3:40:12<2:34:26,  9.95s/it]
 58%|█████▊    | 1305/2235 [3:40:22<2:34:38,  9.98s/it]
 58%|█████▊    | 1306/2235 [3:40:32<2:34:41,  9.99s/it]
 58%|█████▊    
+0: {'loss': 0.5298, 'grad_norm': 1.0950674073525828, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: | 1307/2235 [3:40:42<2:34:28,  9.99s/it]
 59%|█████▊    | 1308/2235 [3:40:52<2:33:59,  9.97s/it]
 59%|█████▊    | 1309/2235 [3:41:02<2:34:02,  9.98s/it]
 59%|█████▊    | 1310/2235 [3:41:12<2:34:05,  9.99s/it]
                                                       

 59%|█████▊    | 1310/2235 [3:41:12<2:34:05,  9.99s/it]
 59%|█████▊    | 1311/2235 [3:41:22<2:33:42,  9.98s/it]
 59%|█████▊    | 1312/2235 [3:41:32<2:33:03,  9.95s/it]
 59%|█████▊    | 1313/2235 [3:41:42<2:32:52,  9.95s/it]
 59%|█████▉    | 1314/2235 [3:41:52<2:32:28,  9.93s/it]
 59%|█████▉    | 1315/2235 [3:42:02<2:32:24,  9.94s/it]
 59%|█████▉    | 1316/2235 [3:42:12<2:32:07,  9.93s/it]
 59%|█████▉    | 1317/2235 [3:42:22<2:32:08,  9.94s/it]
 59%|█████▉    | 1318/2235 [3:42:32<2:32:18,  9.97s/it]
 59%|█████▉    | 1319/2235 [3:42:42<2:32:16,  9.97s/it]
 59%|█████▉    | 1320/2235 [3:
+0: {'loss': 0.5524, 'grad_norm': 0.9051823724529209, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.35}
+0: {'loss': 0.5402, 'grad_norm': 0.9974191321954503, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0: 42:52<2:32:56, 10.03s/it]
                                                       

 59%|█████▉    | 1320/2235 [3:42:52<2:32:56, 10.03s/it]
 59%|█████▉    | 1321/2235 [3:43:02<2:33:09, 10.05s/it]
 59%|█████▉    | 1322/2235 [3:43:12<2:33:07, 10.06s/it]
 59%|█████▉    | 1323/2235 [3:43:22<2:33:01, 10.07s/it]
 59%|█████▉    | 1324/2235 [3:43:32<2:33:04, 10.08s/it]
 59%|█████▉    | 1325/2235 [3:43:43<2:33:06, 10.10s/it]
 59%|█████▉    | 1326/2235 [3:43:53<2:33:01, 10.10s/it]
 59%|█████▉    | 1327/2235 [3:44:03<2:33:00, 10.11s/it]
 59%|█████▉    | 1328/2235 [3:44:13<2:32:50, 10.11s/it]
 59%|█████▉    | 1329/2235 [3:44:23<2:32:38, 10.11s/it]
 60%|█████▉    | 1330/2235 [3:44:33<2:32:03, 10.08s/it]
                                                       

 60%|█████▉    | 1330/2235 [3:44:33<2:32:03, 10.08s/it]
 60%|█████▉    | 1331/2235 [3:44:43<2:31:41, 10.07s/it]
+0: {'loss': 0.5395, 'grad_norm': 0.9177231211428734, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0:  60%|█████▉    | 1332/2235 [3:44:53<2:31:17, 10.05s/it]
 60%|█████▉    | 1333/2235 [3:45:03<2:31:21, 10.07s/it]
 60%|█████▉    | 1334/2235 [3:45:13<2:31:10, 10.07s/it]
 60%|█████▉    | 1335/2235 [3:45:23<2:30:18, 10.02s/it]
 60%|█████▉    | 1336/2235 [3:45:33<2:29:54, 10.00s/it]
 60%|█████▉    | 1337/2235 [3:45:43<2:29:34,  9.99s/it]
 60%|█████▉    | 1338/2235 [3:45:53<2:29:15,  9.98s/it]
 60%|█████▉    | 1339/2235 [3:46:03<2:29:20, 10.00s/it]
 60%|█████▉    | 1340/2235 [3:46:13<2:29:17, 10.01s/it]
                                                       

 60%|█████▉    | 1340/2235 [3:46:13<2:29:17, 10.01s/it]
 60%|██████    | 1341/2235 [3:46:23<2:29:00, 10.00s/it]
 60%|██████    | 1342/2235 [3:46:33<2:28:47, 10.00s/it]
 60%|██████    | 1343/2235 [3:46:43<2:28:24,  9.98s/it]
 60%|██████    | 1344/2235 [3:46:53<2:27:53,  9.96s/it]
 60%|███�
+0: {'loss': 0.5398, 'grad_norm': 0.9435534974921693, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.36}
+0: ��██    | 1345/2235 [3:47:03<2:27:28,  9.94s/it]
 60%|██████    | 1346/2235 [3:47:13<2:27:16,  9.94s/it]
 60%|██████    | 1347/2235 [3:47:23<2:26:35,  9.90s/it]
 60%|██████    | 1348/2235 [3:47:32<2:26:14,  9.89s/it]
 60%|██████    | 1349/2235 [3:47:42<2:26:15,  9.90s/it]
 60%|██████    | 1350/2235 [3:47:52<2:26:01,  9.90s/it]
                                                       

 60%|██████    | 1350/2235 [3:47:52<2:26:01,  9.90s/it]
 60%|██████    | 1351/2235 [3:48:02<2:26:02,  9.91s/it]
 60%|██████    | 1352/2235 [3:48:12<2:25:45,  9.90s/it]
 61%|██████    | 1353/2235 [3:48:22<2:25:31,  9.90s/it]
 61%|██████    | 1354/2235 [3:48:32<2:25:21,  9.90s/it]
 61%|██████    | 1355/2235 [3:48:42<2:25:08,  9.90s/it]
 61%|██████    | 1356/2235 [3:48:52<2:25:04,  9.90s/it]
 61%|██████    | 1357/2235 [3:49:02<2:24:57,  9.91s/it]
 61%|██████    | 1
+0: {'loss': 0.5257, 'grad_norm': 1.0579617490088613, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0: {'loss': 0.5545, 'grad_norm': 1.1492003617420028, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0: 358/2235 [3:49:12<2:25:08,  9.93s/it]
 61%|██████    | 1359/2235 [3:49:21<2:24:48,  9.92s/it]
 61%|██████    | 1360/2235 [3:49:31<2:24:38,  9.92s/it]
                                                       

 61%|██████    | 1360/2235 [3:49:31<2:24:38,  9.92s/it]
 61%|██████    | 1361/2235 [3:49:41<2:24:28,  9.92s/it]
 61%|██████    | 1362/2235 [3:49:51<2:24:21,  9.92s/it]
 61%|██████    | 1363/2235 [3:50:01<2:23:57,  9.91s/it]
 61%|██████    | 1364/2235 [3:50:11<2:25:18, 10.01s/it]
 61%|██████    | 1365/2235 [3:50:21<2:24:49,  9.99s/it]
 61%|██████    | 1366/2235 [3:50:31<2:24:26,  9.97s/it]
 61%|██████    | 1367/2235 [3:50:41<2:24:02,  9.96s/it]
 61%|██████    | 1368/2235 [3:50:51<2:23:37,  9.94s/it]
 61%|██████▏   | 1369/2235 [3:51:01<2:23:13,  9.92s/it]
 61%|██████▏   | 1370/2235 [3:51:11<2:22:57,  9.92s/it]
                                         
+0: {'loss': 0.5315, 'grad_norm': 1.3576286419702297, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0:               

 61%|██████▏   | 1370/2235 [3:51:11<2:22:57,  9.92s/it]
 61%|██████▏   | 1371/2235 [3:51:21<2:22:55,  9.92s/it]
 61%|██████▏   | 1372/2235 [3:51:31<2:22:47,  9.93s/it]
 61%|██████▏   | 1373/2235 [3:51:41<2:22:38,  9.93s/it]
 61%|██████▏   | 1374/2235 [3:51:51<2:22:52,  9.96s/it]
 62%|██████▏   | 1375/2235 [3:52:01<2:22:28,  9.94s/it]
 62%|██████▏   | 1376/2235 [3:52:11<2:22:16,  9.94s/it]
 62%|██████▏   | 1377/2235 [3:52:21<2:22:26,  9.96s/it]
 62%|██████▏   | 1378/2235 [3:52:30<2:21:53,  9.93s/it]
 62%|██████▏   | 1379/2235 [3:52:40<2:21:27,  9.91s/it]
 62%|██████▏   | 1380/2235 [3:52:50<2:21:41,  9.94s/it]
                                                       

 62%|██████▏   | 1380/2235 [3:52:50<2:21:41,  9.94s/it]
 62%|██████▏   | 1381/2235 [3:53:01<2:23:52, 10.11s/it]
 62%|██████▏   | 1382/2235 
+0: {'loss': 0.5337, 'grad_norm': 0.9461653900562125, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.37}
+0: [3:53:11<2:23:19, 10.08s/it]
 62%|██████▏   | 1383/2235 [3:53:21<2:22:53, 10.06s/it]
 62%|██████▏   | 1384/2235 [3:53:31<2:22:41, 10.06s/it]
 62%|██████▏   | 1385/2235 [3:53:41<2:22:22, 10.05s/it]
 62%|██████▏   | 1386/2235 [3:53:51<2:22:02, 10.04s/it]
 62%|██████▏   | 1387/2235 [3:54:01<2:21:45, 10.03s/it]
 62%|██████▏   | 1388/2235 [3:54:11<2:21:15, 10.01s/it]
 62%|██████▏   | 1389/2235 [3:54:21<2:20:50,  9.99s/it]
 62%|██████▏   | 1390/2235 [3:54:31<2:21:04, 10.02s/it]
                                                       

 62%|██████▏   | 1390/2235 [3:54:31<2:21:04, 10.02s/it]
 62%|██████▏   | 1391/2235 [3:54:41<2:21:16, 10.04s/it]
 62%|██████▏   | 1392/2235 [3:54:51<2:21:06, 10.04s/it]
 62%|██████▏   | 1393/2235 [3:55:01<2:21:01, 10.05s/it]
 62%|██████▏   | 1394/2235 [3:55:11<2:20:49, 10.05s/it]
 62%|██████▏  
+0: {'loss': 0.5255, 'grad_norm': 1.025720014756807, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0:  | 1395/2235 [3:55:21<2:20:29, 10.04s/it]
 62%|██████▏   | 1396/2235 [3:55:31<2:20:21, 10.04s/it]
 63%|██████▎   | 1397/2235 [3:55:41<2:20:07, 10.03s/it]
 63%|██████▎   | 1398/2235 [3:55:51<2:20:16, 10.06s/it]
 63%|██████▎   | 1399/2235 [3:56:01<2:20:19, 10.07s/it]
 63%|██████▎   | 1400/2235 [3:56:12<2:20:13, 10.08s/it]
                                                       

 63%|██████▎   | 1400/2235 [3:56:12<2:20:13, 10.08s/it]
 63%|██████▎   | 1401/2235 [3:56:22<2:20:02, 10.07s/it]
 63%|██████▎   | 1402/2235 [3:56:32<2:19:45, 10.07s/it]
 63%|██████▎   | 1403/2235 [3:56:42<2:19:54, 10.09s/it]
 63%|██████▎   | 1404/2235 [3:56:52<2:20:27, 10.14s/it]
 63%|██████▎   | 1405/2235 [3:57:02<2:20:57, 10.19s/it]
 63%|██████▎   | 1406/2235 [3:57:13<2:21:20, 10.23s/it]
 63%|██████▎   | 1407/2235 [3:57:23<2:21:32, 10.26s/it]
 63%|███�
+0: {'loss': 0.5355, 'grad_norm': 0.8923593075519243, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0: ��██▎   | 1408/2235 [3:57:33<2:21:34, 10.27s/it]
 63%|██████▎   | 1409/2235 [3:57:44<2:21:19, 10.27s/it]
 63%|██████▎   | 1410/2235 [3:57:54<2:21:25, 10.29s/it]
                                                       

 63%|██████▎   | 1410/2235 [3:57:54<2:21:25, 10.29s/it]
 63%|██████▎   | 1411/2235 [3:58:04<2:21:27, 10.30s/it]
 63%|██████▎   | 1412/2235 [3:58:15<2:21:20, 10.30s/it]
 63%|██████▎   | 1413/2235 [3:58:25<2:21:10, 10.31s/it]
 63%|██████▎   | 1414/2235 [3:58:35<2:21:00, 10.31s/it]
 63%|██████▎   | 1415/2235 [3:58:45<2:20:52, 10.31s/it]
 63%|██████▎   | 1416/2235 [3:58:56<2:20:46, 10.31s/it]
 63%|██████▎   | 1417/2235 [3:59:06<2:20:36, 10.31s/it]
 63%|██████▎   | 1418/2235 [3:59:16<2:20:24, 10.31s/it]
 63%|██████▎   | 1419/2235 [3:59:27<2:20:10, 10.31s/it]
 64%|██████▎   | 1420/2235 [3:59:37<2:20:13, 10.32s/it]
  
+0: {'loss': 0.541, 'grad_norm': 0.9239288098741095, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0: {'loss': 0.5443, 'grad_norm': 0.9237919630437944, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.38}
+0:                                                      

 64%|██████▎   | 1420/2235 [3:59:37<2:20:13, 10.32s/it]
 64%|██████▎   | 1421/2235 [3:59:48<2:20:51, 10.38s/it]
 64%|██████▎   | 1422/2235 [3:59:58<2:20:57, 10.40s/it]
 64%|██████▎   | 1423/2235 [4:00:08<2:20:09, 10.36s/it]
 64%|██████▎   | 1424/2235 [4:00:19<2:19:41, 10.33s/it]
 64%|██████▍   | 1425/2235 [4:00:29<2:19:11, 10.31s/it]
 64%|██████▍   | 1426/2235 [4:00:39<2:18:55, 10.30s/it]
 64%|██████▍   | 1427/2235 [4:00:49<2:18:29, 10.28s/it]
 64%|██████▍   | 1428/2235 [4:00:59<2:17:35, 10.23s/it]
 64%|██████▍   | 1429/2235 [4:01:10<2:17:28, 10.23s/it]
 64%|██████▍   | 1430/2235 [4:01:20<2:17:14, 10.23s/it]
                                                       

 64%|██████▍   | 1430/2235 [4:01:20<2:17:14, 10.23s/it]
 64%|██████▍   | 1431/2235 [4:01:30<2:17:01, 10.23s/it]
 6
+0: {'loss': 0.5182, 'grad_norm': 0.969951812496169, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: 4%|██████▍   | 1432/2235 [4:01:40<2:16:17, 10.18s/it]
 64%|██████▍   | 1433/2235 [4:01:50<2:15:52, 10.17s/it]
 64%|██████▍   | 1434/2235 [4:02:01<2:15:44, 10.17s/it]
 64%|██████▍   | 1435/2235 [4:02:11<2:15:24, 10.16s/it]
 64%|██████▍   | 1436/2235 [4:02:21<2:14:53, 10.13s/it]
 64%|██████▍   | 1437/2235 [4:02:31<2:15:12, 10.17s/it]
 64%|██████▍   | 1438/2235 [4:02:41<2:14:50, 10.15s/it]
 64%|██████▍   | 1439/2235 [4:02:51<2:14:11, 10.12s/it]
 64%|██████▍   | 1440/2235 [4:03:01<2:13:39, 10.09s/it]
                                                       

 64%|██████▍   | 1440/2235 [4:03:01<2:13:39, 10.09s/it]
 64%|██████▍   | 1441/2235 [4:03:11<2:13:20, 10.08s/it]
 65%|██████▍   | 1442/2235 [4:03:21<2:12:53, 10.05s/it]
 65%|██████▍   | 1443/2235 [4:03:31<2:12:44, 10.06s/it]
 65%|██████▍   | 1444/2235 [4:03:41<2:12:39, 
+0: {'loss': 0.5295, 'grad_norm': 0.9472136965139941, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: 10.06s/it]
 65%|██████▍   | 1445/2235 [4:03:52<2:12:58, 10.10s/it]
 65%|██████▍   | 1446/2235 [4:04:02<2:13:08, 10.12s/it]
 65%|██████▍   | 1447/2235 [4:04:12<2:13:12, 10.14s/it]
 65%|██████▍   | 1448/2235 [4:04:22<2:12:47, 10.12s/it]
 65%|██████▍   | 1449/2235 [4:04:32<2:12:24, 10.11s/it]
 65%|██████▍   | 1450/2235 [4:04:42<2:12:08, 10.10s/it]
                                                       

 65%|██████▍   | 1450/2235 [4:04:42<2:12:08, 10.10s/it]
 65%|██████▍   | 1451/2235 [4:04:52<2:12:01, 10.10s/it]
 65%|██████▍   | 1452/2235 [4:05:02<2:12:14, 10.13s/it]
 65%|██████▌   | 1453/2235 [4:05:12<2:11:48, 10.11s/it]
 65%|██████▌   | 1454/2235 [4:05:23<2:11:17, 10.09s/it]
 65%|██████▌   | 1455/2235 [4:05:32<2:10:31, 10.04s/it]
 65%|██████▌   | 1456/2235 [4:05:42<2:09:46, 10.00s/it]
 65%|██████▌   | 1457/2235 [4:05
+0: {'loss': 0.5333, 'grad_norm': 1.0644573509401878, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: :52<2:09:13,  9.97s/it]
 65%|██████▌   | 1458/2235 [4:06:02<2:08:35,  9.93s/it]
 65%|██████▌   | 1459/2235 [4:06:12<2:08:05,  9.90s/it]
 65%|██████▌   | 1460/2235 [4:06:22<2:07:38,  9.88s/it]
                                                       

 65%|██████▌   | 1460/2235 [4:06:22<2:07:38,  9.88s/it]
 65%|██████▌   | 1461/2235 [4:06:32<2:07:49,  9.91s/it]
 65%|██████▌   | 1462/2235 [4:06:42<2:07:17,  9.88s/it]
 65%|██████▌   | 1463/2235 [4:06:51<2:07:01,  9.87s/it]
 66%|██████▌   | 1464/2235 [4:07:01<2:06:54,  9.88s/it]
 66%|██████▌   | 1465/2235 [4:07:11<2:06:43,  9.87s/it]
 66%|██████▌   | 1466/2235 [4:07:21<2:06:32,  9.87s/it]
 66%|██████▌   | 1467/2235 [4:07:31<2:06:13,  9.86s/it]
 66%|██████▌   | 1468/2235 [4:07:41<2:05:55,  9.85s/it]
 66%|██████▌   | 1469/2235 [4:07:50<2:05:34,  9.84s/it]
 66%|██████▌   | 14
+0: {'loss': 0.5229, 'grad_norm': 0.9988513201077958, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.39}
+0: {'loss': 0.5323, 'grad_norm': 1.0651772204452805, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: 70/2235 [4:08:00<2:05:26,  9.84s/it]
                                                       

 66%|██████▌   | 1470/2235 [4:08:00<2:05:26,  9.84s/it]
 66%|██████▌   | 1471/2235 [4:08:10<2:05:30,  9.86s/it]
 66%|██████▌   | 1472/2235 [4:08:20<2:05:15,  9.85s/it]
 66%|██████▌   | 1473/2235 [4:08:30<2:05:10,  9.86s/it]
 66%|██████▌   | 1474/2235 [4:08:40<2:05:24,  9.89s/it]
 66%|██████▌   | 1475/2235 [4:08:50<2:05:36,  9.92s/it]
 66%|██████▌   | 1476/2235 [4:09:00<2:05:39,  9.93s/it]
 66%|██████▌   | 1477/2235 [4:09:10<2:05:35,  9.94s/it]
 66%|██████▌   | 1478/2235 [4:09:20<2:05:24,  9.94s/it]
 66%|██████▌   | 1479/2235 [4:09:30<2:05:56, 10.00s/it]
 66%|██████▌   | 1480/2235 [4:09:41<2:09:26, 10.29s/it]
                                                       

 66%|██████▌   | 1480/2235 [4:09:41<2:09:26, 10.29s/it]
 66%|██████▋   | 14
+0: {'loss': 0.5248, 'grad_norm': 1.0806760814073337, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: 81/2235 [4:09:51<2:09:42, 10.32s/it]
 66%|██████▋   | 1482/2235 [4:10:01<2:09:18, 10.30s/it]
 66%|██████▋   | 1483/2235 [4:10:12<2:08:19, 10.24s/it]
 66%|██████▋   | 1484/2235 [4:10:22<2:07:24, 10.18s/it]
 66%|██████▋   | 1485/2235 [4:10:32<2:06:40, 10.13s/it]
 66%|██████▋   | 1486/2235 [4:10:42<2:07:04, 10.18s/it]
 67%|██████▋   | 1487/2235 [4:10:52<2:07:31, 10.23s/it]
 67%|██████▋   | 1488/2235 [4:11:02<2:06:39, 10.17s/it]
 67%|██████▋   | 1489/2235 [4:11:12<2:05:45, 10.12s/it]
 67%|██████▋   | 1490/2235 [4:11:22<2:05:27, 10.10s/it]
                                                       

 67%|██████▋   | 1490/2235 [4:11:22<2:05:27, 10.10s/it]
 67%|██████▋   | 1491/2235 [4:11:32<2:04:38, 10.05s/it]
 67%|██████▋   | 1492/2235 [4:11:42<2:04:05, 10.02s/it]
 67%|██████▋   | 1493/2235 [4:11:52<2:03:42, 10.00s/it]
 67%|█████
+0: {'loss': 0.5253, 'grad_norm': 1.0071617266730406, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.4}
+0: █▋   | 1494/2235 [4:12:02<2:03:20,  9.99s/it]
 67%|██████▋   | 1495/2235 [4:12:12<2:03:10,  9.99s/it]
 67%|██████▋   | 1496/2235 [4:12:22<2:02:52,  9.98s/it]
 67%|██████▋   | 1497/2235 [4:12:32<2:02:22,  9.95s/it]
 67%|██████▋   | 1498/2235 [4:12:42<2:02:07,  9.94s/it]
 67%|██████▋   | 1499/2235 [4:12:52<2:01:51,  9.93s/it]
 67%|██████▋   | 1500/2235 [4:13:02<2:01:40,  9.93s/it]
                                                       

 67%|██████▋   | 1500/2235 [4:13:02<2:01:40,  9.93s/it]
 67%|██████▋   | 1501/2235 [4:13:12<2:01:29,  9.93s/it]
 67%|██████▋   | 1502/2235 [4:13:22<2:01:14,  9.92s/it]
 67%|██████▋   | 1503/2235 [4:13:32<2:01:01,  9.92s/it]
 67%|██████▋   | 1504/2235 [4:13:41<2:00:47,  9.91s/it]
 67%|██████▋   | 1505/2235 [4:13:51<2:00:35,  9.91s/it]
 67%|██████▋   | 1506/2235 [4:14:01<2:00:29,  9.92s/it]
 67%|�
+0: {'loss': 0.5176, 'grad_norm': 0.9355454125125093, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: �█████▋   | 1507/2235 [4:14:11<2:00:34,  9.94s/it]
 67%|██████▋   | 1508/2235 [4:14:21<2:00:38,  9.96s/it]
 68%|██████▊   | 1509/2235 [4:14:31<2:00:40,  9.97s/it]
 68%|██████▊   | 1510/2235 [4:14:41<2:00:32,  9.98s/it]
                                                       

 68%|██████▊   | 1510/2235 [4:14:41<2:00:32,  9.98s/it]
 68%|██████▊   | 1511/2235 [4:14:51<2:00:26,  9.98s/it]
 68%|██████▊   | 1512/2235 [4:15:01<2:00:03,  9.96s/it]
 68%|██████▊   | 1513/2235 [4:15:11<1:59:44,  9.95s/it]
 68%|██████▊   | 1514/2235 [4:15:21<1:59:52,  9.98s/it]
 68%|██████▊   | 1515/2235 [4:15:31<1:59:02,  9.92s/it]
 68%|██████▊   | 1516/2235 [4:15:41<1:58:47,  9.91s/it]
 68%|██████▊   | 1517/2235 [4:15:51<1:58:28,  9.90s/it]
 68%|██████▊   | 1518/2235 [4:16:01<1:58:34,  9.92s/it]
 68%|██████▊   | 1519/2235 [4:16:11<1:58:53,  9.96
+0: {'loss': 0.5143, 'grad_norm': 0.8762942058523264, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: {'loss': 0.5496, 'grad_norm': 1.1138772658610565, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: s/it]
 68%|██████▊   | 1520/2235 [4:16:21<1:58:44,  9.96s/it]
                                                       

 68%|██████▊   | 1520/2235 [4:16:21<1:58:44,  9.96s/it]
 68%|██████▊   | 1521/2235 [4:16:31<1:58:37,  9.97s/it]
 68%|██████▊   | 1522/2235 [4:16:41<1:58:33,  9.98s/it]
 68%|██████▊   | 1523/2235 [4:16:51<1:58:25,  9.98s/it]
 68%|██████▊   | 1524/2235 [4:17:01<1:58:06,  9.97s/it]
 68%|██████▊   | 1525/2235 [4:17:11<1:58:01,  9.97s/it]
 68%|██████▊   | 1526/2235 [4:17:21<1:58:01,  9.99s/it]
 68%|██████▊   | 1527/2235 [4:17:31<1:58:02, 10.00s/it]
 68%|██████▊   | 1528/2235 [4:17:41<1:57:45,  9.99s/it]
 68%|██████▊   | 1529/2235 [4:17:51<1:57:45, 10.01s/it]
 68%|██████▊   | 1530/2235 [4:18:01<1:57:21,  9.99s/it]
                                                       

 68%|██████▊   | 1530/2235 [4:18:01<1:57:21,  9.99
+0: {'loss': 0.5236, 'grad_norm': 0.9811894137033534, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.41}
+0: s/it]
 69%|██████▊   | 1531/2235 [4:18:10<1:56:58,  9.97s/it]
 69%|██████▊   | 1532/2235 [4:18:21<1:57:02,  9.99s/it]
 69%|██████▊   | 1533/2235 [4:18:30<1:56:41,  9.97s/it]
 69%|██████▊   | 1534/2235 [4:18:40<1:56:13,  9.95s/it]
 69%|██████▊   | 1535/2235 [4:18:50<1:55:52,  9.93s/it]
 69%|██████▊   | 1536/2235 [4:19:00<1:55:58,  9.96s/it]
 69%|██████▉   | 1537/2235 [4:19:10<1:55:51,  9.96s/it]
 69%|██████▉   | 1538/2235 [4:19:20<1:55:50,  9.97s/it]
 69%|██████▉   | 1539/2235 [4:19:30<1:56:01, 10.00s/it]
 69%|██████▉   | 1540/2235 [4:19:40<1:56:06, 10.02s/it]
                                                       

 69%|██████▉   | 1540/2235 [4:19:40<1:56:06, 10.02s/it]
 69%|██████▉   | 1541/2235 [4:19:50<1:56:11, 10.05s/it]
 69%|██████▉   | 1542/2235 [4:20:01<1:56:07, 10.05s/it]
 69%|██████▉   | 1543/2235 [4:20:11<1
+0: {'loss': 0.5304, 'grad_norm': 0.8776346074119433, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: :55:52, 10.05s/it]
 69%|██████▉   | 1544/2235 [4:20:21<1:55:40, 10.04s/it]
 69%|██████▉   | 1545/2235 [4:20:32<2:00:26, 10.47s/it]
 69%|██████▉   | 1546/2235 [4:20:43<2:00:22, 10.48s/it]
 69%|██████▉   | 1547/2235 [4:20:57<2:13:52, 11.68s/it]
 69%|██████▉   | 1548/2235 [4:21:07<2:08:41, 11.24s/it]
 69%|██████▉   | 1549/2235 [4:21:17<2:04:39, 10.90s/it]
 69%|██████▉   | 1550/2235 [4:21:28<2:01:46, 10.67s/it]
                                                       

 69%|██████▉   | 1550/2235 [4:21:28<2:01:46, 10.67s/it]
 69%|██████▉   | 1551/2235 [4:21:38<1:59:49, 10.51s/it]
 69%|██████▉   | 1552/2235 [4:21:48<1:57:59, 10.37s/it]
 69%|██████▉   | 1553/2235 [4:21:58<1:56:32, 10.25s/it]
 70%|██████▉   | 1554/2235 [4:22:08<1:55:17, 10.16s/it]
 70%|██████▉   | 1555/2235 [4:22:18<1:54:22, 10.09s/it]
 70%|██████▉   | 1556/22
+0: {'loss': 0.527, 'grad_norm': 0.9090640484096179, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: 35 [4:22:28<1:53:50, 10.06s/it]
 70%|██████▉   | 1557/2235 [4:22:38<1:53:39, 10.06s/it]
 70%|██████▉   | 1558/2235 [4:22:48<1:53:24, 10.05s/it]
 70%|██████▉   | 1559/2235 [4:22:58<1:53:08, 10.04s/it]
 70%|██████▉   | 1560/2235 [4:23:08<1:52:56, 10.04s/it]
                                                       

 70%|██████▉   | 1560/2235 [4:23:08<1:52:56, 10.04s/it]
 70%|██████▉   | 1561/2235 [4:23:18<1:52:31, 10.02s/it]
 70%|██████▉   | 1562/2235 [4:23:28<1:52:17, 10.01s/it]
 70%|██████▉   | 1563/2235 [4:23:38<1:52:27, 10.04s/it]
 70%|██████▉   | 1564/2235 [4:23:48<1:52:16, 10.04s/it]
 70%|███████   | 1565/2235 [4:23:58<1:52:20, 10.06s/it]
 70%|███████   | 1566/2235 [4:24:08<1:52:23, 10.08s/it]
 70%|███████   | 1567/2235 [4:24:18<1:52:18, 10.09s/it]
 70%|███████   | 1568/2235 [4:24:28<1:51:58, 10.07s/it]
 70%|██████�
+0: {'loss': 0.5236, 'grad_norm': 1.6255095246947362, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: {'loss': 0.5417, 'grad_norm': 1.0059922053571586, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.42}
+0: �   | 1569/2235 [4:24:38<1:51:45, 10.07s/it]
 70%|███████   | 1570/2235 [4:24:48<1:51:35, 10.07s/it]
                                                       

 70%|███████   | 1570/2235 [4:24:48<1:51:35, 10.07s/it]
 70%|███████   | 1571/2235 [4:24:58<1:51:27, 10.07s/it]
 70%|███████   | 1572/2235 [4:25:08<1:51:27, 10.09s/it]
 70%|███████   | 1573/2235 [4:25:19<1:51:14, 10.08s/it]
 70%|███████   | 1574/2235 [4:25:29<1:50:59, 10.08s/it]
 70%|███████   | 1575/2235 [4:25:39<1:50:23, 10.04s/it]
 71%|███████   | 1576/2235 [4:25:49<1:50:11, 10.03s/it]
 71%|███████   | 1577/2235 [4:25:59<1:50:05, 10.04s/it]
 71%|███████   | 1578/2235 [4:26:09<1:49:47, 10.03s/it]
 71%|███████   | 1579/2235 [4:26:19<1:49:29, 10.01s/it]
 71%|███████   | 1580/2235 [4:26:29<1:50:05, 10.08s/it]
                                                       

 71%|██████�
+0: {'loss': 0.5249, 'grad_norm': 1.082360033356086, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: �   | 1580/2235 [4:26:29<1:50:05, 10.08s/it]
 71%|███████   | 1581/2235 [4:26:39<1:49:41, 10.06s/it]
 71%|███████   | 1582/2235 [4:26:49<1:49:14, 10.04s/it]
 71%|███████   | 1583/2235 [4:26:59<1:48:58, 10.03s/it]
 71%|███████   | 1584/2235 [4:27:09<1:48:52, 10.03s/it]
 71%|███████   | 1585/2235 [4:27:19<1:49:10, 10.08s/it]
 71%|███████   | 1586/2235 [4:27:29<1:48:34, 10.04s/it]
 71%|███████   | 1587/2235 [4:27:39<1:47:57, 10.00s/it]
 71%|███████   | 1588/2235 [4:27:49<1:47:22,  9.96s/it]
 71%|███████   | 1589/2235 [4:27:59<1:46:34,  9.90s/it]
 71%|███████   | 1590/2235 [4:28:08<1:46:08,  9.87s/it]
                                                       

 71%|███████   | 1590/2235 [4:28:08<1:46:08,  9.87s/it]
 71%|███████   | 1591/2235 [4:28:18<1:46:14,  9.90s/it]
 71%|███████   | 1592/2235 [4:28:28<1:46:13,  9.91s/it]
 71%|██�
+0: {'loss': 0.5253, 'grad_norm': 1.0487080271625273, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: ��████▏  | 1593/2235 [4:28:38<1:46:00,  9.91s/it]
 71%|███████▏  | 1594/2235 [4:28:48<1:45:40,  9.89s/it]
 71%|███████▏  | 1595/2235 [4:28:58<1:45:29,  9.89s/it]
 71%|███████▏  | 1596/2235 [4:29:08<1:45:09,  9.87s/it]
 71%|███████▏  | 1597/2235 [4:29:18<1:44:54,  9.87s/it]
 71%|███████▏  | 1598/2235 [4:29:27<1:44:31,  9.84s/it]
 72%|███████▏  | 1599/2235 [4:29:37<1:44:30,  9.86s/it]
 72%|███████▏  | 1600/2235 [4:29:47<1:44:12,  9.85s/it]
                                                       

 72%|███████▏  | 1600/2235 [4:29:47<1:44:12,  9.85s/it]
 72%|███████▏  | 1601/2235 [4:29:57<1:44:07,  9.85s/it]
 72%|███████▏  | 1602/2235 [4:30:07<1:44:22,  9.89s/it]
 72%|███████▏  | 1603/2235 [4:30:17<1:44:28,  9.92s/it]
 72%|███████▏  | 1604/2235 [4:30:27<1:44:22,  9.92s/it]
 72%|███████▏  | 1605/2235 
+0: {'loss': 0.5109, 'grad_norm': 0.9464030506243681, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: [4:30:37<1:44:11,  9.92s/it]
 72%|███████▏  | 1606/2235 [4:30:47<1:43:46,  9.90s/it]
 72%|███████▏  | 1607/2235 [4:30:56<1:43:23,  9.88s/it]
 72%|███████▏  | 1608/2235 [4:31:06<1:43:04,  9.86s/it]
 72%|███████▏  | 1609/2235 [4:31:16<1:42:57,  9.87s/it]
 72%|███████▏  | 1610/2235 [4:31:26<1:42:49,  9.87s/it]
                                                       

 72%|███████▏  | 1610/2235 [4:31:26<1:42:49,  9.87s/it]
 72%|███████▏  | 1611/2235 [4:31:36<1:42:46,  9.88s/it]
 72%|███████▏  | 1612/2235 [4:31:46<1:42:40,  9.89s/it]
 72%|███████▏  | 1613/2235 [4:31:56<1:42:30,  9.89s/it]
 72%|███████▏  | 1614/2235 [4:32:06<1:42:04,  9.86s/it]
 72%|███████▏  | 1615/2235 [4:32:15<1:42:08,  9.88s/it]
 72%|███████▏  | 1616/2235 [4:32:25<1:41:58,  9.89s/it]
 72%|███████▏  | 1617/2235 [4:32:35<1:41:52,  9.89s/it]
 7
+0: {'loss': 0.5317, 'grad_norm': 1.2862524944875569, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.43}
+0: 2%|███████▏  | 1618/2235 [4:32:45<1:41:37,  9.88s/it]
 72%|███████▏  | 1619/2235 [4:32:55<1:41:18,  9.87s/it]
 72%|███████▏  | 1620/2235 [4:33:05<1:41:15,  9.88s/it]
                                                       

 72%|███████▏  | 1620/2235 [4:33:05<1:41:15,  9.88s/it]
 73%|███████▎  | 1621/2235 [4:33:15<1:41:04,  9.88s/it]
 73%|███████▎  | 1622/2235 [4:33:25<1:40:56,  9.88s/it]
 73%|███████▎  | 1623/2235 [4:33:35<1:40:49,  9.89s/it]
 73%|███████▎  | 1624/2235 [4:33:44<1:40:42,  9.89s/it]
 73%|███████▎  | 1625/2235 [4:33:55<1:41:28,  9.98s/it]
 73%|███████▎  | 1626/2235 [4:34:05<1:40:58,  9.95s/it]
 73%|███████▎  | 1627/2235 [4:34:14<1:40:41,  9.94s/it]
 73%|███████▎  | 1628/2235 [4:34:24<1:40:18,  9.92s/it]
 73%|███████▎  | 1629/2235 [4:34:34<1:40:11,  9.92s/it]
 73%|███████▎  | 
+0: {'loss': 0.5317, 'grad_norm': 1.1278391508778831, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0: {'loss': 0.5264, 'grad_norm': 0.9140223871129838, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0: 1630/2235 [4:34:44<1:40:28,  9.96s/it]
                                                       

 73%|███████▎  | 1630/2235 [4:34:44<1:40:28,  9.96s/it]
 73%|███████▎  | 1631/2235 [4:34:54<1:40:21,  9.97s/it]
 73%|███████▎  | 1632/2235 [4:35:04<1:40:01,  9.95s/it]
 73%|███████▎  | 1633/2235 [4:35:14<1:40:08,  9.98s/it]
 73%|███████▎  | 1634/2235 [4:35:24<1:39:45,  9.96s/it]
 73%|███████▎  | 1635/2235 [4:35:34<1:39:23,  9.94s/it]
 73%|███████▎  | 1636/2235 [4:35:44<1:38:54,  9.91s/it]
 73%|███████▎  | 1637/2235 [4:35:54<1:38:32,  9.89s/it]
 73%|███████▎  | 1638/2235 [4:36:04<1:38:14,  9.87s/it]
 73%|███████▎  | 1639/2235 [4:36:13<1:38:13,  9.89s/it]
 73%|███████▎  | 1640/2235 [4:36:24<1:38:35,  9.94s/it]
                                                       

 73%|███████▎  | 1640/2235 [4:36:24<1:38:35,  9.94s/it]
 73%|�
+0: {'loss': 0.5193, 'grad_norm': 1.0599793732601184, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.44}
+0: �██████▎  | 1641/2235 [4:36:34<1:38:34,  9.96s/it]
 73%|███████▎  | 1642/2235 [4:36:44<1:38:52, 10.00s/it]
 74%|███████▎  | 1643/2235 [4:36:54<1:38:43, 10.01s/it]
 74%|███████▎  | 1644/2235 [4:37:04<1:38:27, 10.00s/it]
 74%|███████▎  | 1645/2235 [4:37:14<1:38:08,  9.98s/it]
 74%|███████▎  | 1646/2235 [4:37:24<1:37:56,  9.98s/it]
 74%|███████▎  | 1647/2235 [4:37:34<1:37:46,  9.98s/it]
 74%|███████▎  | 1648/2235 [4:37:43<1:37:34,  9.97s/it]
 74%|███████▍  | 1649/2235 [4:37:53<1:37:08,  9.95s/it]
 74%|███████▍  | 1650/2235 [4:38:03<1:36:53,  9.94s/it]
                                                       

 74%|███████▍  | 1650/2235 [4:38:03<1:36:53,  9.94s/it]
 74%|███████▍  | 1651/2235 [4:38:13<1:36:48,  9.95s/it]
 74%|███████▍  | 1652/2235 [4:38:23<1:36:31,  9.93s/it]
 74%|███████▍  | 1653/
+0: {'loss': 0.5268, 'grad_norm': 1.0658683513367957, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: 2235 [4:38:33<1:36:22,  9.94s/it]
 74%|███████▍  | 1654/2235 [4:38:43<1:36:08,  9.93s/it]
 74%|███████▍  | 1655/2235 [4:38:53<1:35:53,  9.92s/it]
 74%|███████▍  | 1656/2235 [4:39:03<1:35:36,  9.91s/it]
 74%|███████▍  | 1657/2235 [4:39:13<1:35:25,  9.91s/it]
 74%|███████▍  | 1658/2235 [4:39:23<1:35:13,  9.90s/it]
 74%|███████▍  | 1659/2235 [4:39:34<1:40:02, 10.42s/it]
 74%|███████▍  | 1660/2235 [4:39:44<1:38:23, 10.27s/it]
                                                       

 74%|███████▍  | 1660/2235 [4:39:44<1:38:23, 10.27s/it]
 74%|███████▍  | 1661/2235 [4:39:54<1:37:03, 10.15s/it]
 74%|███████▍  | 1662/2235 [4:40:04<1:36:01, 10.06s/it]
 74%|███████▍  | 1663/2235 [4:40:14<1:35:21, 10.00s/it]
 74%|███████▍  | 1664/2235 [4:40:24<1:35:11, 10.00s/it]
 74%|███████▍  | 1665/2235 [4:40:34<1:35:02, 10.01s/i
+0: {'loss': 0.5286, 'grad_norm': 0.9831547304695571, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: t]
 75%|███████▍  | 1666/2235 [4:40:44<1:36:28, 10.17s/it]
 75%|███████▍  | 1667/2235 [4:40:54<1:35:47, 10.12s/it]
 75%|███████▍  | 1668/2235 [4:41:05<1:36:42, 10.23s/it]
 75%|███████▍  | 1669/2235 [4:41:15<1:37:15, 10.31s/it]
 75%|███████▍  | 1670/2235 [4:41:26<1:37:18, 10.33s/it]
                                                       

 75%|███████▍  | 1670/2235 [4:41:26<1:37:18, 10.33s/it]
 75%|███████▍  | 1671/2235 [4:41:37<1:39:06, 10.54s/it]
 75%|███████▍  | 1672/2235 [4:41:47<1:37:19, 10.37s/it]
 75%|███████▍  | 1673/2235 [4:41:57<1:38:13, 10.49s/it]
 75%|███████▍  | 1674/2235 [4:42:08<1:37:41, 10.45s/it]
 75%|███████▍  | 1675/2235 [4:42:18<1:35:57, 10.28s/it]
 75%|███████▍  | 1676/2235 [4:42:28<1:34:40, 10.16s/it]
 75%|███████▌  | 1677/2235 [4:42:37<1:33:40, 10.07s/it]
 75%|███████�
+0: {'loss': 0.504, 'grad_norm': 0.8846135503402132, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: �  | 1678/2235 [4:42:47<1:32:51, 10.00s/it]
 75%|███████▌  | 1679/2235 [4:42:57<1:32:20,  9.97s/it]
 75%|███████▌  | 1680/2235 [4:43:07<1:31:53,  9.93s/it]
                                                       

 75%|███████▌  | 1680/2235 [4:43:07<1:31:53,  9.93s/it]
 75%|███████▌  | 1681/2235 [4:43:17<1:31:50,  9.95s/it]
 75%|███████▌  | 1682/2235 [4:43:27<1:31:44,  9.95s/it]
 75%|███████▌  | 1683/2235 [4:43:37<1:31:32,  9.95s/it]
 75%|███████▌  | 1684/2235 [4:43:47<1:31:18,  9.94s/it]
 75%|███████▌  | 1685/2235 [4:43:57<1:31:07,  9.94s/it]
 75%|███████▌  | 1686/2235 [4:44:07<1:31:44, 10.03s/it]
 75%|███████▌  | 1687/2235 [4:44:17<1:31:45, 10.05s/it]
 76%|███████▌  | 1688/2235 [4:44:27<1:32:23, 10.13s/it]
 76%|███████▌  | 1689/2235 [4:44:38<1:33:03, 10.23s/it]
 76%|███████▌  | 1690/2235 [4:44:48<1:32:33
+0: {'loss': 0.5236, 'grad_norm': 0.8524855065497912, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.45}
+0: {'loss': 0.5179, 'grad_norm': 0.9424870621495516, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: , 10.19s/it]
                                                       

 76%|███████▌  | 1690/2235 [4:44:48<1:32:33, 10.19s/it]
 76%|███████▌  | 1691/2235 [4:44:58<1:32:19, 10.18s/it]
 76%|███████▌  | 1692/2235 [4:45:08<1:32:19, 10.20s/it]
 76%|███████▌  | 1693/2235 [4:45:19<1:32:05, 10.20s/it]
 76%|███████▌  | 1694/2235 [4:45:29<1:31:47, 10.18s/it]
 76%|███████▌  | 1695/2235 [4:45:38<1:30:38, 10.07s/it]
 76%|███████▌  | 1696/2235 [4:45:48<1:29:33,  9.97s/it]
 76%|███████▌  | 1697/2235 [4:45:58<1:28:48,  9.90s/it]
 76%|███████▌  | 1698/2235 [4:46:08<1:28:53,  9.93s/it]
 76%|███████▌  | 1699/2235 [4:46:18<1:28:33,  9.91s/it]
 76%|███████▌  | 1700/2235 [4:46:28<1:28:30,  9.93s/it]
                                                       

 76%|███████▌  | 1700/2235 [4:46:28<1:28:30,  9.93s/it]
 76%|███████▌  | 
+0: {'loss': 0.5334, 'grad_norm': 0.8782059513402034, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: 1701/2235 [4:46:38<1:28:17,  9.92s/it]
 76%|███████▌  | 1702/2235 [4:46:48<1:28:30,  9.96s/it]
 76%|███████▌  | 1703/2235 [4:46:58<1:28:27,  9.98s/it]
 76%|███████▌  | 1704/2235 [4:47:08<1:27:57,  9.94s/it]
 76%|███████▋  | 1705/2235 [4:47:18<1:27:42,  9.93s/it]
 76%|███████▋  | 1706/2235 [4:47:28<1:28:26, 10.03s/it]
 76%|███████▋  | 1707/2235 [4:47:38<1:28:01, 10.00s/it]
 76%|███████▋  | 1708/2235 [4:47:48<1:27:54, 10.01s/it]
 76%|███████▋  | 1709/2235 [4:47:58<1:27:52, 10.02s/it]
 77%|███████▋  | 1710/2235 [4:48:08<1:27:43, 10.03s/it]
                                                       

 77%|███████▋  | 1710/2235 [4:48:08<1:27:43, 10.03s/it]
 77%|███████▋  | 1711/2235 [4:48:18<1:27:12,  9.99s/it]
 77%|███████▋  | 1712/2235 [4:48:28<1:26:49,  9.96s/it]
 77%|███████▋  | 1713/2235 [4:48:38<1:26:30,  9.
+0: {'loss': 0.5297, 'grad_norm': 1.234223153908368, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: 94s/it]
 77%|███████▋  | 1714/2235 [4:48:48<1:26:53, 10.01s/it]
 77%|███████▋  | 1715/2235 [4:48:58<1:26:23,  9.97s/it]
 77%|███████▋  | 1716/2235 [4:49:08<1:26:11,  9.97s/it]
 77%|█���█████▋  | 1717/2235 [4:49:18<1:26:05,  9.97s/it]
 77%|███████▋  | 1718/2235 [4:49:28<1:25:55,  9.97s/it]
 77%|███████▋  | 1719/2235 [4:49:37<1:25:43,  9.97s/it]
 77%|███████▋  | 1720/2235 [4:49:47<1:25:25,  9.95s/it]
                                                       

 77%|███████▋  | 1720/2235 [4:49:47<1:25:25,  9.95s/it]
 77%|███████▋  | 1721/2235 [4:49:57<1:25:20,  9.96s/it]
 77%|███████▋  | 1722/2235 [4:50:07<1:25:13,  9.97s/it]
 77%|███████▋  | 1723/2235 [4:50:17<1:25:06,  9.97s/it]
 77%|███████▋  | 1724/2235 [4:50:28<1:25:40, 10.06s/it]
 77%|███████▋  | 1725/2235 [4:50:38<1:25:27, 10.05s/it]
 77%|██████
+0: {'loss': 0.5337, 'grad_norm': 0.9348934392490296, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.46}
+0: █▋  | 1726/2235 [4:50:48<1:25:02, 10.03s/it]
 77%|███████▋  | 1727/2235 [4:50:58<1:24:49, 10.02s/it]
 77%|███████▋  | 1728/2235 [4:51:08<1:24:31, 10.00s/it]
 77%|███████▋  | 1729/2235 [4:51:18<1:24:17, 10.00s/it]
 77%|███████▋  | 1730/2235 [4:51:27<1:24:02,  9.98s/it]
                                                       

 77%|███████▋  | 1730/2235 [4:51:28<1:24:02,  9.98s/it]
 77%|███████▋  | 1731/2235 [4:51:37<1:23:46,  9.97s/it]
 77%|███████▋  | 1732/2235 [4:51:47<1:23:25,  9.95s/it]
 78%|███████▊  | 1733/2235 [4:51:57<1:23:01,  9.92s/it]
 78%|███████▊  | 1734/2235 [4:52:07<1:22:32,  9.89s/it]
 78%|███████▊  | 1735/2235 [4:52:17<1:22:18,  9.88s/it]
 78%|███████▊  | 1736/2235 [4:52:27<1:21:55,  9.85s/it]
 78%|███████▊  | 1737/2235 [4:52:36<1:21:35,  9.83s/it]
 78%|███████▊  | 1738/2235 [4:52:46<1:
+0: {'loss': 0.5165, 'grad_norm': 0.9446889154890559, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.47}
+0: 21:33,  9.85s/it]
 78%|███████▊  | 1739/2235 [4:52:56<1:21:26,  9.85s/it]
 78%|███████▊  | 1740/2235 [4:53:06<1:21:19,  9.86s/it]
                                                       

 78%|███████▊  | 1740/2235 [4:53:06<1:21:19,  9.86s/it]
 78%|███████▊  | 1741/2235 [4:53:16<1:21:19,  9.88s/it]
 78%|███████▊  | 1742/2235 [4:53:26<1:21:05,  9.87s/it]
 78%|███████▊  | 1743/2235 [4:53:36<1:21:03,  9.88s/it]
 78%|███████▊  | 1744/2235 [4:53:46<1:21:01,  9.90s/it]
 78%|███████▊  | 1745/2235 [4:53:56<1:20:46,  9.89s/it]
 78%|███████▊  | 1746/2235 [4:54:05<1:20:37,  9.89s/it]
 78%|███████▊  | 1747/2235 [4:54:15<1:20:26,  9.89s/it]
 78%|███████▊  | 1748/2235 [4:54:25<1:20:26,  9.91s/it]
 78%|███████▊  | 1749/2235 [4:54:35<1:20:50,  9.98s/it]
 78%|███████▊  | 1750/2235 [4:54:46<1:21:14, 10.05s/it]
             
+0: {'loss': 0.5279, 'grad_norm': 0.8773205039301344, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.47}
+0: {'loss': 0.5136, 'grad_norm': 0.8953346924053497, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.47}
+0:                                           

 78%|███████▊  | 1750/2235 [4:54:46<1:21:14, 10.05s/it]
 78%|███████▊  | 1751/2235 [4:54:56<1:21:32, 10.11s/it]
 78%|███████▊  | 1752/2235 [4:55:06<1:21:26, 10.12s/it]
 78%|███████▊  | 1753/2235 [4:55:16<1:21:08, 10.10s/it]
 78%|███████▊  | 1754/2235 [4:55:26<1:21:00, 10.11s/it]
 79%|███████▊  | 1755/2235 [4:55:36<1:20:36, 10.08s/it]
 79%|███████▊  | 1756/2235 [4:55:46<1:20:15, 10.05s/it]
 79%|███████▊  | 1757/2235 [4:55:56<1:20:12, 10.07s/it]
 79%|███████▊  | 1758/2235 [4:56:06<1:20:07, 10.08s/it]
 79%|███████▊  | 1759/2235 [4:56:17<1:19:58, 10.08s/it]
 79%|███████▊  | 1760/2235 [4:56:27<1:19:59, 10.10s/it]
                                                       

 79%|███████▊  | 1760/2235 [4:56:27<1:19:59, 10.10s/it]
 79%|███████▉  | 1761/2235 [4:56:37<1:20:01
+0: {'loss': 0.5126, 'grad_norm': 0.9959745700099923, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: , 10.13s/it]
 79%|███████▉  | 1762/2235 [4:56:47<1:19:51, 10.13s/it]
 79%|███████▉  | 1763/2235 [4:56:57<1:19:33, 10.11s/it]
 79%|███████▉  | 1764/2235 [4:57:07<1:19:16, 10.10s/it]
 79%|███████▉  | 1765/2235 [4:57:17<1:19:02, 10.09s/it]
 79%|███████▉  | 1766/2235 [4:57:27<1:18:46, 10.08s/it]
 79%|███████▉  | 1767/2235 [4:57:37<1:18:48, 10.10s/it]
 79%|███████▉  | 1768/2235 [4:57:47<1:18:36, 10.10s/it]
 79%|███████▉  | 1769/2235 [4:57:58<1:18:18, 10.08s/it]
 79%|███████▉  | 1770/2235 [4:58:08<1:18:10, 10.09s/it]
                                                       

 79%|███████▉  | 1770/2235 [4:58:08<1:18:10, 10.09s/it]
 79%|███████▉  | 1771/2235 [4:58:18<1:17:35, 10.03s/it]
 79%|███████▉  | 1772/2235 [4:58:27<1:16:59,  9.98s/it]
 79%|███████▉  | 1773/2235 [4:58:37<1:16:31,  9.94s/it]
 79%|████�
+0: {'loss': 0.5172, 'grad_norm': 0.8928086942913689, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: ��██▉  | 1774/2235 [4:58:47<1:16:06,  9.91s/it]
 79%|███████▉  | 1775/2235 [4:58:57<1:15:58,  9.91s/it]
 79%|███████▉  | 1776/2235 [4:59:07<1:15:47,  9.91s/it]
 80%|███████▉  | 1777/2235 [4:59:17<1:15:28,  9.89s/it]
 80%|███████▉  | 1778/2235 [4:59:27<1:15:19,  9.89s/it]
 80%|███████▉  | 1779/2235 [4:59:37<1:15:13,  9.90s/it]
 80%|███████▉  | 1780/2235 [4:59:46<1:15:05,  9.90s/it]
                                                       

 80%|███████▉  | 1780/2235 [4:59:46<1:15:05,  9.90s/it]
 80%|███████▉  | 1781/2235 [4:59:56<1:14:55,  9.90s/it]
 80%|███████▉  | 1782/2235 [5:00:06<1:14:52,  9.92s/it]
 80%|███████▉  | 1783/2235 [5:00:16<1:14:50,  9.94s/it]
 80%|███████▉  | 1784/2235 [5:00:26<1:14:49,  9.95s/it]
 80%|███████▉  | 1785/2235 [5:00:36<1:15:00, 10.00s/it]
 80%|███████▉  | 1786/2235 [5:00:
+0: {'loss': 0.5335, 'grad_norm': 0.9776265772147773, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: 47<1:15:05, 10.04s/it]
 80%|███████▉  | 1787/2235 [5:00:57<1:15:03, 10.05s/it]
 80%|████████  | 1788/2235 [5:01:07<1:15:00, 10.07s/it]
 80%|████████  | 1789/2235 [5:01:17<1:14:53, 10.08s/it]
 80%|████████  | 1790/2235 [5:01:27<1:14:50, 10.09s/it]
                                                       

 80%|████████  | 1790/2235 [5:01:27<1:14:50, 10.09s/it]
 80%|████████  | 1791/2235 [5:01:37<1:14:54, 10.12s/it]
 80%|████████  | 1792/2235 [5:01:47<1:14:50, 10.14s/it]
 80%|████████  | 1793/2235 [5:01:58<1:14:49, 10.16s/it]
 80%|████████  | 1794/2235 [5:02:08<1:14:45, 10.17s/it]
 80%|████████  | 1795/2235 [5:02:18<1:14:42, 10.19s/it]
 80%|████████  | 1796/2235 [5:02:28<1:14:36, 10.20s/it]
 80%|████████  | 1797/2235 [5:02:38<1:14:02, 10.14s/it]
 80%|████████  | 1798/2235 [5:02:48<1:13:32, 10.10s/it]
 80%|█
+0: {'loss': 0.5178, 'grad_norm': 0.9836743217775504, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.48}
+0: ███████  | 1799/2235 [5:02:58<1:13:07, 10.06s/it]
 81%|████████  | 1800/2235 [5:03:08<1:13:26, 10.13s/it]
                                                       

 81%|████████  | 1800/2235 [5:03:08<1:13:26, 10.13s/it]
 81%|████████  | 1801/2235 [5:03:19<1:13:30, 10.16s/it]
 81%|████████  | 1802/2235 [5:03:29<1:13:41, 10.21s/it]
 81%|████████  | 1803/2235 [5:03:39<1:13:42, 10.24s/it]
 81%|████████  | 1804/2235 [5:03:50<1:13:55, 10.29s/it]
 81%|████████  | 1805/2235 [5:04:00<1:14:08, 10.35s/it]
 81%|████████  | 1806/2235 [5:04:11<1:14:21, 10.40s/it]
 81%|████████  | 1807/2235 [5:04:21<1:14:21, 10.42s/it]
 81%|████████  | 1808/2235 [5:04:32<1:14:16, 10.44s/it]
 81%|████████  | 1809/2235 [5:04:42<1:13:43, 10.38s/it]
 81%|████████  | 1810/2235 [5:04:52<1:12:53, 10.29s/it]
                                       
+0: {'loss': 0.5268, 'grad_norm': 0.866784589946435, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0: {'loss': 0.5145, 'grad_norm': 0.9512714842598161, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0:                 

 81%|████████  | 1810/2235 [5:04:52<1:12:53, 10.29s/it]
 81%|████████  | 1811/2235 [5:05:02<1:12:11, 10.22s/it]
 81%|████████  | 1812/2235 [5:05:12<1:11:33, 10.15s/it]
 81%|████████  | 1813/2235 [5:05:22<1:11:06, 10.11s/it]
 81%|████████  | 1814/2235 [5:05:32<1:10:25, 10.04s/it]
 81%|████████  | 1815/2235 [5:05:42<1:09:59, 10.00s/it]
 81%|████████▏ | 1816/2235 [5:05:52<1:09:39,  9.97s/it]
 81%|████████▏ | 1817/2235 [5:06:02<1:09:21,  9.96s/it]
 81%|████████▏ | 1818/2235 [5:06:12<1:09:02,  9.93s/it]
 81%|████████▏ | 1819/2235 [5:06:21<1:08:46,  9.92s/it]
 81%|████████▏ | 1820/2235 [5:06:31<1:08:30,  9.91s/it]
                                                       

 81%|████████▏ | 1820/2235 [5:06:31<1:08:30,  9.91s/it]
 81%|████████▏ | 1821/2235 [5:06:41<1:08:22,  9.91s/it]
+0: {'loss': 0.5219, 'grad_norm': 0.8844129139189374, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0: 
 82%|████████▏ | 1822/2235 [5:06:51<1:08:12,  9.91s/it]
 82%|████████▏ | 1823/2235 [5:07:01<1:08:11,  9.93s/it]
 82%|████████▏ | 1824/2235 [5:07:11<1:08:07,  9.94s/it]
 82%|████████▏ | 1825/2235 [5:07:21<1:08:04,  9.96s/it]
 82%|████████▏ | 1826/2235 [5:07:31<1:08:00,  9.98s/it]
 82%|████████▏ | 1827/2235 [5:07:41<1:07:55,  9.99s/it]
 82%|████████▏ | 1828/2235 [5:07:51<1:07:47,  9.99s/it]
 82%|████████▏ | 1829/2235 [5:08:01<1:07:44, 10.01s/it]
 82%|████████▏ | 1830/2235 [5:08:11<1:07:37, 10.02s/it]
                                                       

 82%|████████▏ | 1830/2235 [5:08:11<1:07:37, 10.02s/it]
 82%|████████▏ | 1831/2235 [5:08:21<1:07:28, 10.02s/it]
 82%|████████▏ | 1832/2235 [5:08:31<1:07:23, 10.03s/it]
 82%|████████▏ | 1833/2235 [5:08:41<1:07:18, 10.05s/it]
 82%
+0: {'loss': 0.5153, 'grad_norm': 0.8715667768112283, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.49}
+0: |████████▏ | 1834/2235 [5:08:51<1:07:08, 10.05s/it]
 82%|████████▏ | 1835/2235 [5:09:02<1:07:20, 10.10s/it]
 82%|████████▏ | 1836/2235 [5:09:12<1:07:05, 10.09s/it]
 82%|████████▏ | 1837/2235 [5:09:22<1:06:49, 10.07s/it]
 82%|████████▏ | 1838/2235 [5:09:32<1:06:31, 10.05s/it]
 82%|████████▏ | 1839/2235 [5:09:42<1:06:19, 10.05s/it]
 82%|████████▏ | 1840/2235 [5:09:52<1:06:10, 10.05s/it]
                                                       

 82%|████████▏ | 1840/2235 [5:09:52<1:06:10, 10.05s/it]
 82%|████████▏ | 1841/2235 [5:10:02<1:06:07, 10.07s/it]
 82%|████████▏ | 1842/2235 [5:10:12<1:05:58, 10.07s/it]
 82%|████████▏ | 1843/2235 [5:10:22<1:05:49, 10.08s/it]
 83%|████████▎ | 1844/2235 [5:10:32<1:05:34, 10.06s/it]
 83%|████████▎ | 1845/2235 [5:10:42<1:05:25, 10.06s/it]
 83%|█�
+0: {'loss': 0.5159, 'grad_norm': 3.067333300353315, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: ��██████▎ | 1846/2235 [5:10:52<1:05:17, 10.07s/it]
 83%|████████▎ | 1847/2235 [5:11:02<1:05:00, 10.05s/it]
 83%|████████▎ | 1848/2235 [5:11:12<1:04:50, 10.05s/it]
 83%|████████▎ | 1849/2235 [5:11:22<1:04:44, 10.06s/it]
 83%|████████▎ | 1850/2235 [5:11:33<1:04:38, 10.07s/it]
                                                       

 83%|████████▎ | 1850/2235 [5:11:33<1:04:38, 10.07s/it]
 83%|████████▎ | 1851/2235 [5:11:43<1:04:31, 10.08s/it]
 83%|████████▎ | 1852/2235 [5:11:53<1:04:18, 10.08s/it]
 83%|████████▎ | 1853/2235 [5:12:03<1:04:11, 10.08s/it]
 83%|████████▎ | 1854/2235 [5:12:13<1:04:01, 10.08s/it]
 83%|████████▎ | 1855/2235 [5:12:23<1:03:43, 10.06s/it]
 83%|████████▎ | 1856/2235 [5:12:33<1:03:41, 10.08s/it]
 83%|████████▎ | 1857/2235 [5:12:44<1:04:14, 10.20s/it]
 83%|███
+0: {'loss': 0.5143, 'grad_norm': 1.7432421601327481, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: █████▎ | 1858/2235 [5:12:54<1:04:04, 10.20s/it]
 83%|████████▎ | 1859/2235 [5:13:04<1:03:49, 10.18s/it]
 83%|████████▎ | 1860/2235 [5:13:14<1:03:27, 10.15s/it]
                                                       

 83%|███��████▎ | 1860/2235 [5:13:14<1:03:27, 10.15s/it]
 83%|████████▎ | 1861/2235 [5:13:24<1:03:30, 10.19s/it]
 83%|████████▎ | 1862/2235 [5:13:35<1:03:36, 10.23s/it]
 83%|████████▎ | 1863/2235 [5:13:45<1:03:32, 10.25s/it]
 83%|████████▎ | 1864/2235 [5:13:55<1:03:23, 10.25s/it]
 83%|████████▎ | 1865/2235 [5:14:05<1:03:14, 10.25s/it]
 83%|████████▎ | 1866/2235 [5:14:16<1:02:56, 10.24s/it]
 84%|████████▎ | 1867/2235 [5:14:26<1:02:32, 10.20s/it]
 84%|████████▎ | 1868/2235 [5:14:36<1:02:19, 10.19s/it]
 84%|████████▎ | 1869/2235 [5:14:46<1:02:59, 10.33s/it]
 84%|████�
+0: {'loss': 0.5125, 'grad_norm': 1.0713278584559178, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: {'loss': 0.515, 'grad_norm': 0.9573515175378171, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.5}
+0: �███▎ | 1870/2235 [5:14:57<1:03:04, 10.37s/it]
                                                       

 84%|████████▎ | 1870/2235 [5:14:57<1:03:04, 10.37s/it]
 84%|████████▎ | 1871/2235 [5:15:07<1:02:34, 10.32s/it]
 84%|████████▍ | 1872/2235 [5:15:17<1:01:52, 10.23s/it]
 84%|████████▍ | 1873/2235 [5:15:27<1:01:26, 10.18s/it]
 84%|████████▍ | 1874/2235 [5:15:37<1:01:05, 10.15s/it]
 84%|████████▍ | 1875/2235 [5:15:47<1:00:51, 10.14s/it]
 84%|████████▍ | 1876/2235 [5:15:58<1:00:41, 10.14s/it]
 84%|████████▍ | 1877/2235 [5:16:08<1:00:37, 10.16s/it]
 84%|████████▍ | 1878/2235 [5:16:18<1:00:24, 10.15s/it]
 84%|████████▍ | 1879/2235 [5:16:28<1:00:31, 10.20s/it]
 84%|████████▍ | 1880/2235 [5:16:39<1:00:30, 10.23s/it]
                                                       

 84%|████████▍ | 1880/2
+0: {'loss': 0.5231, 'grad_norm': 0.8392778750268665, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: 235 [5:16:39<1:00:30, 10.23s/it]
 84%|████████▍ | 1881/2235 [5:16:49<1:00:56, 10.33s/it]
 84%|████████▍ | 1882/2235 [5:17:00<1:01:24, 10.44s/it]
 84%|████████▍ | 1883/2235 [5:17:10<1:01:41, 10.52s/it]
 84%|████████▍ | 1884/2235 [5:17:21<1:01:37, 10.53s/it]
 84%|████████▍ | 1885/2235 [5:17:32<1:01:37, 10.56s/it]
 84%|████████▍ | 1886/2235 [5:17:42<1:01:45, 10.62s/it]
 84%|████████▍ | 1887/2235 [5:17:53<1:01:45, 10.65s/it]
 84%|████████▍ | 1888/2235 [5:18:04<1:01:41, 10.67s/it]
 85%|████████▍ | 1889/2235 [5:18:15<1:01:58, 10.75s/it]
 85%|████████▍ | 1890/2235 [5:18:26<1:01:46, 10.74s/it]
                                                       

 85%|████████▍ | 1890/2235 [5:18:26<1:01:46, 10.74s/it]
 85%|████████▍ | 1891/2235 [5:18:36<1:01:35, 10.74s/it]
 85%|████████▍ | 1892/2235 [
+0: {'loss': 0.5136, 'grad_norm': 0.9744966528160444, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: 5:18:47<1:01:22, 10.74s/it]
 85%|████████▍ | 1893/2235 [5:18:58<1:01:07, 10.72s/it]
 85%|████████▍ | 1894/2235 [5:19:08<1:00:53, 10.71s/it]
 85%|████████▍ | 1895/2235 [5:19:19<1:00:41, 10.71s/it]
 85%|████████▍ | 1896/2235 [5:19:30<1:00:05, 10.64s/it]
 85%|████████▍ | 1897/2235 [5:19:40<59:20, 10.53s/it]  
 85%|████████▍ | 1898/2235 [5:19:50<58:38, 10.44s/it]
 85%|████████▍ | 1899/2235 [5:20:00<57:58, 10.35s/it]
 85%|████████▌ | 1900/2235 [5:20:10<57:34, 10.31s/it]
                                                     

 85%|████████▌ | 1900/2235 [5:20:10<57:34, 10.31s/it]
 85%|████████▌ | 1901/2235 [5:20:21<57:29, 10.33s/it]
 85%|████████▌ | 1902/2235 [5:20:31<57:31, 10.37s/it]
 85%|████████▌ | 1903/2235 [5:20:42<57:23, 10.37s/it]
 85%|████████▌ | 1904/2235 [5:20:52<57:14, 10.38s
+0: {'loss': 0.5184, 'grad_norm': 1.000797856695467, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.51}
+0: /it]
 85%|████████▌ | 1905/2235 [5:21:02<57:00, 10.37s/it]
 85%|████████▌ | 1906/2235 [5:21:13<56:44, 10.35s/it]
 85%|████████▌ | 1907/2235 [5:21:23<56:27, 10.33s/it]
 85%|████████▌ | 1908/2235 [5:21:33<56:04, 10.29s/it]
 85%|████████▌ | 1909/2235 [5:21:43<55:53, 10.29s/it]
 85%|████████▌ | 1910/2235 [5:21:54<55:43, 10.29s/it]
                                                     

 85%|████████▌ | 1910/2235 [5:21:54<55:43, 10.29s/it]
 86%|████████▌ | 1911/2235 [5:22:04<55:30, 10.28s/it]
 86%|████████▌ | 1912/2235 [5:22:14<55:23, 10.29s/it]
 86%|████████▌ | 1913/2235 [5:22:24<55:03, 10.26s/it]
 86%|████████▌ | 1914/2235 [5:22:35<54:56, 10.27s/it]
 86%|████████▌ | 1915/2235 [5:22:45<54:37, 10.24s/it]
 86%|████████▌ | 1916/2235 [5:22:55<54:19, 10.22s/it]
 86%|███████�
+0: {'loss': 0.5319, 'grad_norm': 0.9223380737280957, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0: �▌ | 1917/2235 [5:23:05<54:06, 10.21s/it]
 86%|████████▌ | 1918/2235 [5:23:15<53:53, 10.20s/it]
 86%|████████▌ | 1919/2235 [5:23:26<53:45, 10.21s/it]
 86%|████████▌ | 1920/2235 [5:23:36<53:37, 10.21s/it]
                                                     

 86%|████████▌ | 1920/2235 [5:23:36<53:37, 10.21s/it]
 86%|████████▌ | 1921/2235 [5:23:46<53:17, 10.18s/it]
 86%|████████▌ | 1922/2235 [5:23:56<53:05, 10.18s/it]
 86%|████████▌ | 1923/2235 [5:24:06<52:57, 10.18s/it]
 86%|████████▌ | 1924/2235 [5:24:16<52:35, 10.15s/it]
 86%|████████▌ | 1925/2235 [5:24:27<52:20, 10.13s/it]
 86%|████████▌ | 1926/2235 [5:24:37<52:09, 10.13s/it]
 86%|████████▌ | 1927/2235 [5:24:47<52:16, 10.18s/it]
 86%|████████▋ | 1928/2235 [5:24:57<52:10, 10.20s/it]
 86%|████████▋ | 1929/2235 [5:25:07<51:49, 
+0: {'loss': 0.5126, 'grad_norm': 0.9760502625077748, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0: {'loss': 0.5176, 'grad_norm': 0.9002825302804719, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0: 10.16s/it]
 86%|████████▋ | 1930/2235 [5:25:17<51:36, 10.15s/it]
                                                     

 86%|████████▋ | 1930/2235 [5:25:17<51:36, 10.15s/it]
 86%|████████▋ | 1931/2235 [5:25:28<51:33, 10.18s/it]
 86%|████████▋ | 1932/2235 [5:25:38<51:26, 10.19s/it]
 86%|████████▋ | 1933/2235 [5:25:48<51:05, 10.15s/it]
 87%|████████▋ | 1934/2235 [5:25:58<51:09, 10.20s/it]
 87%|████████▋ | 1935/2235 [5:26:08<50:46, 10.16s/it]
 87%|████████▋ | 1936/2235 [5:26:18<50:34, 10.15s/it]
 87%|████████▋ | 1937/2235 [5:26:29<50:22, 10.14s/it]
 87%|████████▋ | 1938/2235 [5:26:39<50:11, 10.14s/it]
 87%|████████▋ | 1939/2235 [5:26:49<49:58, 10.13s/it]
 87%|████████▋ | 1940/2235 [5:26:59<49:45, 10.12s/it]
                                                     

 87%|████████▋ | 1940
+0: {'loss': 0.5195, 'grad_norm': 1.0769404574035393, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.52}
+0: /2235 [5:26:59<49:45, 10.12s/it]
 87%|████████▋ | 1941/2235 [5:27:09<49:44, 10.15s/it]
 87%|████████▋ | 1942/2235 [5:27:19<49:36, 10.16s/it]
 87%|████████▋ | 1943/2235 [5:27:29<49:30, 10.17s/it]
 87%|████████▋ | 1944/2235 [5:27:40<49:19, 10.17s/it]
 87%|████████▋ | 1945/2235 [5:27:50<49:08, 10.17s/it]
 87%|████████▋ | 1946/2235 [5:28:00<49:03, 10.19s/it]
 87%|████████▋ | 1947/2235 [5:28:10<48:56, 10.20s/it]
 87%|████████▋ | 1948/2235 [5:28:21<48:57, 10.24s/it]
 87%|████████▋ | 1949/2235 [5:28:31<48:50, 10.25s/it]
 87%|████████▋ | 1950/2235 [5:28:41<48:38, 10.24s/it]
                                                     

 87%|████████▋ | 1950/2235 [5:28:41<48:38, 10.24s/it]
 87%|████████▋ | 1951/2235 [5:28:51<48:25, 10.23s/it]
 87%|████████▋ | 1952/2235 [5:29:02<48:14, 10.23s/it]
+0: {'loss': 0.5167, 'grad_norm': 2.0621119674591313, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0:  87%|████████▋ | 1953/2235 [5:29:12<48:02, 10.22s/it]
 87%|████████▋ | 1954/2235 [5:29:22<47:51, 10.22s/it]
 87%|████████▋ | 1955/2235 [5:29:32<47:39, 10.21s/it]
 88%|████████▊ | 1956/2235 [5:29:42<47:30, 10.22s/it]
 88%|████████▊ | 1957/2235 [5:29:53<47:15, 10.20s/it]
 88%|████████▊ | 1958/2235 [5:30:03<46:58, 10.18s/it]
 88%|████████▊ | 1959/2235 [5:30:13<46:46, 10.17s/it]
 88%|████████▊ | 1960/2235 [5:30:23<46:30, 10.15s/it]
                                                     

 88%|████████▊ | 1960/2235 [5:30:23<46:30, 10.15s/it]
 88%|████████▊ | 1961/2235 [5:30:33<46:22, 10.16s/it]
 88%|████████▊ | 1962/2235 [5:30:43<46:06, 10.13s/it]
 88%|████████▊ | 1963/2235 [5:30:53<45:52, 10.12s/it]
 88%|████████▊ | 1964/2235 [5:31:03<45:40, 10.11s/it]
 88%|████████▊ 
+0: {'loss': 0.5076, 'grad_norm': 1.3810502119765489, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0: | 1965/2235 [5:31:13<45:32, 10.12s/it]
 88%|████████▊ | 1966/2235 [5:31:23<45:14, 10.09s/it]
 88%|████████▊ | 1967/2235 [5:31:34<44:58, 10.07s/it]
 88%|████████▊ | 1968/2235 [5:31:43<44:41, 10.04s/it]
 88%|████████▊ | 1969/2235 [5:31:54<44:28, 10.03s/it]
 88%|████████▊ | 1970/2235 [5:32:04<44:24, 10.06s/it]
                                                     

 88%|████████▊ | 1970/2235 [5:32:04<44:24, 10.06s/it]
 88%|████████▊ | 1971/2235 [5:32:14<44:15, 10.06s/it]
 88%|████████▊ | 1972/2235 [5:32:24<44:05, 10.06s/it]
 88%|████████▊ | 1973/2235 [5:32:34<43:56, 10.06s/it]
 88%|████████▊ | 1974/2235 [5:32:44<43:51, 10.08s/it]
 88%|████████▊ | 1975/2235 [5:32:54<43:41, 10.08s/it]
 88%|████████▊ | 1976/2235 [5:33:04<43:28, 10.07s/it]
 88%|████████▊ | 1977/2235 [5:33:14<43:11, 10.05
+0: {'loss': 0.5095, 'grad_norm': 0.9735380069138082, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0: s/it]
 89%|████████▊ | 1978/2235 [5:33:24<42:55, 10.02s/it]
 89%|████████▊ | 1979/2235 [5:33:34<42:46, 10.03s/it]
 89%|████████▊ | 1980/2235 [5:33:44<42:35, 10.02s/it]
                                                     

 89%|████████▊ | 1980/2235 [5:33:44<42:35, 10.02s/it]
 89%|████████▊ | 1981/2235 [5:33:54<42:32, 10.05s/it]
 89%|████████▊ | 1982/2235 [5:34:04<42:36, 10.11s/it]
 89%|████████▊ | 1983/2235 [5:34:15<42:35, 10.14s/it]
 89%|████████▉ | 1984/2235 [5:34:25<42:24, 10.14s/it]
 89%|████████▉ | 1985/2235 [5:34:35<42:11, 10.13s/it]
 89%|████████▉ | 1986/2235 [5:34:45<41:57, 10.11s/it]
 89%|████████▉ | 1987/2235 [5:34:55<41:52, 10.13s/it]
 89%|████████▉ | 1988/2235 [5:35:05<41:39, 10.12s/it]
 89%|████████▉ | 1989/2235 [5:35:15<41:35, 10.15s/it]
 89%|███████�
+0: {'loss': 0.499, 'grad_norm': 1.4023710601949035, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.53}
+0: {'loss': 0.5036, 'grad_norm': 0.9530606115922439, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: ��▉ | 1990/2235 [5:35:25<41:18, 10.12s/it]
                                                     

 89%|████████▉ | 1990/2235 [5:35:25<41:18, 10.12s/it]
 89%|████████▉ | 1991/2235 [5:35:36<41:02, 10.09s/it]
 89%|████████▉ | 1992/2235 [5:35:46<40:50, 10.09s/it]
 89%|████████▉ | 1993/2235 [5:35:56<40:40, 10.09s/it]
 89%|████████▉ | 1994/2235 [5:36:06<40:38, 10.12s/it]
 89%|████████▉ | 1995/2235 [5:36:16<40:31, 10.13s/it]
 89%|████████▉ | 1996/2235 [5:36:26<40:16, 10.11s/it]
 89%|████████▉ | 1997/2235 [5:36:36<40:00, 10.08s/it]
 89%|████████▉ | 1998/2235 [5:36:46<39:49, 10.08s/it]
 89%|████████▉ | 1999/2235 [5:36:56<39:29, 10.04s/it]
 89%|████████▉ | 2000/2235 [5:37:06<39:05,  9.98s/it]
                                                     

 89%|████████▉ | 2000/2235 [5:37:06<39:05,  9.98s/it]
 90%|
+0: {'loss': 0.5152, 'grad_norm': 1.5109115501984043, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: ████████▉ | 2001/2235 [5:37:16<38:47,  9.95s/it]
 90%|████████▉ | 2002/2235 [5:37:26<38:30,  9.92s/it]
 90%|████████▉ | 2003/2235 [5:37:36<38:18,  9.91s/it]
 90%|████████▉ | 2004/2235 [5:37:46<38:16,  9.94s/it]
 90%|████████▉ | 2005/2235 [5:37:56<38:23, 10.02s/it]
 90%|████████▉ | 2006/2235 [5:38:06<38:25, 10.07s/it]
 90%|████████▉ | 2007/2235 [5:38:16<38:32, 10.14s/it]
 90%|████████▉ | 2008/2235 [5:38:27<38:37, 10.21s/it]
 90%|████████▉ | 2009/2235 [5:38:37<38:36, 10.25s/it]
 90%|████████▉ | 2010/2235 [5:38:47<38:33, 10.28s/it]
                                                     

 90%|████████▉ | 2010/2235 [5:38:47<38:33, 10.28s/it]
 90%|████████▉ | 2011/2235 [5:38:58<38:22, 10.28s/it]
 90%|█████████ | 2012/2235 [5:39:08<38:20, 10.32s/it]
 90%|█████████ | 201
+0: {'loss': 0.5168, 'grad_norm': 0.9406258219580814, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: 3/2235 [5:39:18<38:18, 10.35s/it]
 90%|█████████ | 2014/2235 [5:39:29<37:57, 10.31s/it]
 90%|█████████ | 2015/2235 [5:39:39<37:39, 10.27s/it]
 90%|█████████ | 2016/2235 [5:39:49<37:30, 10.28s/it]
 90%|█████████ | 2017/2235 [5:39:59<37:07, 10.22s/it]
 90%|█████████ | 2018/2235 [5:40:09<36:42, 10.15s/it]
 90%|█████████ | 2019/2235 [5:40:19<36:28, 10.13s/it]
 90%|█████████ | 2020/2235 [5:40:29<36:13, 10.11s/it]
                                                     

 90%|█████████ | 2020/2235 [5:40:29<36:13, 10.11s/it]
 90%|█████████ | 2021/2235 [5:40:39<36:01, 10.10s/it]
 90%|█████████ | 2022/2235 [5:40:49<35:41, 10.05s/it]
 91%|█████████ | 2023/2235 [5:40:59<35:32, 10.06s/it]
 91%|█████████ | 2024/2235 [5:41:09<35:19, 10.05s/it]
 91%|█████████ | 2025/2235 [5:41:19<35:05, 10.03s/it]
+0: {'loss': 0.5015, 'grad_norm': 0.894299133813123, 'learning_rate': 5e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.54}
+0: 
 91%|█████████ | 2026/2235 [5:41:29<34:53, 10.02s/it]
 91%|█████████ | 2027/2235 [5:41:40<34:47, 10.04s/it]
 91%|█████████ | 2028/2235 [5:41:49<34:31, 10.01s/it]
 91%|█████████ | 2029/2235 [5:41:59<34:20, 10.00s/it]
 91%|█████████ | 2030/2235 [5:42:10<34:25, 10.07s/it]
                                                     

 91%|█████████ | 2030/2235 [5:42:10<34:25, 10.07s/it]
 91%|█████████ | 2031/2235 [5:42:20<34:11, 10.06s/it]
 91%|█████████ | 2032/2235 [5:42:30<33:57, 10.03s/it]
 91%|█████████ | 2033/2235 [5:42:40<33:45, 10.02s/it]
 91%|█████████ | 2034/2235 [5:42:50<33:53, 10.11s/it]
 91%|█████████ | 2035/2235 [5:43:00<33:39, 10.10s/it]
 91%|█████████ | 2036/2235 [5:43:10<33:28, 10.09s/it]
 91%|█████████ | 2037/2235 [5:43:20<33:11, 10.06s/it]
 91%|█████████
+0: {'loss': 0.517, 'grad_norm': 1.2832814310992608, 'learning_rate': 4.995560138963612e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0:  | 2038/2235 [5:43:30<32:52, 10.01s/it]
 91%|█████████ | 2039/2235 [5:43:40<32:34,  9.97s/it]
 91%|█████████▏| 2040/2235 [5:43:50<32:25,  9.98s/it]
                                                     

 91%|█████████▏| 2040/2235 [5:43:50<32:25,  9.98s/it]
 91%|█████████▏| 2041/2235 [5:44:00<32:24, 10.02s/it]
 91%|█████████▏| 2042/2235 [5:44:10<32:14, 10.02s/it]
 91%|█████████▏| 2043/2235 [5:44:20<32:10, 10.06s/it]
 91%|█████████▏| 2044/2235 [5:44:30<32:10, 10.11s/it]
 91%|█████████▏| 2045/2235 [5:44:41<32:10, 10.16s/it]
 92%|█████████▏| 2046/2235 [5:44:51<32:07, 10.20s/it]
 92%|█████████▏| 2047/2235 [5:45:01<31:59, 10.21s/it]
 92%|█████████▏| 2048/2235 [5:45:11<31:49, 10.21s/it]
 92%|█████████▏| 2049/2235 [5:45:22<31:40, 10.22s/it]
 92%|█████████▏| 2050/2
+0: {'loss': 0.513, 'grad_norm': 1.2186184359710643, 'learning_rate': 4.945812714362182e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0: {'loss': 0.5102, 'grad_norm': 0.8536684638145462, 'learning_rate': 4.841997093248566e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.55}
+0: 235 [5:45:32<31:25, 10.19s/it]
                                                     

 92%|█████████▏| 2050/2235 [5:45:32<31:25, 10.19s/it]
 92%|█████████▏| 2051/2235 [5:45:42<31:14, 10.19s/it]
 92%|█████████▏| 2052/2235 [5:45:52<31:00, 10.17s/it]
 92%|█████████▏| 2053/2235 [5:46:02<30:44, 10.13s/it]
 92%|█████████▏| 2054/2235 [5:46:12<30:24, 10.08s/it]
 92%|█████████▏| 2055/2235 [5:46:22<30:23, 10.13s/it]
 92%|█████████▏| 2056/2235 [5:46:32<30:00, 10.06s/it]
 92%|█████████▏| 2057/2235 [5:46:42<29:40, 10.00s/it]
 92%|█████████▏| 2058/2235 [5:46:52<29:22,  9.96s/it]
 92%|█████████▏| 2059/2235 [5:47:02<29:19, 10.00s/it]
 92%|█████████▏| 2060/2235 [5:47:12<29:14, 10.02s/it]
                                                     

 92%|█████████▏| 2060/2235 [5:47:12<29:14, 10.02s
+0: {'loss': 0.5183, 'grad_norm': 0.9755560091733374, 'learning_rate': 4.686669560758874e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0: /it]
 92%|█████████▏| 2061/2235 [5:47:22<29:08, 10.05s/it]
 92%|█████████▏| 2062/2235 [5:47:32<29:04, 10.08s/it]
 92%|█████████▏| 2063/2235 [5:47:43<28:56, 10.10s/it]
 92%|█████████▏| 2064/2235 [5:47:53<28:48, 10.11s/it]
 92%|█████████▏| 2065/2235 [5:48:03<28:39, 10.12s/it]
 92%|█████████▏| 2066/2235 [5:48:13<28:28, 10.11s/it]
 92%|█████████▏| 2067/2235 [5:48:23<28:17, 10.10s/it]
 93%|█████████▎| 2068/2235 [5:48:33<28:02, 10.07s/it]
 93%|█████████▎| 2069/2235 [5:48:43<27:46, 10.04s/it]
 93%|█████████▎| 2070/2235 [5:48:53<27:34, 10.03s/it]
                                                     

 93%|█████████▎| 2070/2235 [5:48:53<27:34, 10.03s/it]
 93%|█████████▎| 2071/2235 [5:49:03<27:28, 10.05s/it]
 93%|█████████▎| 2072/2235 [5:49:13<27:20, 10.06s/it]
 9
+0: {'loss': 0.5116, 'grad_norm': 0.871225156037512, 'learning_rate': 4.483654796245526e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0: 3%|█████████▎| 2073/2235 [5:49:23<27:12, 10.08s/it]
 93%|█████████▎| 2074/2235 [5:49:34<27:09, 10.12s/it]
 93%|█████████▎| 2075/2235 [5:49:44<27:13, 10.21s/it]
 93%|█████████▎| 2076/2235 [5:49:54<26:57, 10.17s/it]
 93%|█████████▎| 2077/2235 [5:50:04<26:42, 10.14s/it]
 93%|█████████▎| 2078/2235 [5:50:14<26:29, 10.12s/it]
 93%|█████████▎| 2079/2235 [5:50:24<26:13, 10.09s/it]
 93%|█████████▎| 2080/2235 [5:50:34<25:59, 10.06s/it]
                                                     

 93%|█████████▎| 2080/2235 [5:50:34<25:59, 10.06s/it]
 93%|█████████▎| 2081/2235 [5:50:44<25:48, 10.05s/it]
 93%|█████████▎| 2082/2235 [5:50:54<25:38, 10.06s/it]
 93%|█████████▎| 2083/2235 [5:51:04<25:31, 10.08s/it]
 93%|█████████▎| 2084/2235 [5:51:15<25:24, 10.09s/it]
 93%|█�
+0: {'loss': 0.504, 'grad_norm': 0.8732093570045909, 'learning_rate': 4.237951696978217e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0: ��███████▎| 2085/2235 [5:51:25<25:13, 10.09s/it]
 93%|█████████▎| 2086/2235 [5:51:35<25:02, 10.08s/it]
 93%|█████████▎| 2087/2235 [5:51:45<24:51, 10.08s/it]
 93%|█████████▎| 2088/2235 [5:51:56<25:36, 10.45s/it]
 93%|█████████▎| 2089/2235 [5:52:06<25:10, 10.35s/it]
 94%|█████████▎| 2090/2235 [5:52:16<24:42, 10.22s/it]
                                                     

 94%|█████████▎| 2090/2235 [5:52:16<24:42, 10.22s/it]
 94%|█████████▎| 2091/2235 [5:52:26<24:23, 10.17s/it]
 94%|█████████▎| 2092/2235 [5:52:36<24:08, 10.13s/it]
 94%|█████████▎| 2093/2235 [5:52:46<23:50, 10.07s/it]
 94%|█████████▎| 2094/2235 [5:52:56<23:45, 10.11s/it]
 94%|█████████▎| 2095/2235 [5:53:07<23:41, 10.15s/it]
 94%|█████████▍| 2096/2235 [5:53:17<23:35, 10.18s/it]
 94%|███�
+0: {'loss': 0.5226, 'grad_norm': 0.9430245374112134, 'learning_rate': 3.9556102887027425e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.56}
+0: �█████▍| 2097/2235 [5:53:28<24:07, 10.49s/it]
 94%|█████████▍| 2098/2235 [5:53:38<23:48, 10.43s/it]
 94%|█████████▍| 2099/2235 [5:53:48<23:28, 10.36s/it]
 94%|█████████▍| 2100/2235 [5:54:00<24:01, 10.68s/it]
                                                     

 94%|█████████▍| 2100/2235 [5:54:00<24:01, 10.68s/it]
 94%|█████████▍| 2101/2235 [5:54:10<23:28, 10.51s/it]
 94%|█████████▍| 2102/2235 [5:54:21<23:24, 10.56s/it]
 94%|█████████▍| 2103/2235 [5:54:31<23:04, 10.49s/it]
 94%|█████████▍| 2104/2235 [5:54:42<23:22, 10.71s/it]
 94%|█████████▍| 2105/2235 [5:54:53<23:01, 10.62s/it]
 94%|█████████▍| 2106/2235 [5:55:03<22:40, 10.55s/it]
 94%|█████████▍| 2107/2235 [5:55:13<22:26, 10.52s/it]
 94%|█████████▍| 2108/2235 [5:55:24<22:09, 10.47s/it]
 94%|██████
+0: {'loss': 0.501, 'grad_norm': 0.8203794901543918, 'learning_rate': 3.6435827539282564e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0: ███▍| 2109/2235 [5:55:34<21:46, 10.37s/it]
 94%|█████████▍| 2110/2235 [5:55:44<21:26, 10.30s/it]
                                                     

 94%|█████████▍| 2110/2235 [5:55:44<21:26, 10.30s/it]
 94%|█████████▍| 2111/2235 [5:55:54<21:11, 10.25s/it]
 94%|█████████▍| 2112/2235 [5:56:05<21:08, 10.32s/it]
 95%|█████████▍| 2113/2235 [5:56:15<20:49, 10.24s/it]
 95%|█████████▍| 2114/2235 [5:56:25<20:29, 10.16s/it]
 95%|█████████▍| 2115/2235 [5:56:35<20:14, 10.12s/it]
 95%|█████████▍| 2116/2235 [5:56:45<19:58, 10.07s/it]
 95%|█████████▍| 2117/2235 [5:56:55<19:45, 10.05s/it]
 95%|█████████▍| 2118/2235 [5:57:05<19:35, 10.05s/it]
 95%|█████████▍| 2119/2235 [5:57:15<19:25, 10.04s/it]
 95%|█████████▍| 2120/2235 [5:57:25<19:14, 10.04s/it]
                              
+0: {'loss': 0.5207, 'grad_norm': 0.8886635875634513, 'learning_rate': 3.309552246120924e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0: {'loss': 0.5106, 'grad_norm': 0.9473384129511132, 'learning_rate': 2.9617437049666582e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0:                        

 95%|█████████▍| 2120/2235 [5:57:25<19:14, 10.04s/it]
 95%|█████████▍| 2121/2235 [5:57:35<19:08, 10.08s/it]
 95%|█████████▍| 2122/2235 [5:57:45<18:53, 10.04s/it]
 95%|█████████▍| 2123/2235 [5:57:55<18:41, 10.01s/it]
 95%|█████████▌| 2124/2235 [5:58:05<18:28,  9.99s/it]
 95%|█████████▌| 2125/2235 [5:58:15<18:19,  9.99s/it]
 95%|█████████▌| 2126/2235 [5:58:25<18:08,  9.98s/it]
 95%|█████████▌| 2127/2235 [5:58:35<17:58,  9.99s/it]
 95%|█████████▌| 2128/2235 [5:58:45<17:50, 10.01s/it]
 95%|█████████▌| 2129/2235 [5:58:55<17:39,  9.99s/it]
 95%|█████████▌| 2130/2235 [5:59:05<17:27,  9.97s/it]
                                                     

 95%|█████████▌| 2130/2235 [5:59:05<17:27,  9.97s/it]
 95%|█████████▌| 2131/2235 [5:59:15<
+0: {'loss': 0.4948, 'grad_norm': 1.2948495142559764, 'learning_rate': 2.6087213310590455e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.57}
+0: 17:25, 10.05s/it]
 95%|█████████▌| 2132/2235 [5:59:25<17:21, 10.11s/it]
 95%|█████████▌| 2133/2235 [5:59:35<17:13, 10.13s/it]
 95%|█████████▌| 2134/2235 [5:59:46<17:03, 10.14s/it]
 96%|█████████▌| 2135/2235 [5:59:56<16:48, 10.09s/it]
 96%|█████████▌| 2136/2235 [6:00:05<16:35, 10.05s/it]
 96%|█████████▌| 2137/2235 [6:00:15<16:21, 10.02s/it]
 96%|█████████▌| 2138/2235 [6:00:25<16:09, 10.00s/it]
 96%|█████████▌| 2139/2235 [6:00:35<15:58,  9.99s/it]
 96%|█████████▌| 2140/2235 [6:00:45<15:49,  9.99s/it]
                                                     

 96%|█████████▌| 2140/2235 [6:00:45<15:49,  9.99s/it]
 96%|█████████▌| 2141/2235 [6:00:55<15:40, 10.00s/it]
 96%|█████████▌| 2142/2235 [6:01:06<15:35, 10.06s/it]
 96%|█████████▌| 2143/2235 [6:01:16<15:30, 
+0: {'loss': 0.518, 'grad_norm': 0.8138566364964517, 'learning_rate': 2.2591777068577802e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.58}
+0: 10.11s/it]
 96%|█████████▌| 2144/2235 [6:01:26<15:22, 10.13s/it]
 96%|█████████▌| 2145/2235 [6:01:36<15:15, 10.17s/it]
 96%|█████████▌| 2146/2235 [6:01:47<15:15, 10.29s/it]
 96%|█████████▌| 2147/2235 [6:01:57<15:02, 10.25s/it]
 96%|█████████▌| 2148/2235 [6:02:07<14:50, 10.23s/it]
 96%|█████████▌| 2149/2235 [6:02:18<14:45, 10.30s/it]
 96%|█████████▌| 2150/2235 [6:02:28<14:31, 10.25s/it]
                                                     

 96%|█████████▌| 2150/2235 [6:02:28<14:31, 10.25s/it]
 96%|█████████▌| 2151/2235 [6:02:38<14:22, 10.27s/it]
 96%|█████████▋| 2152/2235 [6:02:48<14:13, 10.28s/it]
 96%|█████████▋| 2153/2235 [6:02:59<14:02, 10.28s/it]
 96%|█████████▋| 2154/2235 [6:03:09<14:00, 10.38s/it]
 96%|█████████▋| 2155/2235 [6:03:20<13:51, 10.39s/
+0: {'loss': 0.5054, 'grad_norm': 0.8824150145384203, 'learning_rate': 1.9217197564594746e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.58}
+0: it]
 96%|█████████▋| 2156/2235 [6:03:30<13:41, 10.39s/it]
 97%|█████████▋| 2157/2235 [6:03:41<13:31, 10.41s/it]
 97%|█████████▋| 2158/2235 [6:03:51<13:22, 10.42s/it]
 97%|█████████▋| 2159/2235 [6:04:01<13:12, 10.43s/it]
 97%|█████████▋| 2160/2235 [6:04:12<13:03, 10.44s/it]
                                                     

 97%|█████████▋| 2160/2235 [6:04:12<13:03, 10.44s/it]
 97%|█████████▋| 2161/2235 [6:04:22<12:51, 10.43s/it]
 97%|█████████▋| 2162/2235 [6:04:33<12:40, 10.42s/it]
 97%|█████████▋| 2163/2235 [6:04:43<12:28, 10.39s/it]
 97%|█████████▋| 2164/2235 [6:04:53<12:16, 10.37s/it]
 97%|█████████▋| 2165/2235 [6:05:04<12:04, 10.35s/it]
 97%|█████████▋| 2166/2235 [6:05:14<11:54, 10.35s/it]
 97%|█████████▋| 2167/2235 [6:05:24<11:44, 10.37s/it]
 97
+0: {'loss': 0.5085, 'grad_norm': 5.248305602369885, 'learning_rate': 1.6046568145616647e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.58}
+0: %|█████████▋| 2168/2235 [6:05:35<11:34, 10.37s/it]
 97%|█████████▋| 2169/2235 [6:05:45<11:23, 10.36s/it]
 97%|█████████▋| 2170/2235 [6:05:55<11:13, 10.36s/it]
                                                     

 97%|█████████▋| 2170/2235 [6:05:55<11:13, 10.36s/it]
 97%|█████████▋| 2171/2235 [6:06:06<11:03, 10.36s/it]
 97%|█████████▋| 2172/2235 [6:06:16<10:52, 10.36s/it]
 97%|█████████▋| 2173/2235 [6:06:27<10:42, 10.36s/it]
 97%|█████████▋| 2174/2235 [6:06:37<10:30, 10.34s/it]
 97%|█████████▋| 2175/2235 [6:06:47<10:19, 10.33s/it]
 97%|█████████▋| 2176/2235 [6:06:58<10:10, 10.35s/it]
 97%|█████████▋| 2177/2235 [6:07:08<10:01, 10.37s/it]
 97%|█████████▋| 2178/2235 [6:07:18<09:52, 10.39s/it]
 97%|█████████▋| 2179/2235 [6:07:29<09:40, 10.37s/it]
 98%|█�
+0: {'loss': 0.496, 'grad_norm': 0.9599463777152424, 'learning_rate': 1.3157960230654482e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: {'loss': 0.5037, 'grad_norm': 0.826007548646021, 'learning_rate': 1.0622500933314661e-06, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: �███████▊| 2180/2235 [6:07:39<09:29, 10.36s/it]
                                                     

 98%|█████████▊| 2180/2235 [6:07:39<09:29, 10.36s/it]
 98%|█████████▊| 2181/2235 [6:07:49<09:18, 10.35s/it]
 98%|█████████▊| 2182/2235 [6:08:00<09:07, 10.33s/it]
 98%|█████████▊| 2183/2235 [6:08:10<08:56, 10.32s/it]
 98%|█████████▊| 2184/2235 [6:08:20<08:45, 10.31s/it]
 98%|█████████▊| 2185/2235 [6:08:31<08:35, 10.30s/it]
 98%|█████████▊| 2186/2235 [6:08:41<08:21, 10.23s/it]
 98%|█████████▊| 2187/2235 [6:08:51<08:08, 10.19s/it]
 98%|█████████▊| 2188/2235 [6:09:01<07:56, 10.14s/it]
 98%|█████████▊| 2189/2235 [6:09:11<07:51, 10.25s/it]
 98%|█████████▊| 2190/2235 [6:09:21<07:41, 10.25s/it]
                                                     

 98%|█████████▊|
+0: {'loss': 0.4899, 'grad_norm': 0.8030553782478187, 'learning_rate': 8.502621676204667e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0:  2190/2235 [6:09:21<07:41, 10.25s/it]
 98%|█████████▊| 2191/2235 [6:09:32<07:30, 10.23s/it]
 98%|█████████▊| 2192/2235 [6:09:42<07:19, 10.22s/it]
 98%|█████████▊| 2193/2235 [6:09:52<07:08, 10.20s/it]
 98%|█████████▊| 2194/2235 [6:10:02<06:56, 10.15s/it]
 98%|█████████▊| 2195/2235 [6:10:12<06:44, 10.12s/it]
 98%|█████████▊| 2196/2235 [6:10:22<06:33, 10.10s/it]
 98%|█████████▊| 2197/2235 [6:10:32<06:22, 10.07s/it]
 98%|█████████▊| 2198/2235 [6:10:42<06:11, 10.04s/it]
 98%|█████████▊| 2199/2235 [6:10:52<06:00, 10.01s/it]
 98%|█████████▊| 2200/2235 [6:11:02<05:49,  9.99s/it]
                                                     

 98%|█████████▊| 2200/2235 [6:11:02<05:49,  9.99s/it]
 98%|█████████▊| 2201/2235 [6:11:12<05:39,  9.99s/it]
 99%|█████████▊| 2202/2
+0: {'loss': 0.4943, 'grad_norm': 0.7635787589722961, 'learning_rate': 6.850520922110425e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.59}
+0: 235 [6:11:22<05:29,  9.98s/it]
 99%|█████████▊| 2203/2235 [6:11:32<05:19,  9.98s/it]
 99%|█████████▊| 2204/2235 [6:11:42<05:09,  9.99s/it]
 99%|█████████▊| 2205/2235 [6:11:52<04:59,  9.98s/it]
 99%|█████████▊| 2206/2235 [6:12:02<04:49,  9.99s/it]
 99%|█████████▊| 2207/2235 [6:12:12<04:39,  9.99s/it]
 99%|█████████▉| 2208/2235 [6:12:22<04:28,  9.96s/it]
 99%|█████████▉| 2209/2235 [6:12:32<04:19,  9.99s/it]
 99%|█████████▉| 2210/2235 [6:12:42<04:10, 10.04s/it]
                                                     

 99%|█████████▉| 2210/2235 [6:12:42<04:10, 10.04s/it]
 99%|█████████▉| 2211/2235 [6:12:52<04:01, 10.08s/it]
 99%|█████████▉| 2212/2235 [6:13:02<03:52, 10.10s/it]
 99%|█████████▉| 2213/2235 [6:13:13<03:42, 10.13s/it]
 99%|█████████▉| 2214/2235 [6:
+0: {'loss': 0.5123, 'grad_norm': 0.8161815977999385, 'learning_rate': 5.706878874605801e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.6}
+0: 13:23<03:32, 10.14s/it]
 99%|█████████▉| 2215/2235 [6:13:33<03:22, 10.15s/it]
 99%|█████████▉| 2216/2235 [6:13:44<03:16, 10.32s/it]
 99%|█████████▉| 2217/2235 [6:13:54<03:05, 10.28s/it]
 99%|█████████▉| 2218/2235 [6:14:04<02:53, 10.21s/it]
 99%|█████████▉| 2219/2235 [6:14:14<02:42, 10.14s/it]
 99%|█████████▉| 2220/2235 [6:14:24<02:31, 10.12s/it]
                                                     

 99%|█████████▉| 2220/2235 [6:14:24<02:31, 10.12s/it]
 99%|█████████▉| 2221/2235 [6:14:34<02:21, 10.13s/it]
 99%|█████████▉| 2222/2235 [6:14:44<02:11, 10.10s/it]
 99%|█████████▉| 2223/2235 [6:14:54<02:01, 10.14s/it]
100%|█████████▉| 2224/2235 [6:15:05<01:52, 10.20s/it]
100%|█████████▉| 2225/2235 [6:15:15<01:42, 10.28s/it]
100%|█████████▉| 2226/2235 [6:15:26<0
+0: {'loss': 0.5008, 'grad_norm': 0.8639484050666172, 'learning_rate': 5.0998557964307e-07, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.6}
+0: [2025-11-23 21:08:14,656] [INFO] [axolotl.core.trainers.base._save:613] [PID:3958317] [RANK:0] Saving model checkpoint to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5/checkpoint-2235
+0: [2025-11-23 21:08:21,740] [INFO] [axolotl.core.trainers.base._save:662] [PID:3958317] [RANK:0] Saving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`
+0: {'train_runtime': 22639.456, 'train_samples_per_second': 1.58, 'train_steps_per_second': 0.099, 'train_loss': 0.5339095356747074, 'memory/max_mem_active(gib)': 57.09, 'memory/max_mem_allocated(gib)': 57.09, 'memory/device_mem_reserved(gib)': 66.93, 'epoch': 0.6}
+0: 1:33, 10.39s/it]
100%|█████████▉| 2227/2235 [6:15:36<01:23, 10.40s/it]
100%|█████████▉| 2228/2235 [6:15:46<01:12, 10.37s/it]
100%|█████████▉| 2229/2235 [6:15:57<01:01, 10.32s/it]
100%|█████████▉| 2230/2235 [6:16:07<00:51, 10.28s/it]
                                                     

100%|█████████▉| 2230/2235 [6:16:07<00:51, 10.28s/it]
100%|█████████▉| 2231/2235 [6:16:17<00:41, 10.25s/it]
100%|█████████▉| 2232/2235 [6:16:27<00:30, 10.24s/it]
100%|█████████▉| 2233/2235 [6:16:37<00:20, 10.22s/it]
100%|█████████▉| 2234/2235 [6:16:48<00:10, 10.22s/it]
100%|██████████| 2235/2235 [6:16:58<00:00, 10.22s/it]
                                                     

100%|██████████| 2235/2235 [6:17:19<00:00, 10.22s/it]
100%|██████████| 2235/2235 [6:17:19<00:00, 10.13s/it]
+0: [2025-11-23 21:08:33,197] [INFO] [axolotl.train.save_trained_model:228] [PID:3958317] [RANK:0] Training completed! Saving trained model to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5.
+0: [2025-11-23 21:08:45,265] [INFO] [axolotl.core.trainers.base._save:613] [PID:3958317] [RANK:0] Saving model checkpoint to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5
+0: [2025-11-23 21:08:52,113] [INFO] [axolotl.core.trainers.base._save:662] [PID:3958317] [RANK:0] Saving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`
+0: [2025-11-23 21:08:52,641] [INFO] [axolotl.train.save_trained_model:350] [PID:3958317] [RANK:0] Model successfully saved to /lustre/fswork/projects/rech/dgo/udv55np/ift/Nemotron-Super-49B-v1_5/gemma-3-4b/0.5