
----------------------------------------------Summary-----------------------------------------------
batch_size_per_gpu: 2
max_batch_size_per_gpu: 2
gradient_accumulation_steps: 2
global_batch_size: 2560
dp_size: 640
tp_size: 1
pp_size: 1
sp_size: 1
ds_zero: STAGE_3
total_num_gpus: 640
seq_len: 2048
total_num_tokens: 300.0 G
num_params_total: 529.51 G
activation_recomputation: FULL
achived_flops: 138.0
flops_efficiency: 0.4423076923076923
hbm_memory_efficiency: 1
num_flops_total_per_micro_batch: 17639.67 T
weight_memory_per_gpu: 3.71 GB
gradient_memory_per_gpu: 1.65 GB
optimizer_state_memory_per_gpu: 13.21 GB
activation_memory_bs1: 8.81 GB
activation_memory_per_gpu: 17.62 GB
latency_per_micro_batch: 2.13 minutes
latency_fwd: 32.0 s
latency_fwd_attn: 10.98 s
latency_fwd_mlp: 20.91 s
latency_fwd_layernorm: 45.31 ms
latency_fwd_tp_comm: 0.0 us
latency_fwd_input_embedding: 1.32 ms
latency_fwd_output_embedding_loss: 61.12 ms
latency_per_iter: 4.26 minutes
total_training_latency: 169.31 days
gpu_hours: 2600558
----------------------------------------------------------------------------------------------------
