File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -73,15 +73,7 @@ mask_padding_tokens: True
7373attention_sharding_uniform : True
7474dropout : 0.1
7575
76- flash_block_sizes : {
77- " block_q" : 2048,
78- " block_kv_compute" : 512,
79- " block_kv" : 2048,
80- " block_q_dkv" : 2048,
81- " block_kv_dkv" : 2048,
82- " block_kv_dkv_compute" : 512,
83- " use_fused_bwd_kernel " : True
84- }
76+ flash_block_sizes : {}
8577# Use on v6e
8678# flash_block_sizes: {
8779# "block_q" : 3024,
Original file line number Diff line number Diff line change @@ -72,16 +72,7 @@ mask_padding_tokens: True
7272attention_sharding_uniform : True
7373dropout : 0.1
7474
75- flash_block_sizes : {
76- " block_q" : 1024,
77- " block_kv_compute" : 256,
78- " block_kv" : 1024,
79- " block_q_dkv" : 1024,
80- " block_kv_dkv" : 1024,
81- " block_kv_dkv_compute" : 256,
82- " block_q_dq" : 1024,
83- " block_kv_dq" : 1024
84- }
75+ flash_block_sizes : {}
8576# Use on v6e
8677# flash_block_sizes: {
8778# "block_q" : 3024,
You can’t perform that action at this time.
0 commit comments