Can you please use above code and calculate the FLOPs again? I wonder why your numbers are inconsistent with ours (yours: 5GFLOPs vs ours: 4.6 GFLOPs)? Can you please share the checkpoint for DeiT-S + ConvPatchEmbedding? How much is the improvement? I can evaluate it and add it to the next version of arXiv.

This is the flops and parameters result of DeiT-S->SimA with ConvPatchEmbedding using fvcore to calculate.

module	#parameters or shape	#flops
model	22.588M	4.967G
cls_token	(1, 1, 384)
patch_embed.proj	0.874M	0.412G
patch_embed.proj.0	1.392K	19.268M
patch_embed.proj.0.0	1.296K	16.257M
patch_embed.proj.0.0.weight	(48, 3, 3, 3)
patch_embed.proj.0.1	96	3.011M
patch_embed.proj.0.1.weight	(48,)
patch_embed.proj.0.1.bias	(48,)
patch_embed.proj.2	41.664K	0.132G
patch_embed.proj.2.0	41.472K	0.13G
patch_embed.proj.2.0.weight	(96, 48, 3, 3)
patch_embed.proj.2.1	0.192K	1.505M
patch_embed.proj.2.1.weight	(96,)
patch_embed.proj.2.1.bias	(96,)
patch_embed.proj.4	0.166M	0.131G
patch_embed.proj.4.0	0.166M	0.13G
patch_embed.proj.4.0.weight	(192, 96, 3, 3)
patch_embed.proj.4.1	0.384K	0.753M
patch_embed.proj.4.1.weight	(192,)
patch_embed.proj.4.1.bias	(192,)
patch_embed.proj.6	0.664M	0.13G
patch_embed.proj.6.0	0.664M	0.13G
patch_embed.proj.6.0.weight	(384, 192, 3, 3)
patch_embed.proj.6.1	0.768K	0.376M
patch_embed.proj.6.1.weight	(384,)
patch_embed.proj.6.1.bias	(384,)
blocks	21.303M	4.55G
blocks.0	1.775M	0.379G
blocks.0.gamma1	(384,)
blocks.0.gamma2	(384,)
blocks.0.norm1	0.768K	0.378M
blocks.0.norm1.weight	(384,)
blocks.0.norm1.bias	(384,)
blocks.0.attn	0.591M	0.146G
blocks.0.attn.qkv	0.444M	87.146M
blocks.0.attn.proj	0.148M	29.049M
blocks.0.norm2	0.768K	0.378M
blocks.0.norm2.weight	(384,)
blocks.0.norm2.bias	(384,)
blocks.0.mlp	1.182M	0.232G
blocks.0.mlp.fc1	0.591M	0.116G
blocks.0.mlp.fc2	0.59M	0.116G
blocks.1	1.775M	0.379G
blocks.1.gamma1	(384,)
blocks.1.gamma2	(384,)
blocks.1.norm1	0.768K	0.378M
blocks.1.norm1.weight	(384,)
blocks.1.norm1.bias	(384,)
blocks.1.attn	0.591M	0.146G
blocks.1.attn.qkv	0.444M	87.146M
blocks.1.attn.proj	0.148M	29.049M
blocks.1.norm2	0.768K	0.378M
blocks.1.norm2.weight	(384,)
blocks.1.norm2.bias	(384,)
blocks.1.mlp	1.182M	0.232G
blocks.1.mlp.fc1	0.591M	0.116G
blocks.1.mlp.fc2	0.59M	0.116G
blocks.2	1.775M	0.379G
blocks.2.gamma1	(384,)
blocks.2.gamma2	(384,)
blocks.2.norm1	0.768K	0.378M
blocks.2.norm1.weight	(384,)
blocks.2.norm1.bias	(384,)
blocks.2.attn	0.591M	0.146G
blocks.2.attn.qkv	0.444M	87.146M
blocks.2.attn.proj	0.148M	29.049M
blocks.2.norm2	0.768K	0.378M
blocks.2.norm2.weight	(384,)
blocks.2.norm2.bias	(384,)
blocks.2.mlp	1.182M	0.232G
blocks.2.mlp.fc1	0.591M	0.116G
blocks.2.mlp.fc2	0.59M	0.116G
blocks.3	1.775M	0.379G
blocks.3.gamma1	(384,)
blocks.3.gamma2	(384,)
blocks.3.norm1	0.768K	0.378M
blocks.3.norm1.weight	(384,)
blocks.3.norm1.bias	(384,)
blocks.3.attn	0.591M	0.146G
blocks.3.attn.qkv	0.444M	87.146M
blocks.3.attn.proj	0.148M	29.049M
blocks.3.norm2	0.768K	0.378M
blocks.3.norm2.weight	(384,)
blocks.3.norm2.bias	(384,)
blocks.3.mlp	1.182M	0.232G
blocks.3.mlp.fc1	0.591M	0.116G
blocks.3.mlp.fc2	0.59M	0.116G
blocks.4	1.775M	0.379G
blocks.4.gamma1	(384,)
blocks.4.gamma2	(384,)
blocks.4.norm1	0.768K	0.378M
blocks.4.norm1.weight	(384,)
blocks.4.norm1.bias	(384,)
blocks.4.attn	0.591M	0.146G
blocks.4.attn.qkv	0.444M	87.146M
blocks.4.attn.proj	0.148M	29.049M
blocks.4.norm2	0.768K	0.378M
blocks.4.norm2.weight	(384,)
blocks.4.norm2.bias	(384,)
blocks.4.mlp	1.182M	0.232G
blocks.4.mlp.fc1	0.591M	0.116G
blocks.4.mlp.fc2	0.59M	0.116G
blocks.5	1.775M	0.379G
blocks.5.gamma1	(384,)
blocks.5.gamma2	(384,)
blocks.5.norm1	0.768K	0.378M
blocks.5.norm1.weight	(384,)
blocks.5.norm1.bias	(384,)
blocks.5.attn	0.591M	0.146G
blocks.5.attn.qkv	0.444M	87.146M
blocks.5.attn.proj	0.148M	29.049M
blocks.5.norm2	0.768K	0.378M
blocks.5.norm2.weight	(384,)
blocks.5.norm2.bias	(384,)
blocks.5.mlp	1.182M	0.232G
blocks.5.mlp.fc1	0.591M	0.116G
blocks.5.mlp.fc2	0.59M	0.116G
blocks.6	1.775M	0.379G
blocks.6.gamma1	(384,)
blocks.6.gamma2	(384,)
blocks.6.norm1	0.768K	0.378M
blocks.6.norm1.weight	(384,)
blocks.6.norm1.bias	(384,)
blocks.6.attn	0.591M	0.146G
blocks.6.attn.qkv	0.444M	87.146M
blocks.6.attn.proj	0.148M	29.049M
blocks.6.norm2	0.768K	0.378M
blocks.6.norm2.weight	(384,)
blocks.6.norm2.bias	(384,)
blocks.6.mlp	1.182M	0.232G
blocks.6.mlp.fc1	0.591M	0.116G
blocks.6.mlp.fc2	0.59M	0.116G
blocks.7	1.775M	0.379G
blocks.7.gamma1	(384,)
blocks.7.gamma2	(384,)
blocks.7.norm1	0.768K	0.378M
blocks.7.norm1.weight	(384,)
blocks.7.norm1.bias	(384,)
blocks.7.attn	0.591M	0.146G
blocks.7.attn.qkv	0.444M	87.146M
blocks.7.attn.proj	0.148M	29.049M
blocks.7.norm2	0.768K	0.378M
blocks.7.norm2.weight	(384,)
blocks.7.norm2.bias	(384,)
blocks.7.mlp	1.182M	0.232G
blocks.7.mlp.fc1	0.591M	0.116G
blocks.7.mlp.fc2	0.59M	0.116G
blocks.8	1.775M	0.379G
blocks.8.gamma1	(384,)
blocks.8.gamma2	(384,)
blocks.8.norm1	0.768K	0.378M
blocks.8.norm1.weight	(384,)
blocks.8.norm1.bias	(384,)
blocks.8.attn	0.591M	0.146G
blocks.8.attn.qkv	0.444M	87.146M
blocks.8.attn.proj	0.148M	29.049M
blocks.8.norm2	0.768K	0.378M
blocks.8.norm2.weight	(384,)
blocks.8.norm2.bias	(384,)
blocks.8.mlp	1.182M	0.232G
blocks.8.mlp.fc1	0.591M	0.116G
blocks.8.mlp.fc2	0.59M	0.116G
blocks.9	1.775M	0.379G
blocks.9.gamma1	(384,)
blocks.9.gamma2	(384,)
blocks.9.norm1	0.768K	0.378M
blocks.9.norm1.weight	(384,)
blocks.9.norm1.bias	(384,)
blocks.9.attn	0.591M	0.146G
blocks.9.attn.qkv	0.444M	87.146M
blocks.9.attn.proj	0.148M	29.049M
blocks.9.norm2	0.768K	0.378M
blocks.9.norm2.weight	(384,)
blocks.9.norm2.bias	(384,)
blocks.9.mlp	1.182M	0.232G
blocks.9.mlp.fc1	0.591M	0.116G
blocks.9.mlp.fc2	0.59M	0.116G
blocks.10	1.775M	0.379G
blocks.10.gamma1	(384,)
blocks.10.gamma2	(384,)
blocks.10.norm1	0.768K	0.378M
blocks.10.norm1.weight	(384,)
blocks.10.norm1.bias	(384,)
blocks.10.attn	0.591M	0.146G
blocks.10.attn.qkv	0.444M	87.146M
blocks.10.attn.proj	0.148M	29.049M
blocks.10.norm2	0.768K	0.378M
blocks.10.norm2.weight	(384,)
blocks.10.norm2.bias	(384,)
blocks.10.mlp	1.182M	0.232G
blocks.10.mlp.fc1	0.591M	0.116G
blocks.10.mlp.fc2	0.59M	0.116G
blocks.11	1.775M	0.379G
blocks.11.gamma1	(384,)
blocks.11.gamma2	(384,)
blocks.11.norm1	0.768K	0.378M
blocks.11.norm1.weight	(384,)
blocks.11.norm1.bias	(384,)
blocks.11.attn	0.591M	0.146G
blocks.11.attn.qkv	0.444M	87.146M
blocks.11.attn.proj	0.148M	29.049M
blocks.11.norm2	0.768K	0.378M
blocks.11.norm2.weight	(384,)
blocks.11.norm2.bias	(384,)
blocks.11.mlp	1.182M	0.232G
blocks.11.mlp.fc1	0.591M	0.116G
blocks.11.mlp.fc2	0.59M	0.116G
norm	0.768K	0.378M
norm.weight	(384,)
norm.bias	(384,)
head	0.385M	0.384M
head.weight	(1000, 384)
head.bias	(1000,)
pos_embeder.token_projection	24.96K	4.817M
pos_embeder.token_projection.weight	(384, 64, 1, 1)
pos_embeder.token_projection.bias	(384,)

I haven't finished my training on DeiT-S + ConvPatchEmbedding but the validation accuracy has gone a lot higher than original DeiT-S.

How to replicate DeiT-SimA result #4

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions