CLIP-convnext_large_d_320.laion2B-s29B-b131K-ft

/opt/slurm/sbin/srun --cpu_bind=v --accel-bind=gn python -m training.main \
    --save-frequency 1 \
    --name "convnext_large_320" \
    --pretrained ""/runs/convnext_large_256/epoch_128.pt" \
    --resume 'latest' \
    --train-data="pipe:aws s3 cp s3://mybucket/path/{laion{00000..xxxxx}.tar -" \
    --train-num-samples 203666042 \
    --dataset-type webdataset \
    --precision amp_bfloat16 \
    --beta2 0.98 \
    --warmup 2000 \
    --batch-size=256 \
    --epochs=12 \
    --dataset-resampled \
    --aug-cfg use_timm=True scale='(0.5, 1.0)' re_prob=0.4 \
    --clip-grad-norm 5.0 \
    --lr 5e-5 \
    --workers=6 \
    --model "convnext_large_d_320" \
    --seed 0 \
    --ddp-static-graph \
    --local-loss \
    --gather-with-grad \
    --grad-checkpointing

@inproceedings{schuhmann2022laionb,
  title={{LAION}-5B: An open large-scale dataset for training next generation image-text models},
  author={Christoph Schuhmann and
          Romain Beaumont and
          Richard Vencu and
          Cade W Gordon and
          Ross Wightman and
          Mehdi Cherti and
          Theo Coombes and
          Aarush Katta and
          Clayton Mullis and
          Mitchell Wortsman and
          Patrick Schramowski and
          Srivatsa R Kundurthy and
          Katherine Crowson and
          Ludwig Schmidt and
          Robert Kaczmarczyk and
          Jenia Jitsev},
  booktitle={Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track},
  year={2022},
  url={https://openreview.net/forum?id=M3Y74vmsMcY}
}