OneScience-Group/CodonTransformer

1#!/bin/bash2#SBATCH --job-name=pretrain3#SBATCH --output=your_output_directory/output_%j.out4#SBATCH --error=your_error_directory/error_%j.err5#SBATCH --nodes=46#SBATCH --gpus-per-node=47#SBATCH --ntasks-per-node=48#SBATCH --time=23:59:009#SBATCH -p compute_full_node10 11# Load required modules12module --ignore_cache load cuda/11.4.413module --ignore_cache load anaconda314source activate your_environment15 16# Change to the working directory17cd your_working_directory18 19# Set environment variables20export CUBLAS_WORKSPACE_CONFIG=:4096:221export PYTHONPATH="${PWD}/model:${PYTHONPATH:-}"22export NCCL_DEBUG=INFO23export PYTHONFAULTHANDLER=124 25# Run the Python script with arguments26stdbuf -oL -eL srun python scripts/pretrain.py \27    --tokenizer_path model/src/CodonTransformerTokenizer.json \28    --train_data_path your_data_directory/pretrain_dataset.json \29    --checkpoint_dir your_checkpoint_directory \30    --batch_size 6 \31    --max_epochs 5 \32    --num_workers 5 \33    --accumulate_grad_batches 1 \34    --num_gpus 16 \35    --learning_rate 0.00005 \36    --warmup_fraction 0.1 \37    --save_interval 5 \38    --seed 12339