codeShare/lora-training-data
2785
1{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":15952942,"datasetId":8022630,"databundleVersionId":16912023}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"How to use:\n1) Create a private repo , here its 'image_caption_dataset' for your own kaggle account\n2) Run a Google Colab cell to prepp your LoRa images. To comply with Kaggle TOS , its best your images are non visible to users snooping on your notebook. The images are therefore encrypted.\n\nTo prepare a dataset , run this google colab cell\n\n``` # Session A , encrypt dataset\n\n# =============================================================================\n#@markdown # **CELL 1A**: Mount Drive + HF auth + Install dependencies\n# =============================================================================\n# This cell mounts Google Drive, logs into Hugging Face, removes old diffusers,\n# and installs the latest diffusers + encryption libraries (pynacl + datasets).\nfrom google.colab import drive, userdata\nfrom huggingface_hub import login\nimport torch\nimport os\nimport gc\nimport shutil\n\ndrive.mount('/content/drive')\n\nhf_token = userdata.get('HF_TOKEN')\nif hf_token:\n login(token=hf_token)\nelse:\n print(\"⚠️ No HF_TOKEN found in secrets.\")\n\nprint(\"🧹 Removing old diffusers...\")\n!pip uninstall -y diffusers > /dev/null 2>&1\n!rm -rf /usr/local/lib/python3.12/dist-packages/diffusers* ~/.cache/pip/*diffusers*\n\nprint(\"🔄 Installing latest diffusers...\")\n!pip install -q git+https://github.com/huggingface/diffusers.git --force-reinstall --no-deps\n!python -m pip cache purge\n\nprint(\"🔐 Installing encryption + datasets...\")\n!pip install -q pynacl datasets\n\nprint(\"✅ Cell 1A complete!\")\n\n# =============================================================================\n#@markdown # **CELL 2A**: Set input ZIP path + optional upload widget\n# =============================================================================\n# This cell defines where the source ZIP is (Drive or upload) and sets zip_path\n# for Cell 3A. You can enable the widget to upload images or a ZIP directly.\nupload_from_widget = False #@param {type:'boolean'}\ninput_zip_path = '/content/drive/MyDrive/gow.zip' #@param {type:'string'}\n\nzip_path = None\n\nif not upload_from_widget:\n zip_path = input_zip_path\n print(f\"✅ Using ZIP from Drive: {zip_path}\")\nelse:\n print(\"📤 Widget upload enabled – run the upload section below.\")\n\n#---#\n#@title **Upload files via widget (only run if upload_from_widget = True)**\nimport os\nfrom google.colab import files\nimport zipfile\nimport shutil\n\nif upload_from_widget:\n print(\"Please upload your ZIP file or individual image files now.\")\n uploaded = files.upload()\n\n if not uploaded:\n raise ValueError(\"No files uploaded.\")\n\n if len(uploaded) == 1 and list(uploaded.keys())[0].endswith('.zip'):\n uploaded_filename = list(uploaded.keys())[0]\n shutil.move(uploaded_filename, '/content/' + uploaded_filename)\n zip_path = '/content/' + uploaded_filename\n print(f\"✅ Using uploaded ZIP: {zip_path}\")\n else:\n temp_img_dir = '/content/uploaded_images_temp'\n os.makedirs(temp_img_dir, exist_ok=True)\n image_count = 0\n for fname, content in uploaded.items():\n if fname.lower().endswith(('.png', '.jpg', '.jpeg', '.webp', '.gif')):\n with open(os.path.join(temp_img_dir, fname), 'wb') as f:\n f.write(content)\n image_count += 1\n else:\n print(f\"Skipping non-image: {fname}\")\n\n if image_count == 0:\n raise ValueError(\"No valid images uploaded.\")\n\n zip_path = '/content/uploaded_images.zip'\n with zipfile.ZipFile(zip_path, 'w') as zf:\n for root, _, files_in_dir in os.walk(temp_img_dir):\n for file_in_dir in files_in_dir:\n zf.write(os.path.join(root, file_in_dir), os.path.basename(file_in_dir))\n shutil.rmtree(temp_img_dir)\n print(f\"✅ Created ZIP from {image_count} images: {zip_path}\")\nelse:\n print(\"Widget upload disabled – using Drive path from above.\")\n\nprint(f\"Final zip_path ready for Cell 3A: {zip_path}\")\n\n# =============================================================================\n#@markdown # **CELL 3A**: Build ENCRYPTED INPUT DATASET\n# =============================================================================\n# This cell extracts the ZIP, encrypts every image using your password,\n# creates a Hugging Face Dataset, and saves it to disk.\nimport hashlib\nimport io\nimport glob\nfrom PIL import Image\nfrom datasets import Dataset\nfrom nacl.secret import SecretBox\nfrom nacl.utils import random\nimport zipfile\n\ndebug = False #@param {type:\"boolean\"}\n\n# 🔑 USER KEY\nencryption_password = \"banana\" #@param {type:\"string\"}\n\n# ===== KEY DERIVATION =====\ndef derive_key(password):\n return hashlib.sha256(password.encode()).digest()\n\nSECRET_KEY = derive_key(encryption_password)\nbox = SecretBox(SECRET_KEY)\n\n# ===== IMAGE SERIALIZATION =====\ndef pil_to_bytes(img):\n buf = io.BytesIO()\n img.save(buf, format=\"PNG\")\n return buf.getvalue()\n\n# ===== ENCRYPT =====\ndef encrypt_bytes(data):\n nonce = random(SecretBox.NONCE_SIZE)\n return box.encrypt(data, nonce)\n\n# ====================== LOAD ZIP ======================\nif debug:\n print(f\"📦 Building encrypted dataset from: {zip_path}\")\n\nwith zipfile.ZipFile(zip_path, 'r') as z:\n z.extractall('/content/input_images')\n\nimage_files = sorted(glob.glob('/content/input_images/*.*'))\nimage_files = [f for f in image_files if f.lower().endswith(('.png','.jpg','.jpeg','.webp'))]\n\nif debug:\n print(f\"Found {len(image_files)} images to encrypt.\")\n\nrecords = []\n\nfor img_path in image_files:\n img = Image.open(img_path).convert(\"RGB\")\n img_bytes = pil_to_bytes(img)\n enc = encrypt_bytes(img_bytes)\n records.append({\"image_encrypted\": enc})\n\nprint(f\"✅ Encrypted {len(records)} images\")\n\n# ====================== SAVE DATASET ======================\nds = Dataset.from_list(records)\ndataset_path = \"/content/drive/MyDrive/encrypted_input_dataset\"\nds.save_to_disk(dataset_path)\n\nif debug:\n print(f\"📦 Dataset saved to: {dataset_path}\")\n\nprint(f\"✅ Encrypted input dataset ready at {dataset_path}\")\nprint(\"👉 You can now run Session B.\")\n\n# ================================================\n#@title 🔌 End Session A. Auto Disconnect Colab Session (optional)\n# ================================================\nenable = False #@param {type:'boolean'}\nif enable:\n print(\"🔌 Disconnecting Colab session in 3 seconds...\")\n import time\n time.sleep(3)\n from google.colab import runtime\n runtime.unassign()\n print(\"Session disconnected.\")\n```\n\n\n2 ) Upload the contents of xxx , data-00000-of-00001.arrow , dataset_info.json and state.json to your Kaggle repo\n\n3) Upload the password youn use to encrypt the images password.txt in your kaggle repo\n\nThe kaggle repo should now look like this\n\nimage_caption_dataset\n - data-00000-of-00001.arrow\n - dataset_info.json\n - password.txt\n - state.json\n\n4) Run the Kaggle notebook , by uploading the notebook and click 'Save Version' using accelerator 2xT4\n\n5) Just close the notebook and do something else for awhile. All kaggle notebooks auto disconnect once the all the cells are complete , up to a maximum session of 12 hours\n\n6) Re-visit your notebook , and retrieve the encoded images zip.\n\n7) Back to google colab , decrypt the results using this cell\n\n```\n# Session C , Decrypt a dataset\n\n# =============================================================================\n#@markdown # **CELL 1C**: Mount Drive + HF auth + Install dependencies\n# =============================================================================\n# Same setup as previous sessions.\nfrom google.colab import drive, userdata\nfrom huggingface_hub import login\nimport torch\nimport os\nimport gc\nimport shutil\n\ndrive.mount('/content/drive')\n\nhf_token = userdata.get('HF_TOKEN')\nif hf_token:\n login(token=hf_token)\nelse:\n print(\"⚠️ No HF_TOKEN found in secrets.\")\n\nprint(\"🧹 Removing old diffusers...\")\n!pip uninstall -y diffusers > /dev/null 2>&1\n!rm -rf /usr/local/lib/python3.12/dist-packages/diffusers* ~/.cache/pip/*diffusers*\n\nprint(\"🔄 Installing latest diffusers...\")\n!pip install -q git+https://github.com/huggingface/diffusers.git --force-reinstall --no-deps\n!python -m pip cache purge\n\nprint(\"🔐 Installing encryption + datasets...\")\n!pip install -q pynacl datasets\n\nprint(\"✅ Cell 1C complete!\")\n\n# =============================================================================\n#@markdown # **CELL 2C**: Decrypt dataset / checkpoints / ZIP → final images ZIP\n# =============================================================================\n# Choose one mode with the checkboxes. Outputs a clean ZIP of decrypted images.\n# Supports all checkpoints (default), dataset, or single ZIP.\nimport os\nimport zipfile\nimport io\nimport hashlib\nfrom PIL import Image\nfrom nacl.secret import SecretBox\nfrom datasets import load_from_disk\nfrom google.colab import files\n\n# ================= MODE CONTROLS =================\nprocess_all_checkpoints = True #@param {type:\"boolean\"}\ndecrypt_from_dataset = False #@param {type:\"boolean\"}\ndecrypt_from_zip = False #@param {type:\"boolean\"}\nupload_zip_manually = False #@param {type:\"boolean\"}\ndownload_output_zip = True #@param {type:\"boolean\"}\n\ndataset_to_decrypt = \"/content/encrypted_output_dataset\" #@param {type:\"string\"}\nencrypted_zip_path = \"/content/drive/MyDrive/klein_checkpoints/checkpoint_2.zip\" #@param {type:\"string\"}\noutput_zip_name = \"decrypted_results.zip\" #@param {type:\"string\"}\n\nimage_format = \"JPG\" #@param [\"PNG\", \"JPG\"]\n\noutput_base_folder = \"/content/zip_outputs\"\nos.makedirs(output_base_folder, exist_ok=True)\ncheckpoint_dir = \"/content/drive/MyDrive/klein_checkpoints\"\n\ndebug = False #@param {type:\"boolean\"}\n\n# ================= 🔑 KEY =================\ndecryption_password = \"banana\" #@param {type:\"string\"}\ndef derive_key(password):\n return hashlib.sha256(password.encode()).digest()\n\nSECRET_KEY = derive_key(decryption_password)\nbox = SecretBox(SECRET_KEY)\n\ndef decrypt_bytes(enc):\n return box.decrypt(enc)\n\n# ================= IMAGE IO =================\ndef bytes_to_pil(b):\n return Image.open(io.BytesIO(b)).convert(\"RGB\")\n\n# ================= DECRYPT HELPERS =================\ndef decrypt_stream(enc_bytes_list):\n out = []\n for enc in enc_bytes_list:\n dec = decrypt_bytes(enc)\n out.append(bytes_to_pil(dec))\n return out\n\ndef decrypt_zip(zip_path):\n with zipfile.ZipFile(zip_path, \"r\") as z:\n enc_files = [z.read(n) for n in z.namelist() if n.endswith(\".enc\")]\n return decrypt_stream(enc_files)\n\ndef decrypt_dataset(ds):\n return decrypt_stream([x[\"image_encrypted\"] for x in ds])\n\n# ================= ZIP BUILDER =================\ndef write_images_to_zip(image_pil_list, out_zip_path, fmt):\n with zipfile.ZipFile(out_zip_path, \"w\", compression=zipfile.ZIP_DEFLATED) as zf:\n ext = \"png\" if fmt == \"PNG\" else \"jpg\"\n for i, img in enumerate(image_pil_list):\n buf = io.BytesIO()\n if fmt == \"JPG\":\n img.save(buf, format=\"JPEG\", quality=95, optimize=True)\n else:\n img.save(buf, format=fmt)\n zf.writestr(f\"{i:05d}.{ext}\", buf.getvalue())\n\n# ================= OUTPUT BUFFER =================\nall_image_pil = []\n\n# =============================================================================\n# 🔥 MODE 1: ALL CHECKPOINTS (recommended)\n# =============================================================================\nif process_all_checkpoints:\n print(f\"📂 Scanning checkpoints in: {checkpoint_dir}\")\n checkpoint_zips = sorted([\n os.path.join(checkpoint_dir, f)\n for f in os.listdir(checkpoint_dir)\n if f.endswith(\".zip\") and \"checkpoint\" in f\n ])\n print(f\"🔍 Found {len(checkpoint_zips)} checkpoint ZIPs\")\n for idx, zpath in enumerate(checkpoint_zips):\n if debug:\n print(f\" Processing {zpath}\")\n imgs = decrypt_zip(zpath)\n all_image_pil.extend(imgs)\n\n# =============================================================================\n# 🔓 MODE 2: ENCRYPTED DATASET\n# =============================================================================\nelif decrypt_from_dataset:\n print(f\"📦 Loading dataset: {dataset_to_decrypt}\")\n ds = load_from_disk(dataset_to_decrypt)\n imgs = decrypt_dataset(ds)\n all_image_pil.extend(imgs)\n\n# =============================================================================\n# 🔓 MODE 3: SINGLE ZIP\n# =============================================================================\nelif decrypt_from_zip:\n if upload_zip_manually:\n print(\"📤 Please upload your encrypted ZIP now...\")\n uploaded = files.upload()\n zip_name = list(uploaded.keys())[0]\n zip_path = \"/content/\" + zip_name\n with open(zip_path, \"wb\") as f:\n f.write(uploaded[zip_name])\n else:\n zip_path = encrypted_zip_path\n print(f\"📦 Decrypting single ZIP: {zip_path}\")\n imgs = decrypt_zip(zip_path)\n all_image_pil.extend(imgs)\n\n# =============================================================================\n# 📦 FINAL PACKAGING\n# =============================================================================\nif download_output_zip and all_image_pil:\n final_zip = os.path.join(output_base_folder, output_zip_name)\n print(f\"📦 Building final {image_format} ZIP with {len(all_image_pil)} images → {final_zip}\")\n write_images_to_zip(all_image_pil, final_zip, image_format)\n files.download(final_zip)\n print(\"⬇️ Download started via google.colab.files.download\")\nelse:\n print(\"✅ Decryption complete – no download requested.\")\n\nprint(\"🎯 Session C finished!\")\n\n# ================================================\n#@title 🔌 End session C. Auto Disconnect Colab Session\n# ================================================\n\nenable = False #@param {type:'boolean'}\nif enable:\n print(\"🔌 Disconnecting Colab session in 3 seconds...\")\n import time\n time.sleep(3)\n\n from google.colab import runtime\n runtime.unassign()\n\n print(\"Session disconnected.\")\n```","metadata":{}},{"cell_type":"markdown","source":"Cell 1B + 2B","metadata":{}},{"cell_type":"code","source":"# **CELL 1B**: Kaggle Secrets + HF auth + Install dependencies\n# =============================================================================\n# This cell sets up Kaggle secrets for HF_TOKEN, logs into Hugging Face,\n# removes old diffusers, and installs the latest diffusers + encryption + SDNQ libraries.\nfrom kaggle_secrets import UserSecretsClient\nfrom huggingface_hub import login\nimport torch\nimport os\nimport gc\nimport shutil\n\n# === Kaggle Secrets ===\nsecrets = UserSecretsClient()\ntry:\n hf_token = secrets.get_secret(\"HF_TOKEN\")\n login(token=hf_token)\n print(\"✅ HF login successful\")\nexcept Exception:\n print(\"⚠️ No HF_TOKEN secret found or login failed.\")\n\nprint(\"🧹 Removing old diffusers...\")\n!pip uninstall -y diffusers > /dev/null 2>&1\n!rm -rf /usr/local/lib/python3.12/dist-packages/diffusers* ~/.cache/pip/*diffusers*\n\nprint(\"🔄 Installing latest diffusers...\")\n!pip install -q git+https://github.com/huggingface/diffusers.git --force-reinstall --no-deps\n!python -m pip cache purge\n\nprint(\"🔐 Installing encryption + datasets + SDNQ...\")\n!pip install -q pynacl datasets sdnq\n\nprint(\"✅ Cell 1B complete! (Kaggle 2-GPU ready)\")\n\n# **CELL 2B**: Kaggle paths + inference parameters\n# =============================================================================\n# This cell contains ALL user-adjustable settings for the Kaggle run:\n# path to your Kaggle Dataset (repo), password .txt file, model, prompt, resolution, etc.\nkaggle_repo_path = \"/kaggle/input/datasets/nekos4lyfe/image-caption-dataset\" #[@param](https://x.com/param) {type:\"string\"}\npassword_txt_file = \"password.txt\" #[@param](https://x.com/param) {type:\"string\"}\n\nMODEL_ID = \"codeShare/FLUX.2-klein-AIO-SDNQ-4bit-dynamic\" #[@param](https://x.com/param) ['codeShare/Flux-Klein-SDNQ-4bit','codeShare/FLUX.2-klein-AIO-SDNQ-4bit-dynamic', 'codeShare/unstableRevolution_SDNQ']\nedit_prompt = 'improve this illustration. fine art color contrast with pleasant quality. the background is dark gray.' #[@param](https://x.com/param) {type:'string'}\n\nresolution = '1024 x 1024 (Square)' #[@param](https://x.com/param) [\"1024 x 1024 (Square)\", \"512 x 1024 (Portrait)\", \"768 x 1024 (Slight Portrait)\", \"1536 x 1024 (Landscape)\", \"2048 x 1024 (Wide Landscape)\"]\n\nmax_image_dimension = 2048 # resize image with pixel length (longest side) \n # exceeding this value before loading into VAE \n#-----#\nsave_checkpoint_every_n = True #[@param](https://x.com/param) {type:\"boolean\"}\nsave_every_n = 30 #[@param](https://x.com/param) {type:\"slider\", min:1, max:100, step:1}\n\ndebug = True #[@param](https://x.com/param) {type:\"boolean\"}\n\n# Derived paths\nencrypted_input_dataset_path = os.path.join(kaggle_repo_path, \"\")\npassword_path = os.path.join(kaggle_repo_path, password_txt_file)\n\nprint(\"✅ All Kaggle parameters set.\")\nif debug:\n print(f\" Kaggle repo : {kaggle_repo_path}\")\n print(f\" Dataset path : {encrypted_input_dataset_path}\")\n print(f\" Password file : {password_path}\")\n print(f\" Model : {MODEL_ID}\")\n print(f\" Resolution : {resolution}\")\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-26T19:45:38.495266Z","iopub.execute_input":"2026-04-26T19:45:38.495860Z","iopub.status.idle":"2026-04-26T19:45:57.800727Z","shell.execute_reply.started":"2026-04-26T19:45:38.495825Z","shell.execute_reply":"2026-04-26T19:45:57.799288Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[{"name":"stdout","text":"✅ HF login successful\n🧹 Removing old diffusers...\n🔄 Installing latest diffusers...\n Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n Building wheel for diffusers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\nFiles removed: 24\n🔐 Installing encryption + datasets + SDNQ...\n✅ Cell 1B complete! (Kaggle 2-GPU ready)\n✅ All Kaggle parameters set.\n Kaggle repo : /kaggle/input/datasets/nekos4lyfe/image-caption-dataset\n Dataset path : /kaggle/input/datasets/nekos4lyfe/image-caption-dataset/\n Password file : /kaggle/input/datasets/nekos4lyfe/image-caption-dataset/password.txt\n Model : codeShare/FLUX.2-klein-AIO-SDNQ-4bit-dynamic\n Resolution : 1024 x 1024 (Square)\n","output_type":"stream"}],"execution_count":10},{"cell_type":"markdown","source":"Cell 3B","metadata":{}},{"cell_type":"code","source":"#@markdown # **CELL 3B**: Load TWO SDNQ-optimized FLUX.2-Klein models (one per GPU)# =============================================================================\nimport torch\nimport gc\nimport os\nfrom diffusers import Flux2KleinPipeline\nfrom sdnq.common import use_torch_compile as triton_is_available\nfrom sdnq.loader import apply_sdnq_options_to_model\ngc.collect()\ntorch.cuda.empty_cache()\n# ====================== LOAD PIPELINE (one per GPU) ======================\ndef load_klein_pipe(gpu_id: int):\n \n torch.cuda.set_device(gpu_id)\n print(f\" [GPU {gpu_id}] Loading model: {MODEL_ID}\")\n pipe = Flux2KleinPipeline.from_pretrained(\n MODEL_ID,\n torch_dtype=torch.float16,\n low_cpu_mem_usage=True,\n device_map=\"cpu\",\n )\n\n print(f\" Base pipeline loaded on GPU {gpu_id} (CPU-safe)\")\n \n # === SDNQ on transformer + text_encoder (exact same as working single-GPU version) ===\n print(f\" [GPU {gpu_id}] Applying SDNQ optimizations...\")\n os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n \n if torch.cuda.is_available() and triton_is_available:\n pipe.transformer = apply_sdnq_options_to_model(\n pipe.transformer, use_quantized_matmul=True\n )\n pipe.text_encoder = apply_sdnq_options_to_model(\n pipe.text_encoder, use_quantized_matmul=True\n )\n print(f\" Quantized matmul enabled on transformer + text_encoder (GPU {gpu_id})\")\n\n # === MEMORY OPTIMIZATIONS ===\n print(f\" [GPU {gpu_id}] Enabling model CPU offload + VAE optimizations...\")\n pipe.enable_model_cpu_offload(gpu_id=gpu_id)\n pipe.vae.enable_slicing()\n pipe.vae.enable_tiling()\n \n gc.collect()\n torch.cuda.empty_cache()\n torch.cuda.reset_peak_memory_stats(gpu_id)\n return pipe\n#----#\n# ====================== LOAD BOTH MODELS ======================\npipe0 = load_klein_pipe(0)\ngc.collect()\ntorch.cuda.empty_cache()\npipe1 = load_klein_pipe(1)\nprint(\"\\n Both SDNQ-optimized Klein models ready on separate GPUs!\")\nprint(f\"GPU 0 VRAM: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB\")\nprint(f\"GPU 1 VRAM: {torch.cuda.memory_allocated(1) / 1e9:.2f} GB\")\nprint(\" Run the new Cell 4B below\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-26T19:58:44.013964Z","iopub.execute_input":"2026-04-26T19:58:44.014844Z","iopub.status.idle":"2026-04-26T19:58:58.497587Z","shell.execute_reply.started":"2026-04-26T19:58:44.014804Z","shell.execute_reply":"2026-04-26T19:58:58.496689Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[{"name":"stdout","text":" [GPU 0] Loading model: codeShare/FLUX.2-klein-AIO-SDNQ-4bit-dynamic\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading pipeline components...: 0%| | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fcad4b77dde44a25815d609abb6f51f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights: 0%| | 0/901 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"89bff95380ad43728dceba72ba235f4c"}},"metadata":{}},{"name":"stdout","text":" Base pipeline loaded on GPU 0 (CPU-safe)\n [GPU 0] Applying SDNQ optimizations...\n Quantized matmul enabled on transformer + text_encoder (GPU 0)\n [GPU 0] Enabling model CPU offload + VAE optimizations...\n [GPU 1] Loading model: codeShare/FLUX.2-klein-AIO-SDNQ-4bit-dynamic\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading pipeline components...: 0%| | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a596474735f94fb8894f213d95e8981e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights: 0%| | 0/901 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a29b56a8d5f4d24bb407196727b0dee"}},"metadata":{}},{"name":"stdout","text":" Base pipeline loaded on GPU 1 (CPU-safe)\n [GPU 1] Applying SDNQ optimizations...\n Quantized matmul enabled on transformer + text_encoder (GPU 1)\n [GPU 1] Enabling model CPU offload + VAE optimizations...\n\n Both SDNQ-optimized Klein models ready on separate GPUs!\nGPU 0 VRAM: 4.42 GB\nGPU 1 VRAM: 0.01 GB\n Run the new Cell 4B below\n","output_type":"stream"}],"execution_count":17},{"cell_type":"markdown","source":"Cell 4B","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n#@markdown # **CELL 4B **\n# =============================================================================\nimport os\nimport shutil\nimport torch\nimport gc\nimport datetime\nimport io\nimport hashlib\nimport time\nimport traceback\nfrom PIL import Image\nfrom nacl.secret import SecretBox\nfrom nacl.utils import random\nfrom datasets import load_from_disk, Dataset\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\n# ================= SETTINGS =================\noutput_folder0 = \"/kaggle/working/edited_images_gpu0\"\noutput_folder1 = \"/kaggle/working/edited_images_gpu1\"\ncheckpoint_folder = \"/kaggle/working/checkpoints\"\nimage_edit_results_folder = \"/kaggle/working/image_edit_results\"\n\nprint(\"📍 Checkpoints will be saved in:\", checkpoint_folder)\nprint(\"📍 Final full edited zip will be saved in:\", image_edit_results_folder)\nprint(\" (Note: /kaggle/input/... is read-only, so we use /kaggle/working/)\")\n\n# ================= CLEAN WORKSPACE =================\nprint(\"🧹 Clearing old folders...\")\nfor p in [output_folder0, output_folder1, checkpoint_folder, image_edit_results_folder]:\n if os.path.exists(p):\n shutil.rmtree(p)\nos.makedirs(output_folder0, exist_ok=True)\nos.makedirs(output_folder1, exist_ok=True)\nos.makedirs(checkpoint_folder, exist_ok=True)\nos.makedirs(image_edit_results_folder, exist_ok=True)\n\n# ================= LOAD & SPLIT DATASET =================\nprint(f\"📦 Loading encrypted dataset: {encrypted_input_dataset_path}\")\ninput_dataset = load_from_disk(encrypted_input_dataset_path)\nn = len(input_dataset)\nhalf = n // 2\nprint(f\"✅ Loaded {n} images → split: {half} on GPU0 | {n-half} on GPU1\")\n\n# ================= PRE-FLIGHT CHECKS =================\nprint(\"\\n🔍 PRE-FLIGHT CHECKS...\")\nprint(f\" pipe0 exists : {'pipe0' in globals()}\")\nprint(f\" pipe1 exists : {'pipe1' in globals()}\")\nif 'pipe0' in globals():\n print(f\" GPU0 VRAM: {torch.cuda.memory_allocated(0)/1e9:.2f} GB\")\nif 'pipe1' in globals():\n print(f\" GPU1 VRAM: {torch.cuda.memory_allocated(1)/1e9:.2f} GB\")\nprint(\"✅ Models ready\\n\")\n\n# ================= WORKER THREAD =================\ndef worker_thread(pipe, gpu_id: int, start_idx: int, num_images: int,\n input_dataset, kaggle_repo_path: str, password_txt_file: str,\n edit_prompt: str, resolution: str, max_image_dimension: int,\n save_checkpoint_every_n: bool, save_every_n: int,\n output_folder: str, checkpoint_folder: str, total_images: int):\n\n torch.cuda.set_device(gpu_id)\n gpu_name = f\"GPU{gpu_id}\"\n start_time = datetime.datetime.now().strftime('%H:%M:%S')\n\n print(f\"🚀 [{gpu_name}] Worker thread STARTED at {start_time} | Processing {num_images} images\")\n\n pipe.set_progress_bar_config(disable=True)\n \n # === Sequential CPU offload ===\n pipe.vae.enable_slicing()\n pipe.vae.enable_tiling()\n\n # === Dynamo + SDNQ compatibility ===\n torch._dynamo.config.suppress_errors = True\n torch._dynamo.config.verbose = False\n torch._dynamo.config.raise_on_ctx_manager_usage = False\n torch._dynamo.config.error_on_nested_fx_trace = False # FIX for FX tracing on dynamo-compiled SDNQ matmuls\n #---#\n os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\n # Extra stability for Kaggle + SDNQ + Flux\n torch._dynamo.config.cache_size_limit = 64\n torch._dynamo.config.automatic_dynamic_shapes = False\n torch.backends.cuda.matmul.allow_tf32 = True\n torch.backends.cudnn.allow_tf32 = True\n\n # === Crypto & utils ===\n password_path = os.path.join(kaggle_repo_path, password_txt_file)\n with open(password_path, \"r\", encoding=\"utf-8\") as f:\n encryption_password = f.read().strip()\n\n def derive_key(pw): return hashlib.sha256(pw.encode()).digest()\n SECRET_KEY = derive_key(encryption_password)\n box = SecretBox(SECRET_KEY)\n\n def decrypt_bytes(enc): return box.decrypt(enc)\n def encrypt_bytes(data):\n nonce = random(SecretBox.NONCE_SIZE)\n return box.encrypt(data, nonce)\n\n def bytes_to_pil(b): return Image.open(io.BytesIO(b)).convert(\"RGB\")\n def pil_to_bytes(img):\n buf = io.BytesIO()\n img.save(buf, format=\"JPEG\", quality=90, optimize=True)\n return buf.getvalue()\n\n res_map = {\n \"1024 x 1024 (Square)\": (1024, 1024),\n \"512 x 1024 (Portrait)\": (512, 1024),\n \"768 x 1024 (Slight Portrait)\": (768, 1024),\n \"1536 x 1024 (Landscape)\": (1536, 1024),\n \"2048 x 1024 (Wide Landscape)\": (2048, 1024),\n }\n target_width, target_height = res_map[resolution]\n reference_image_to_pair = Image.new(\"RGB\", (target_width, target_height), \"#181818\")\n\n dataset_half = input_dataset.select(range(start_idx, start_idx + num_images))\n\n def save_checkpoint(batch_files, checkpoint_id_local):\n if not save_checkpoint_every_n or not batch_files:\n return\n zip_path = f\"{checkpoint_folder}/checkpoint_{gpu_name}_{checkpoint_id_local}.zip\"\n tmp = f\"{checkpoint_folder}/tmp_{gpu_name}_{checkpoint_id_local}\"\n os.makedirs(tmp, exist_ok=True)\n for f in batch_files:\n shutil.copy(f, tmp)\n shutil.make_archive(zip_path.replace('.zip',''), 'zip', tmp)\n shutil.rmtree(tmp)\n print(f\"💾 [{gpu_name}] Checkpoint saved: {zip_path} (every {save_every_n} items)\")\n\n print(f\"🚀 [{gpu_name}] Starting processing loop...\")\n\n batch_outputs = []\n checkpoint_id_local = 0\n output_records = []\n\n for local_i, item in enumerate(dataset_half):\n global_i = start_idx + local_i\n \n # === Memory + Dynamo reset before every single image ===\n gc.collect()\n torch.cuda.empty_cache()\n torch.cuda.reset_peak_memory_stats(gpu_id)\n torch._dynamo.reset()\n\n print(f\" [{gpu_name}] [{global_i+1}/{total_images}] Loading & decrypting image...\")\n\n enc_bytes = item[\"image_encrypted\"]\n input_image = bytes_to_pil(decrypt_bytes(enc_bytes))\n\n if max(input_image.width, input_image.height) > max_image_dimension:\n aspect = input_image.width / input_image.height\n new_w = max_image_dimension if input_image.width > input_image.height else int(max_image_dimension * aspect)\n new_h = int(max_image_dimension / aspect) if input_image.width > input_image.height else max_image_dimension\n input_image = input_image.resize((new_w, new_h), Image.LANCZOS)\n\n reference_images = [input_image, reference_image_to_pair]\n\n inf_start = datetime.datetime.now().strftime('%H:%M:%S')\n vram_before = torch.cuda.memory_allocated(gpu_id) / 1e9\n print(f\" [{gpu_name}] [{global_i+1}/{total_images}] RUNNING INFERENCE at {inf_start}... (VRAM before: {vram_before:.2f} GB)\")\n\n try:\n with torch.cuda.device(gpu_id):\n with torch.no_grad():\n with torch.compiler.disable():\n result = pipe(\n prompt=edit_prompt,\n image=reference_images,\n height=target_height,\n width=target_width,\n guidance_scale=1.0,\n num_inference_steps=4,\n generator=torch.Generator(f\"cuda:{gpu_id}\").manual_seed(42),\n output_type=\"pil\",\n ).images[0]\n except Exception as e:\n print(f\" [{gpu_name}] ❌ INFERENCE ERROR: {e}\")\n traceback.print_exc()\n raise\n\n vram_after = torch.cuda.memory_allocated(gpu_id) / 1e9\n print(f\" [{gpu_name}] [{global_i+1}/{total_images}] INFERENCE COMPLETE ✅ (VRAM after: {vram_after:.2f} GB)\")\n\n out_bytes = pil_to_bytes(result)\n enc_out = encrypt_bytes(out_bytes)\n\n out_path = os.path.join(output_folder, f\"edited_{global_i}.enc\")\n with open(out_path, \"wb\") as f:\n f.write(enc_out)\n\n output_records.append({\"image_encrypted\": enc_out})\n batch_outputs.append(out_path)\n\n if save_checkpoint_every_n and len(batch_outputs) >= save_every_n:\n checkpoint_id_local += 1\n save_checkpoint(batch_outputs, checkpoint_id_local)\n batch_outputs = []\n\n # === Final per-image cleanup + small breathing room ===\n del result, input_image, reference_images\n gc.collect()\n torch.cuda.empty_cache()\n torch.cuda.reset_peak_memory_stats(gpu_id)\n time.sleep(0.3) # tiny pause helps Kaggle not kill the notebook\n\n if save_checkpoint_every_n and batch_outputs:\n checkpoint_id_local += 1\n save_checkpoint(batch_outputs, checkpoint_id_local)\n\n print(f\"✅ [{gpu_name}] FINISHED all {num_images} images\")\n return output_records\n\n# ================= LAUNCH =================\nprint(\"\\n🚀 Launching true parallel inference on GPU0 + GPU1...\")\nprint(\" → GPU0 starts immediately\")\nprint(\" → GPU1 starts after 60-seconds\")\n\nwith ThreadPoolExecutor(max_workers=2) as executor:\n future0 = executor.submit(\n worker_thread, pipe0, 0, 0, half, input_dataset,\n kaggle_repo_path, password_txt_file,\n edit_prompt, resolution, max_image_dimension,\n save_checkpoint_every_n, save_every_n,\n output_folder0, checkpoint_folder, n\n )\n\n print(\" ⏳ Waiting 60 seconds for GPU0 to fully warm up and offload...\")\n time.sleep(60)\n\n future1 = executor.submit(\n worker_thread, pipe1, 1, half, n - half, input_dataset,\n kaggle_repo_path, password_txt_file,\n edit_prompt, resolution, max_image_dimension,\n save_checkpoint_every_n, save_every_n,\n output_folder1, checkpoint_folder, n\n )\n\n all_output_records = []\n for future in as_completed([future0, future1]):\n all_output_records += future.result()\n\n# ================= COMBINE + CLEANUP =================\nprint(\"\\n📦 Combining outputs into final ZIP...\")\nfinal_output_folder = \"/kaggle/working/edited_images_final\"\nos.makedirs(final_output_folder, exist_ok=True)\n\nfor folder in [output_folder0, output_folder1]:\n if os.path.exists(folder):\n for fname in os.listdir(folder):\n shutil.copy2(os.path.join(folder, fname), os.path.join(final_output_folder, fname))\n\noutput_dataset = Dataset.from_list(all_output_records)\noutput_dataset.save_to_disk(\"/kaggle/working/encrypted_output_dataset\")\n\ntimestamp = datetime.datetime.now().strftime(\"%Y%m%d_%H%M%S\")\nfinal_zip = f\"/kaggle/working/klein_encrypted_outputs_{timestamp}.zip\"\nshutil.make_archive(final_zip.replace('.zip',''), 'zip', final_output_folder)\nprint(f\"📦 Final encrypted outputs ZIP created → {final_zip}\")\n\nshutil.copy2(final_zip, os.path.join(image_edit_results_folder, os.path.basename(final_zip)))\nprint(f\"✅ Full edited zip copied to {image_edit_results_folder}/\")\n\nprint(\"🧹 Deleting checkpoints folder as requested...\")\nif os.path.exists(checkpoint_folder):\n shutil.rmtree(checkpoint_folder)\n print(\"✅ checkpoints folder deleted\")\n\nfor folder in [output_folder0, output_folder1, final_output_folder]:\n if os.path.exists(folder):\n shutil.rmtree(folder)\n\nprint(\"\\n✅ BATCH COMPLETE!\")\nprint(f\" → Checkpoints were saved every {save_every_n} items (then deleted)\")\nprint(f\" → Final full zip is in: {image_edit_results_folder}\")\nprint(\"🎯 You can now add the image_edit_results folder to your Kaggle dataset version.\")\nprint(\" Session B finished – ready for Session C\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-26T19:59:02.416987Z","iopub.execute_input":"2026-04-26T19:59:02.417587Z","iopub.status.idle":"2026-04-26T20:53:07.487965Z","shell.execute_reply.started":"2026-04-26T19:59:02.417551Z","shell.execute_reply":"2026-04-26T20:53:07.486901Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[{"name":"stdout","text":"📍 Checkpoints will be saved in: /kaggle/working/checkpoints\n📍 Final full edited zip will be saved in: /kaggle/working/image_edit_results\n (Note: /kaggle/input/... is read-only, so we use /kaggle/working/)\n🧹 Clearing old folders...\n📦 Loading encrypted dataset: /kaggle/input/datasets/nekos4lyfe/image-caption-dataset/\n✅ Loaded 129 images → split: 64 on GPU0 | 65 on GPU1\n\n🔍 PRE-FLIGHT CHECKS...\n pipe0 exists : True\n pipe1 exists : True\n GPU0 VRAM: 4.42 GB\n GPU1 VRAM: 0.01 GB\n✅ Models ready\n\n\n🚀 Launching true parallel inference on GPU0 + GPU1...\n → GPU0 starts immediately\n → GPU1 starts after 60-seconds\n🚀 [GPU0] Worker thread STARTED at 19:59:02 | Processing 64 images\n ⏳ Waiting 60 seconds for GPU0 to fully warm up and offload...\n🚀 [GPU0] Starting processing loop...\n [GPU0] [1/129] Loading & decrypting image...\n [GPU0] [1/129] RUNNING INFERENCE at 19:59:03... (VRAM before: 4.42 GB)\n [GPU0] [1/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [2/129] Loading & decrypting image...\n [GPU0] [2/129] RUNNING INFERENCE at 19:59:49... (VRAM before: 4.42 GB)\n🚀 [GPU1] Worker thread STARTED at 20:00:02 | Processing 65 images\n🚀 [GPU1] Starting processing loop...\n [GPU1] [65/129] Loading & decrypting image...\n [GPU1] [65/129] RUNNING INFERENCE at 20:00:06... (VRAM before: 0.01 GB)\n [GPU0] [2/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [3/129] Loading & decrypting image...\n [GPU0] [3/129] RUNNING INFERENCE at 20:00:31... (VRAM before: 4.42 GB)\n [GPU1] [65/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [66/129] Loading & decrypting image...\n [GPU1] [66/129] RUNNING INFERENCE at 20:01:15... (VRAM before: 0.01 GB)\n [GPU0] [3/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [4/129] Loading & decrypting image...\n [GPU0] [4/129] RUNNING INFERENCE at 20:01:19... (VRAM before: 4.42 GB)\n [GPU1] [66/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [67/129] Loading & decrypting image...\n [GPU1] [67/129] RUNNING INFERENCE at 20:02:02... (VRAM before: 0.01 GB)\n [GPU0] [4/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [5/129] Loading & decrypting image...\n [GPU0] [5/129] RUNNING INFERENCE at 20:02:03... (VRAM before: 4.42 GB)\n [GPU1] [67/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [68/129] Loading & decrypting image...\n [GPU0] [5/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [68/129] RUNNING INFERENCE at 20:02:49... (VRAM before: 0.01 GB)\n [GPU0] [6/129] Loading & decrypting image...\n [GPU0] [6/129] RUNNING INFERENCE at 20:02:51... (VRAM before: 4.42 GB)\n [GPU1] [68/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [69/129] Loading & decrypting image...\n [GPU1] [69/129] RUNNING INFERENCE at 20:03:39... (VRAM before: 0.01 GB)\n [GPU0] [6/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [7/129] Loading & decrypting image...\n [GPU0] [7/129] RUNNING INFERENCE at 20:03:41... (VRAM before: 4.42 GB)\n [GPU1] [69/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [70/129] Loading & decrypting image...\n [GPU1] [70/129] RUNNING INFERENCE at 20:04:24... (VRAM before: 0.01 GB)\n [GPU0] [7/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [8/129] Loading & decrypting image...\n [GPU0] [8/129] RUNNING INFERENCE at 20:04:28... (VRAM before: 4.42 GB)\n [GPU1] [70/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [71/129] Loading & decrypting image...\n [GPU1] [71/129] RUNNING INFERENCE at 20:05:12... (VRAM before: 0.01 GB)\n [GPU0] [8/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [9/129] Loading & decrypting image...\n [GPU0] [9/129] RUNNING INFERENCE at 20:05:13... (VRAM before: 4.42 GB)\n [GPU1] [71/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [72/129] Loading & decrypting image...\n [GPU1] [72/129] RUNNING INFERENCE at 20:06:03... (VRAM before: 0.01 GB)\n [GPU0] [9/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [10/129] Loading & decrypting image...\n [GPU0] [10/129] RUNNING INFERENCE at 20:06:06... (VRAM before: 4.42 GB)\n [GPU1] [72/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [73/129] Loading & decrypting image...\n [GPU1] [73/129] RUNNING INFERENCE at 20:06:50... (VRAM before: 0.01 GB)\n [GPU0] [10/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_1.zip (every 10 items)\n [GPU0] [11/129] Loading & decrypting image...\n [GPU0] [11/129] RUNNING INFERENCE at 20:06:54... (VRAM before: 4.42 GB)\n [GPU0] [11/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [12/129] Loading & decrypting image...\n [GPU0] [12/129] RUNNING INFERENCE at 20:07:39... (VRAM before: 4.42 GB)\n [GPU1] [73/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [74/129] Loading & decrypting image...\n [GPU1] [74/129] RUNNING INFERENCE at 20:07:40... (VRAM before: 0.01 GB)\n [GPU0] [12/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [13/129] Loading & decrypting image...\n [GPU0] [13/129] RUNNING INFERENCE at 20:08:26... (VRAM before: 4.42 GB)\n [GPU1] [74/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_1.zip (every 10 items)\n [GPU1] [75/129] Loading & decrypting image...\n [GPU1] [75/129] RUNNING INFERENCE at 20:08:30... (VRAM before: 0.01 GB)\n [GPU0] [13/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [14/129] Loading & decrypting image...\n [GPU0] [14/129] RUNNING INFERENCE at 20:09:14... (VRAM before: 4.42 GB)\n [GPU1] [75/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [76/129] Loading & decrypting image...\n [GPU1] [76/129] RUNNING INFERENCE at 20:09:19... (VRAM before: 0.01 GB)\n [GPU0] [14/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [15/129] Loading & decrypting image...\n [GPU0] [15/129] RUNNING INFERENCE at 20:10:08... (VRAM before: 4.42 GB)\n [GPU1] [76/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [77/129] Loading & decrypting image...\n [GPU1] [77/129] RUNNING INFERENCE at 20:10:12... (VRAM before: 0.01 GB)\n [GPU0] [15/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [16/129] Loading & decrypting image...\n [GPU0] [16/129] RUNNING INFERENCE at 20:10:56... (VRAM before: 4.42 GB)\n [GPU1] [77/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [78/129] Loading & decrypting image...\n [GPU1] [78/129] RUNNING INFERENCE at 20:11:01... (VRAM before: 0.01 GB)\n [GPU0] [16/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [17/129] Loading & decrypting image...\n [GPU0] [17/129] RUNNING INFERENCE at 20:11:45... (VRAM before: 4.42 GB)\n [GPU1] [78/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [79/129] Loading & decrypting image...\n [GPU1] [79/129] RUNNING INFERENCE at 20:11:50... (VRAM before: 0.01 GB)\n [GPU0] [17/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [18/129] Loading & decrypting image...\n [GPU0] [18/129] RUNNING INFERENCE at 20:12:34... (VRAM before: 4.42 GB)\n [GPU1] [79/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [80/129] Loading & decrypting image...\n [GPU1] [80/129] RUNNING INFERENCE at 20:12:38... (VRAM before: 0.01 GB)\n [GPU0] [18/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [19/129] Loading & decrypting image...\n [GPU0] [19/129] RUNNING INFERENCE at 20:13:23... (VRAM before: 4.42 GB)\n [GPU1] [80/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [81/129] Loading & decrypting image...\n [GPU1] [81/129] RUNNING INFERENCE at 20:13:27... (VRAM before: 0.01 GB)\n [GPU0] [19/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [20/129] Loading & decrypting image...\n [GPU0] [20/129] RUNNING INFERENCE at 20:14:13... (VRAM before: 4.42 GB)\n [GPU1] [81/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [82/129] Loading & decrypting image...\n [GPU1] [82/129] RUNNING INFERENCE at 20:14:18... (VRAM before: 0.01 GB)\n [GPU0] [20/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_2.zip (every 10 items)\n [GPU0] [21/129] Loading & decrypting image...\n [GPU0] [21/129] RUNNING INFERENCE at 20:15:05... (VRAM before: 4.42 GB)\n [GPU1] [82/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [83/129] Loading & decrypting image...\n [GPU1] [83/129] RUNNING INFERENCE at 20:15:09... (VRAM before: 0.01 GB)\n [GPU0] [21/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [22/129] Loading & decrypting image...\n [GPU0] [22/129] RUNNING INFERENCE at 20:15:50... (VRAM before: 4.42 GB)\n [GPU1] [83/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [84/129] Loading & decrypting image...\n [GPU1] [84/129] RUNNING INFERENCE at 20:15:55... (VRAM before: 0.01 GB)\n [GPU0] [22/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [23/129] Loading & decrypting image...\n [GPU0] [23/129] RUNNING INFERENCE at 20:16:41... (VRAM before: 4.42 GB)\n [GPU1] [84/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_2.zip (every 10 items)\n [GPU1] [85/129] Loading & decrypting image...\n [GPU1] [85/129] RUNNING INFERENCE at 20:16:46... (VRAM before: 0.01 GB)\n [GPU0] [23/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [24/129] Loading & decrypting image...\n [GPU0] [24/129] RUNNING INFERENCE at 20:17:31... (VRAM before: 4.42 GB)\n [GPU1] [85/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [86/129] Loading & decrypting image...\n [GPU1] [86/129] RUNNING INFERENCE at 20:17:36... (VRAM before: 0.01 GB)\n [GPU0] [24/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [25/129] Loading & decrypting image...\n [GPU0] [25/129] RUNNING INFERENCE at 20:18:18... (VRAM before: 4.42 GB)\n [GPU1] [86/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [87/129] Loading & decrypting image...\n [GPU1] [87/129] RUNNING INFERENCE at 20:18:23... (VRAM before: 0.01 GB)\n [GPU0] [25/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [26/129] Loading & decrypting image...\n [GPU1] [87/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU0] [26/129] RUNNING INFERENCE at 20:19:07... (VRAM before: 4.42 GB)\n [GPU1] [88/129] Loading & decrypting image...\n [GPU1] [88/129] RUNNING INFERENCE at 20:19:09... (VRAM before: 0.01 GB)\n [GPU0] [26/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [27/129] Loading & decrypting image...\n [GPU0] [27/129] RUNNING INFERENCE at 20:19:56... (VRAM before: 4.42 GB)\n [GPU1] [88/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [89/129] Loading & decrypting image...\n [GPU1] [89/129] RUNNING INFERENCE at 20:20:00... (VRAM before: 0.01 GB)\n [GPU0] [27/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [28/129] Loading & decrypting image...\n [GPU0] [28/129] RUNNING INFERENCE at 20:20:45... (VRAM before: 4.42 GB)\n [GPU1] [89/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [90/129] Loading & decrypting image...\n [GPU1] [90/129] RUNNING INFERENCE at 20:20:49... (VRAM before: 0.01 GB)\n [GPU0] [28/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [29/129] Loading & decrypting image...\n [GPU0] [29/129] RUNNING INFERENCE at 20:21:32... (VRAM before: 4.42 GB)\n [GPU1] [90/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [91/129] Loading & decrypting image...\n [GPU1] [91/129] RUNNING INFERENCE at 20:21:37... (VRAM before: 0.01 GB)\n [GPU0] [29/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [30/129] Loading & decrypting image...\n [GPU0] [30/129] RUNNING INFERENCE at 20:22:22... (VRAM before: 4.42 GB)\n [GPU1] [91/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [92/129] Loading & decrypting image...\n [GPU1] [92/129] RUNNING INFERENCE at 20:22:27... (VRAM before: 0.01 GB)\n [GPU0] [30/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_3.zip (every 10 items)\n [GPU0] [31/129] Loading & decrypting image...\n [GPU0] [31/129] RUNNING INFERENCE at 20:23:11... (VRAM before: 4.42 GB)\n [GPU1] [92/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [93/129] Loading & decrypting image...\n [GPU1] [93/129] RUNNING INFERENCE at 20:23:16... (VRAM before: 0.01 GB)\n [GPU0] [31/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [32/129] Loading & decrypting image...\n [GPU0] [32/129] RUNNING INFERENCE at 20:24:00... (VRAM before: 4.42 GB)\n [GPU1] [93/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [94/129] Loading & decrypting image...\n [GPU1] [94/129] RUNNING INFERENCE at 20:24:05... (VRAM before: 0.01 GB)\n [GPU0] [32/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [33/129] Loading & decrypting image...\n [GPU0] [33/129] RUNNING INFERENCE at 20:24:45... (VRAM before: 4.42 GB)\n [GPU1] [94/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_3.zip (every 10 items)\n [GPU1] [95/129] Loading & decrypting image...\n [GPU1] [95/129] RUNNING INFERENCE at 20:24:50... (VRAM before: 0.01 GB)\n [GPU0] [33/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [34/129] Loading & decrypting image...\n [GPU0] [34/129] RUNNING INFERENCE at 20:25:34... (VRAM before: 4.42 GB)\n [GPU1] [95/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [96/129] Loading & decrypting image...\n [GPU1] [96/129] RUNNING INFERENCE at 20:25:39... (VRAM before: 0.01 GB)\n [GPU0] [34/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [35/129] Loading & decrypting image...\n [GPU0] [35/129] RUNNING INFERENCE at 20:26:22... (VRAM before: 4.42 GB)\n [GPU1] [96/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [97/129] Loading & decrypting image...\n [GPU1] [97/129] RUNNING INFERENCE at 20:26:24... (VRAM before: 0.01 GB)\n [GPU0] [35/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [36/129] Loading & decrypting image...\n [GPU0] [36/129] RUNNING INFERENCE at 20:27:11... (VRAM before: 4.42 GB)\n [GPU1] [97/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [98/129] Loading & decrypting image...\n [GPU1] [98/129] RUNNING INFERENCE at 20:27:16... (VRAM before: 0.01 GB)\n [GPU0] [36/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [37/129] Loading & decrypting image...\n [GPU0] [37/129] RUNNING INFERENCE at 20:27:58... (VRAM before: 4.42 GB)\n [GPU1] [98/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [99/129] Loading & decrypting image...\n [GPU1] [99/129] RUNNING INFERENCE at 20:28:01... (VRAM before: 0.01 GB)\n [GPU0] [37/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [38/129] Loading & decrypting image...\n [GPU0] [38/129] RUNNING INFERENCE at 20:28:47... (VRAM before: 4.42 GB)\n [GPU1] [99/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [100/129] Loading & decrypting image...\n [GPU1] [100/129] RUNNING INFERENCE at 20:28:52... (VRAM before: 0.01 GB)\n [GPU0] [38/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [39/129] Loading & decrypting image...\n [GPU0] [39/129] RUNNING INFERENCE at 20:29:37... (VRAM before: 4.42 GB)\n [GPU1] [100/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [101/129] Loading & decrypting image...\n [GPU1] [101/129] RUNNING INFERENCE at 20:29:42... (VRAM before: 0.01 GB)\n [GPU0] [39/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [40/129] Loading & decrypting image...\n [GPU0] [40/129] RUNNING INFERENCE at 20:30:25... (VRAM before: 4.42 GB)\n [GPU1] [101/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [102/129] Loading & decrypting image...\n [GPU1] [102/129] RUNNING INFERENCE at 20:30:30... (VRAM before: 0.01 GB)\n [GPU0] [40/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_4.zip (every 10 items)\n [GPU0] [41/129] Loading & decrypting image...\n [GPU0] [41/129] RUNNING INFERENCE at 20:31:16... (VRAM before: 4.42 GB)\n [GPU1] [102/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [103/129] Loading & decrypting image...\n [GPU1] [103/129] RUNNING INFERENCE at 20:31:20... (VRAM before: 0.01 GB)\n [GPU0] [41/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [42/129] Loading & decrypting image...\n [GPU0] [42/129] RUNNING INFERENCE at 20:32:02... (VRAM before: 4.42 GB)\n [GPU1] [103/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [104/129] Loading & decrypting image...\n [GPU1] [104/129] RUNNING INFERENCE at 20:32:07... (VRAM before: 0.01 GB)\n [GPU0] [42/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [43/129] Loading & decrypting image...\n [GPU0] [43/129] RUNNING INFERENCE at 20:32:53... (VRAM before: 4.42 GB)\n [GPU1] [104/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_4.zip (every 10 items)\n [GPU1] [105/129] Loading & decrypting image...\n [GPU1] [105/129] RUNNING INFERENCE at 20:32:58... (VRAM before: 0.01 GB)\n [GPU0] [43/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [44/129] Loading & decrypting image...\n [GPU0] [44/129] RUNNING INFERENCE at 20:33:40... (VRAM before: 4.42 GB)\n [GPU1] [105/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [106/129] Loading & decrypting image...\n [GPU1] [106/129] RUNNING INFERENCE at 20:33:45... (VRAM before: 0.01 GB)\n [GPU0] [44/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [45/129] Loading & decrypting image...\n [GPU0] [45/129] RUNNING INFERENCE at 20:34:30... (VRAM before: 4.42 GB)\n [GPU1] [106/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [107/129] Loading & decrypting image...\n [GPU1] [107/129] RUNNING INFERENCE at 20:34:36... (VRAM before: 0.01 GB)\n [GPU0] [45/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [46/129] Loading & decrypting image...\n [GPU0] [46/129] RUNNING INFERENCE at 20:35:21... (VRAM before: 4.42 GB)\n [GPU1] [107/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [108/129] Loading & decrypting image...\n [GPU1] [108/129] RUNNING INFERENCE at 20:35:26... (VRAM before: 0.01 GB)\n [GPU1] [108/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [109/129] Loading & decrypting image...\n [GPU1] [109/129] RUNNING INFERENCE at 20:36:11... (VRAM before: 0.01 GB)\n [GPU0] [46/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [47/129] Loading & decrypting image...\n [GPU0] [47/129] RUNNING INFERENCE at 20:36:13... (VRAM before: 4.42 GB)\n [GPU1] [109/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [110/129] Loading & decrypting image...\n [GPU1] [110/129] RUNNING INFERENCE at 20:36:59... (VRAM before: 0.01 GB)\n [GPU0] [47/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [48/129] Loading & decrypting image...\n [GPU0] [48/129] RUNNING INFERENCE at 20:37:04... (VRAM before: 4.42 GB)\n [GPU1] [110/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [111/129] Loading & decrypting image...\n [GPU1] [111/129] RUNNING INFERENCE at 20:37:48... (VRAM before: 0.01 GB)\n [GPU0] [48/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [49/129] Loading & decrypting image...\n [GPU0] [49/129] RUNNING INFERENCE at 20:37:54... (VRAM before: 4.42 GB)\n [GPU1] [111/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [112/129] Loading & decrypting image...\n [GPU1] [112/129] RUNNING INFERENCE at 20:38:40... (VRAM before: 0.01 GB)\n [GPU0] [49/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [50/129] Loading & decrypting image...\n [GPU0] [50/129] RUNNING INFERENCE at 20:38:42... (VRAM before: 4.42 GB)\n [GPU1] [112/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [113/129] Loading & decrypting image...\n [GPU1] [113/129] RUNNING INFERENCE at 20:39:28... (VRAM before: 0.01 GB)\n [GPU0] [50/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_5.zip (every 10 items)\n [GPU0] [51/129] Loading & decrypting image...\n [GPU0] [51/129] RUNNING INFERENCE at 20:39:31... (VRAM before: 4.42 GB)\n [GPU1] [113/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [114/129] Loading & decrypting image...\n [GPU0] [51/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [114/129] RUNNING INFERENCE at 20:40:16... (VRAM before: 0.01 GB)\n [GPU0] [52/129] Loading & decrypting image...\n [GPU0] [52/129] RUNNING INFERENCE at 20:40:19... (VRAM before: 4.42 GB)\n [GPU1] [114/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_5.zip (every 10 items)\n [GPU1] [115/129] Loading & decrypting image...\n [GPU1] [115/129] RUNNING INFERENCE at 20:41:04... (VRAM before: 0.01 GB)\n [GPU0] [52/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [53/129] Loading & decrypting image...\n [GPU0] [53/129] RUNNING INFERENCE at 20:41:07... (VRAM before: 4.42 GB)\n [GPU1] [115/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [116/129] Loading & decrypting image...\n [GPU1] [116/129] RUNNING INFERENCE at 20:41:53... (VRAM before: 0.01 GB)\n [GPU0] [53/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [54/129] Loading & decrypting image...\n [GPU0] [54/129] RUNNING INFERENCE at 20:41:56... (VRAM before: 4.42 GB)\n [GPU1] [116/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [117/129] Loading & decrypting image...\n [GPU1] [117/129] RUNNING INFERENCE at 20:42:41... (VRAM before: 0.01 GB)\n [GPU0] [54/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [55/129] Loading & decrypting image...\n [GPU0] [55/129] RUNNING INFERENCE at 20:42:44... (VRAM before: 4.42 GB)\n [GPU1] [117/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [118/129] Loading & decrypting image...\n [GPU1] [118/129] RUNNING INFERENCE at 20:43:31... (VRAM before: 0.01 GB)\n [GPU0] [55/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [56/129] Loading & decrypting image...\n [GPU0] [56/129] RUNNING INFERENCE at 20:43:34... (VRAM before: 4.42 GB)\n [GPU1] [118/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [119/129] Loading & decrypting image...\n [GPU1] [119/129] RUNNING INFERENCE at 20:44:19... (VRAM before: 0.01 GB)\n [GPU0] [56/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [57/129] Loading & decrypting image...\n [GPU0] [57/129] RUNNING INFERENCE at 20:44:24... (VRAM before: 4.42 GB)\n [GPU1] [119/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [120/129] Loading & decrypting image...\n [GPU0] [57/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [120/129] RUNNING INFERENCE at 20:45:11... (VRAM before: 0.01 GB)\n [GPU0] [58/129] Loading & decrypting image...\n [GPU0] [58/129] RUNNING INFERENCE at 20:45:12... (VRAM before: 4.42 GB)\n [GPU1] [120/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [121/129] Loading & decrypting image...\n [GPU1] [121/129] RUNNING INFERENCE at 20:45:58... (VRAM before: 0.01 GB)\n [GPU0] [58/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [59/129] Loading & decrypting image...\n [GPU0] [59/129] RUNNING INFERENCE at 20:46:01... (VRAM before: 4.42 GB)\n [GPU1] [121/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [122/129] Loading & decrypting image...\n [GPU0] [59/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [122/129] RUNNING INFERENCE at 20:46:47... (VRAM before: 0.01 GB)\n [GPU0] [60/129] Loading & decrypting image...\n [GPU0] [60/129] RUNNING INFERENCE at 20:46:49... (VRAM before: 4.42 GB)\n [GPU1] [122/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [123/129] Loading & decrypting image...\n [GPU0] [60/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [123/129] RUNNING INFERENCE at 20:47:34... (VRAM before: 0.01 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_6.zip (every 10 items)\n [GPU0] [61/129] Loading & decrypting image...\n [GPU0] [61/129] RUNNING INFERENCE at 20:47:37... (VRAM before: 4.42 GB)\n [GPU1] [123/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU0] [61/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU1] [124/129] Loading & decrypting image...\n [GPU1] [124/129] RUNNING INFERENCE at 20:48:22... (VRAM before: 0.01 GB)\n [GPU0] [62/129] Loading & decrypting image...\n [GPU0] [62/129] RUNNING INFERENCE at 20:48:24... (VRAM before: 4.42 GB)\n [GPU0] [62/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [63/129] Loading & decrypting image...\n [GPU0] [63/129] RUNNING INFERENCE at 20:49:08... (VRAM before: 4.42 GB)\n [GPU1] [124/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_6.zip (every 10 items)\n [GPU1] [125/129] Loading & decrypting image...\n [GPU1] [125/129] RUNNING INFERENCE at 20:49:13... (VRAM before: 0.01 GB)\n [GPU0] [63/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n [GPU0] [64/129] Loading & decrypting image...\n [GPU0] [64/129] RUNNING INFERENCE at 20:49:58... (VRAM before: 4.42 GB)\n [GPU1] [125/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [126/129] Loading & decrypting image...\n [GPU1] [126/129] RUNNING INFERENCE at 20:50:04... (VRAM before: 0.01 GB)\n [GPU0] [64/129] INFERENCE COMPLETE ✅ (VRAM after: 4.42 GB)\n💾 [GPU0] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU0_7.zip (every 10 items)\n✅ [GPU0] FINISHED all 64 images\n [GPU1] [126/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [127/129] Loading & decrypting image...\n [GPU1] [127/129] RUNNING INFERENCE at 20:50:57... (VRAM before: 0.01 GB)\n [GPU1] [127/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [128/129] Loading & decrypting image...\n [GPU1] [128/129] RUNNING INFERENCE at 20:51:39... (VRAM before: 0.01 GB)\n [GPU1] [128/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n [GPU1] [129/129] Loading & decrypting image...\n [GPU1] [129/129] RUNNING INFERENCE at 20:52:22... (VRAM before: 0.01 GB)\n [GPU1] [129/129] INFERENCE COMPLETE ✅ (VRAM after: 0.01 GB)\n💾 [GPU1] Checkpoint saved: /kaggle/working/checkpoints/checkpoint_GPU1_7.zip (every 10 items)\n✅ [GPU1] FINISHED all 65 images\n\n📦 Combining outputs into final ZIP...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Saving the dataset (0/1 shards): 0%| | 0/129 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5efbc30889b14c7da6aeaf97e2d92276"}},"metadata":{}},{"name":"stdout","text":"📦 Final encrypted outputs ZIP created → /kaggle/working/klein_encrypted_outputs_20260426_205306.zip\n✅ Full edited zip copied to /kaggle/working/image_edit_results/\n🧹 Deleting checkpoints folder as requested...\n✅ checkpoints folder deleted\n\n✅ BATCH COMPLETE!\n → Checkpoints were saved every 10 items (then deleted)\n → Final full zip is in: /kaggle/working/image_edit_results\n🎯 You can now add the image_edit_results folder to your Kaggle dataset version.\n Session B finished – ready for Session C\n","output_type":"stream"}],"execution_count":18},{"cell_type":"code","source":"# =============================================================================\n#@markdown # **CELL 5B**: Emergency VRAM Cleanup (run if you interrupt Cell 4B)\n# =============================================================================\nimport torch\nimport gc\nimport os\nimport shutil\n\n\nprint(\"🧹 Starting full VRAM + workspace cleanup...\")\n\n# === Unload pipelines (if they exist) ===\nfor i, pipe_var in enumerate([\"pipe0\", \"pipe1\"]):\n if pipe_var in globals():\n try:\n pipe = globals()[pipe_var]\n del pipe\n print(f\" ✅ Unloaded {pipe_var}\")\n except:\n pass\n\n# === Force CUDA cache clear on BOTH GPUs ===\nfor gpu_id in [0, 1]:\n if torch.cuda.is_available():\n torch.cuda.set_device(gpu_id)\n torch.cuda.empty_cache()\n torch.cuda.reset_peak_memory_stats(gpu_id)\n print(f\" ✅ GPU{gpu_id} cache cleared\")\n\n# === Garbage collect ===\ngc.collect()\ntorch.cuda.empty_cache()\n\n# === Final VRAM report ===\nprint(\"\\n📊 Final VRAM status:\")\nfor gpu_id in [0, 1]:\n if torch.cuda.is_available():\n allocated = torch.cuda.memory_allocated(gpu_id) / 1e9\n reserved = torch.cuda.memory_reserved(gpu_id) / 1e9\n print(f\" GPU{gpu_id}: {allocated:.2f} GB allocated | {reserved:.2f} GB reserved\")\n else:\n print(f\" GPU{gpu_id}: Not available\")\n\nprint(\"\\n✅ FULL CLEANUP COMPLETE – You can now re-run Cell 3B + 4B safely!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-26T20:53:07.494829Z","iopub.execute_input":"2026-04-26T20:53:07.495376Z","iopub.status.idle":"2026-04-26T20:53:09.020731Z","shell.execute_reply.started":"2026-04-26T20:53:07.495343Z","shell.execute_reply":"2026-04-26T20:53:09.019981Z"}},"outputs":[{"name":"stdout","text":"🧹 Starting full VRAM + workspace cleanup...\n ✅ Unloaded pipe0\n ✅ Unloaded pipe1\n ✅ GPU0 cache cleared\n ✅ GPU1 cache cleared\n\n📊 Final VRAM status:\n GPU0: 4.42 GB allocated | 5.97 GB reserved\n GPU1: 0.01 GB allocated | 0.03 GB reserved\n\n✅ FULL CLEANUP COMPLETE – You can now re-run Cell 3B + 4B safely!\n","output_type":"stream"}],"execution_count":19}]}