Zwounds/Boolean_Search_Query_Model
0
1"""22025.3.1532025.3.1744.50.0.dev050.15.26__UNSLOTH_VERSIONING__7"""8from torch import Tensor9import torch10import torch.nn as nn11from torch.nn import functional as F12from trl.trainer.ppo_trainer import (Accelerator, BaseImageProcessor, CallbackHandler, DEFAULT_CALLBACKS, DEFAULT_PROGRESS_CALLBACK, DataCollatorWithPadding, DataLoader, Dataset, ExportableState, FeatureExtractionMixin, GenerationConfig, INVALID_LOGPROB, OnlineTrainerState, Optional, PPOConfig, PPOTrainer, PeftConfig, PeftModel, PolicyAndValueWrapper, PreTrainedTokenizerBase, PrinterCallback, ProcessorMixin, Trainer, TrainerCallback, TrainerControl, Union, batch_generation, broadcast, contextmanager, create_reference_model, defaultdict, disable_dropout_in_model, exact_div, first_true_indices, forward, gather_object, gc, generate_model_card, get_comet_experiment_url, get_peft_model, get_reporting_integration_callbacks, get_reward, is_peft_available, is_wandb_available, log_table_to_comet_experiment, masked_mean, masked_whiten, math, nn, np, nullcontext, os, pd, peft_module_casting_to_bf16, prepare_deepspeed, print_rich_table, textwrap, time, torch, truncate_response, unwrap_model_for_generation)13 14 15import os16from typing import *17from dataclasses import dataclass, field18from packaging.version import Version19import torch20import numpy as np21from contextlib import nullcontext22from torch.nn import functional as F23from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling24 25torch_compile_options = {26 "epilogue_fusion" : True,27 "max_autotune" : False,28 "shape_padding" : True,29 "trace.enabled" : False,30 "triton.cudagraphs" : False,31}32 33@torch.compile(dynamic = True, fullgraph = True, options = torch_compile_options,)34def selective_log_softmax(logits, index):35 logits = logits.to(torch.float32)36 selected_logits = torch.gather(logits, dim = -1, index = index.unsqueeze(-1)).squeeze(-1)37 # loop to reduce peak mem consumption38 # logsumexp_values = torch.stack([torch.logsumexp(lg, dim=-1) for lg in logits])39 logsumexp_values = torch.logsumexp(logits, dim = -1)40 per_token_logps = selected_logits - logsumexp_values # log_softmax(x_i) = x_i - logsumexp(x)41 return per_token_logps42@dataclass43class UnslothPPOConfig(PPOConfig):44 """45 46 Configuration class for the [`PPOTrainer`].47 48 Using [`~transformers.HfArgumentParser`] we can turn this class into49 [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the50 command line.51 52 Parameters:53 exp_name (`str`, *optional*, defaults to `os.path.basename(__file__)[:-3]`):54 Name of this experiment.55 reward_model_path (`str`, *optional*, defaults to `"EleutherAI/pythia-160m"`):56 Path to the reward model.57 model_adapter_name (`str` or `None`, *optional*, defaults to `None`):58 Name of the train target PEFT adapter, when using LoRA with multiple adapters.59 ref_adapter_name (`str` or `None`, *optional*, defaults to `None`):60 Name of the reference PEFT adapter, when using LoRA with multiple adapters.61 num_ppo_epochs (`int`, *optional*, defaults to `4`):62 Number of epochs to train.63 whiten_rewards (`bool`, *optional*, defaults to `False`):64 Whether to whiten the rewards.65 kl_coef (`float`, *optional*, defaults to `0.05`):66 KL coefficient.67 cliprange (`float`, *optional*, defaults to `0.2`):68 Clip range.69 vf_coef (`float`, *optional*, defaults to `0.1`):70 Value function coefficient.71 cliprange_value (`float`, *optional*, defaults to `0.2`):72 Clip range for the value function.73 gamma (`float`, *optional*, defaults to `1.0`):74 Discount factor.75 lam (`float`, *optional*, defaults to `0.95`):76 Lambda value for GAE.77 ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):78 This setting applies to DeepSpeed ZeRO-3. If enabled, the policy model weights are gathered for generation,79 improving generation speed. However, disabling this option allows training models that exceed the VRAM80 capacity of a single GPU, albeit at the cost of slower generation.81 82 """83 vllm_sampling_params: Optional[Any] = field(84 default = None,85 metadata = {'help': 'vLLM SamplingParams'},86 )87 unsloth_num_chunks : Optional[int] = field(88 default = -1,89 metadata = {'help': 'Chunk size to reduce memory usage. -1 is most efficient.'},90 )91 def __init__(92 self,93 output_dir = None,94 overwrite_output_dir = None,95 do_train = False,96 do_eval = False,97 do_predict = False,98 eval_strategy = 'no',99 prediction_loss_only = False,100 per_device_train_batch_size = 4,101 per_device_eval_batch_size = 4,102 per_gpu_train_batch_size = None,103 per_gpu_eval_batch_size = None,104 gradient_accumulation_steps = 2,105 eval_accumulation_steps = 2,106 eval_delay = 0,107 torch_empty_cache_steps = 250,108 learning_rate = 5e-05,109 weight_decay = 0.01,110 adam_beta1 = 0.9,111 adam_beta2 = 0.999,112 adam_epsilon = 1e-08,113 max_grad_norm = 1.0,114 num_train_epochs = 3.0,115 max_steps = -1,116 lr_scheduler_type = 'linear',117 warmup_ratio = 0.1,118 warmup_steps = 0,119 log_level = 'passive',120 log_level_replica = 'warning',121 log_on_each_node = True,122 logging_dir = None,123 logging_strategy = 'steps',124 logging_first_step = False,125 logging_steps = 1,126 logging_nan_inf_filter = False,127 save_strategy = 'steps',128 save_steps = 500,129 save_total_limit = None,130 save_safetensors = True,131 save_on_each_node = False,132 save_only_model = False,133 restore_callback_states_from_checkpoint = False,134 no_cuda = False,135 use_cpu = False,136 use_mps_device = False,137 seed = 3407,138 data_seed = 3407,139 jit_mode_eval = False,140 use_ipex = False,141 bf16 = False,142 fp16 = False,143 fp16_opt_level = 'O1',144 half_precision_backend = 'auto',145 bf16_full_eval = False,146 fp16_full_eval = False,147 tf32 = None,148 local_rank = -1,149 ddp_backend = None,150 tpu_num_cores = None,151 tpu_metrics_debug = False,152 debug = '',153 dataloader_drop_last = False,154 eval_steps = None,155 dataloader_num_workers = 0,156 dataloader_prefetch_factor = None,157 past_index = -1,158 run_name = None,159 disable_tqdm = None,160 remove_unused_columns = True,161 label_names = None,162 load_best_model_at_end = False,163 metric_for_best_model = None,164 greater_is_better = None,165 ignore_data_skip = False,166 fsdp = '',167 fsdp_min_num_params = 0,168 fsdp_config = None,169 tp_size = 0,170 fsdp_transformer_layer_cls_to_wrap = None,171 accelerator_config = None,172 deepspeed = None,173 label_smoothing_factor = 0.0,174 optim = 'adamw_8bit',175 optim_args = None,176 adafactor = False,177 group_by_length = False,178 length_column_name = 'length',179 report_to = None,180 ddp_find_unused_parameters = None,181 ddp_bucket_cap_mb = None,182 ddp_broadcast_buffers = None,183 dataloader_pin_memory = True,184 dataloader_persistent_workers = False,185 skip_memory_metrics = True,186 use_legacy_prediction_loop = False,187 push_to_hub = False,188 resume_from_checkpoint = None,189 hub_model_id = None,190 hub_strategy = 'every_save',191 hub_token = None,192 hub_private_repo = None,193 hub_always_push = False,194 gradient_checkpointing = False,195 gradient_checkpointing_kwargs = None,196 include_inputs_for_metrics = False,197 eval_do_concat_batches = True,198 fp16_backend = 'auto',199 evaluation_strategy = None,200 push_to_hub_model_id = None,201 push_to_hub_organization = None,202 push_to_hub_token = None,203 mp_parameters = '',204 auto_find_batch_size = False,205 full_determinism = False,206 torchdynamo = None,207 ray_scope = 'last',208 ddp_timeout = 1800,209 torch_compile = False,210 torch_compile_backend = None,211 torch_compile_mode = None,212 dispatch_batches = None,213 split_batches = None,214 include_tokens_per_second = False,215 include_num_input_tokens_seen = False,216 neftune_noise_alpha = None,217 optim_target_modules = None,218 batch_eval_metrics = False,219 eval_on_start = False,220 use_liger_kernel = False,221 eval_use_gather_object = False,222 average_tokens_across_devices = False,223 dataset_num_proc = None,224 num_mini_batches = 1,225 total_episodes = None,226 local_rollout_forward_batch_size = 64,227 num_sample_generations = 10,228 response_length = 53,229 stop_token = None,230 stop_token_id = None,231 temperature = 0.7,232 missing_eos_penalty = None,233 sft_model_path = 'EleutherAI/pythia-160m',234 world_size = None,235 num_total_batches = None,236 micro_batch_size = None,237 local_batch_size = None,238 batch_size = None,239 local_mini_batch_size = None,240 mini_batch_size = None,241 exp_name = 'ppo_config',242 reward_model_path = 'EleutherAI/pythia-160m',243 model_adapter_name = None,244 ref_adapter_name = None,245 num_ppo_epochs = 4,246 whiten_rewards = False,247 kl_coef = 0.05,248 cliprange = 0.2,249 vf_coef = 0.1,250 cliprange_value = 0.2,251 gamma = 1.0,252 lam = 0.95,253 ds3_gather_for_generation = True,254 vllm_sampling_params = None,255 unsloth_num_chunks = -1,256 **kwargs,257 ):258 if learning_rate < 1e-7: raise FloatingPointError(f'Unsloth: Your learning rate of `{learning_rate}` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!')259 if learning_rate > 1: raise OverflowError(f'Unsloth: Your learning rate of `{learning_rate}` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!')260 if output_dir is None and save_strategy == 'steps' and save_steps == 500:261 output_dir = 'unsloth_training_checkpoints'262 save_strategy = 'no'263 if dataset_num_proc is None:264 from multiprocessing import cpu_count265 dataset_num_proc = cpu_count()266 267 super().__init__(268 output_dir = output_dir,269 overwrite_output_dir = overwrite_output_dir,270 do_train = do_train,271 do_eval = do_eval,272 do_predict = do_predict,273 eval_strategy = eval_strategy,274 prediction_loss_only = prediction_loss_only,275 per_device_train_batch_size = per_device_train_batch_size,276 per_device_eval_batch_size = per_device_eval_batch_size,277 per_gpu_train_batch_size = per_gpu_train_batch_size,278 per_gpu_eval_batch_size = per_gpu_eval_batch_size,279 gradient_accumulation_steps = gradient_accumulation_steps,280 eval_accumulation_steps = eval_accumulation_steps,281 eval_delay = eval_delay,282 torch_empty_cache_steps = torch_empty_cache_steps,283 learning_rate = learning_rate,284 weight_decay = weight_decay,285 adam_beta1 = adam_beta1,286 adam_beta2 = adam_beta2,287 adam_epsilon = adam_epsilon,288 max_grad_norm = max_grad_norm,289 num_train_epochs = num_train_epochs,290 max_steps = max_steps,291 lr_scheduler_type = lr_scheduler_type,292 warmup_ratio = warmup_ratio,293 warmup_steps = warmup_steps,294 log_level = log_level,295 log_level_replica = log_level_replica,296 log_on_each_node = log_on_each_node,297 logging_dir = logging_dir,298 logging_strategy = logging_strategy,299 logging_first_step = logging_first_step,300 logging_steps = logging_steps,301 logging_nan_inf_filter = logging_nan_inf_filter,302 save_strategy = save_strategy,303 save_steps = save_steps,304 save_total_limit = save_total_limit,305 save_safetensors = save_safetensors,306 save_on_each_node = save_on_each_node,307 save_only_model = save_only_model,308 restore_callback_states_from_checkpoint = restore_callback_states_from_checkpoint,309 no_cuda = no_cuda,310 use_cpu = use_cpu,311 use_mps_device = use_mps_device,312 seed = seed,313 data_seed = data_seed,314 jit_mode_eval = jit_mode_eval,315 use_ipex = use_ipex,316 bf16 = bf16,317 fp16 = fp16,318 fp16_opt_level = fp16_opt_level,319 half_precision_backend = half_precision_backend,320 bf16_full_eval = bf16_full_eval,321 fp16_full_eval = fp16_full_eval,322 tf32 = tf32,323 local_rank = local_rank,324 ddp_backend = ddp_backend,325 tpu_num_cores = tpu_num_cores,326 tpu_metrics_debug = tpu_metrics_debug,327 debug = debug,328 dataloader_drop_last = dataloader_drop_last,329 eval_steps = eval_steps,330 dataloader_num_workers = dataloader_num_workers,331 dataloader_prefetch_factor = dataloader_prefetch_factor,332 past_index = past_index,333 run_name = run_name,334 disable_tqdm = disable_tqdm,335 remove_unused_columns = remove_unused_columns,336 label_names = label_names,337 load_best_model_at_end = load_best_model_at_end,338 metric_for_best_model = metric_for_best_model,339 greater_is_better = greater_is_better,340 ignore_data_skip = ignore_data_skip,341 fsdp = fsdp,342 fsdp_min_num_params = fsdp_min_num_params,343 fsdp_config = fsdp_config,344 tp_size = tp_size,345 fsdp_transformer_layer_cls_to_wrap = fsdp_transformer_layer_cls_to_wrap,346 accelerator_config = accelerator_config,347 deepspeed = deepspeed,348 label_smoothing_factor = label_smoothing_factor,349 optim = optim,350 optim_args = optim_args,351 adafactor = adafactor,352 group_by_length = group_by_length,353 length_column_name = length_column_name,354 report_to = report_to,355 ddp_find_unused_parameters = ddp_find_unused_parameters,356 ddp_bucket_cap_mb = ddp_bucket_cap_mb,357 ddp_broadcast_buffers = ddp_broadcast_buffers,358 dataloader_pin_memory = dataloader_pin_memory,359 dataloader_persistent_workers = dataloader_persistent_workers,360 skip_memory_metrics = skip_memory_metrics,361 use_legacy_prediction_loop = use_legacy_prediction_loop,362 push_to_hub = push_to_hub,363 resume_from_checkpoint = resume_from_checkpoint,364 hub_model_id = hub_model_id,365 hub_strategy = hub_strategy,366 hub_token = hub_token,367 hub_private_repo = hub_private_repo,368 hub_always_push = hub_always_push,369 gradient_checkpointing = gradient_checkpointing,370 gradient_checkpointing_kwargs = gradient_checkpointing_kwargs,371 include_inputs_for_metrics = include_inputs_for_metrics,372 eval_do_concat_batches = eval_do_concat_batches,373 fp16_backend = fp16_backend,374 evaluation_strategy = evaluation_strategy,375 push_to_hub_model_id = push_to_hub_model_id,376 push_to_hub_organization = push_to_hub_organization,377 push_to_hub_token = push_to_hub_token,378 mp_parameters = mp_parameters,379 auto_find_batch_size = auto_find_batch_size,380 full_determinism = full_determinism,381 torchdynamo = torchdynamo,382 ray_scope = ray_scope,383 ddp_timeout = ddp_timeout,384 torch_compile = torch_compile,385 torch_compile_backend = torch_compile_backend,386 torch_compile_mode = torch_compile_mode,387 dispatch_batches = dispatch_batches,388 split_batches = split_batches,389 include_tokens_per_second = include_tokens_per_second,390 include_num_input_tokens_seen = include_num_input_tokens_seen,391 neftune_noise_alpha = neftune_noise_alpha,392 optim_target_modules = optim_target_modules,393 batch_eval_metrics = batch_eval_metrics,394 eval_on_start = eval_on_start,395 use_liger_kernel = use_liger_kernel,396 eval_use_gather_object = eval_use_gather_object,397 average_tokens_across_devices = average_tokens_across_devices,398 dataset_num_proc = dataset_num_proc,399 num_mini_batches = num_mini_batches,400 total_episodes = total_episodes,401 local_rollout_forward_batch_size = local_rollout_forward_batch_size,402 num_sample_generations = num_sample_generations,403 response_length = response_length,404 stop_token = stop_token,405 stop_token_id = stop_token_id,406 temperature = temperature,407 missing_eos_penalty = missing_eos_penalty,408 sft_model_path = sft_model_path,409 world_size = world_size,410 num_total_batches = num_total_batches,411 micro_batch_size = micro_batch_size,412 local_batch_size = local_batch_size,413 batch_size = batch_size,414 local_mini_batch_size = local_mini_batch_size,415 mini_batch_size = mini_batch_size,416 exp_name = exp_name,417 reward_model_path = reward_model_path,418 model_adapter_name = model_adapter_name,419 ref_adapter_name = ref_adapter_name,420 num_ppo_epochs = num_ppo_epochs,421 whiten_rewards = whiten_rewards,422 kl_coef = kl_coef,423 cliprange = cliprange,424 vf_coef = vf_coef,425 cliprange_value = cliprange_value,426 gamma = gamma,427 lam = lam,428 ds3_gather_for_generation = ds3_gather_for_generation,**kwargs)429 self.vllm_sampling_params = vllm_sampling_params430 self.unsloth_num_chunks = unsloth_num_chunks431pass432 433class _UnslothPPOTrainer(Trainer):434 _tag_names = ["trl", "ppo"]435 436 def __init__(437 self,438 args: PPOConfig,439 processing_class: Optional[440 Union[PreTrainedTokenizerBase, BaseImageProcessor, FeatureExtractionMixin, ProcessorMixin]441 ],442 model: nn.Module,443 ref_model: Optional[nn.Module],444 reward_model: nn.Module,445 train_dataset: Dataset,446 value_model: Optional[nn.Module] = None,447 data_collator: Optional[DataCollatorWithPadding] = None,448 eval_dataset: Optional[Union[Dataset, dict[str, Dataset]]] = None,449 # less commonly used450 optimizers: tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None),451 callbacks: Optional[list[TrainerCallback]] = None,452 peft_config: Optional["PeftConfig"] = None,453 ) -> None:454 if ref_model is model:455 raise ValueError(456 "`model` and `ref_model` cannot be the same object. If you want `ref_model` to be the "457 "same as `model`, you must make a copy of it, or `None` if you use peft."458 )459 460 self.args = args461 self.processing_class = processing_class462 self.policy_model = model463 464 # Define the collator if not provided465 if data_collator is None:466 data_collator = DataCollatorWithPadding(self.processing_class)467 468 # Handle stop token settings: update policy model's generation_config to use provided stop token469 if args.stop_token and args.stop_token_id:470 raise ValueError("You cannot set both `stop_token` and `stop_token_id`.")471 elif args.stop_token:472 if args.stop_token == "eos":473 self.policy_model.generation_config.eos_token_id = self.stop_token_id = processing_class.eos_token_id474 else:475 raise ValueError(476 f"Unknown `stop_token` {args.stop_token}. Allowed values are: `'eos'` and `None` (no stop token)."477 )478 else:479 self.policy_model.generation_config.eos_token_id = self.stop_token_id = args.stop_token_id # None or int480 481 # peft support482 if not is_peft_available() and peft_config is not None:483 raise ImportError(484 "PEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT models"485 )486 elif is_peft_available() and peft_config is not None:487 # if model is a peft model and we have a peft_confg, we merge and unload it first488 if isinstance(self.policy_model, PeftModel):489 self.policy_model = self.policy_model.merge_and_unload()490 491 # get peft model with the given config492 self.policy_model = get_peft_model(self.policy_model, peft_config)493 if args.bf16 and getattr(self.policy_model, "is_loaded_in_4bit", False):494 peft_module_casting_to_bf16(self.policy_model)495 496 self.is_peft_model = is_peft_available() and isinstance(self.policy_model, PeftModel)497 self.model_adapter_name = args.model_adapter_name498 self.ref_adapter_name = args.ref_adapter_name499 500 if ref_model:501 self.ref_model = ref_model502 elif self.is_peft_model:503 self.ref_model = None504 else:505 self.ref_model = create_reference_model(self.policy_model)506 507 self.reward_model = reward_model508 self.train_dataset = train_dataset509 self.train_dataset_len = len(train_dataset)510 self.value_model = value_model511 self.data_collator = data_collator512 self.eval_dataset = eval_dataset513 self.optimizer, self.lr_scheduler = optimizers514 self.optimizer_cls_and_kwargs = None # needed for transformers >= 4.47515 516 #########517 # calculate various batch sizes518 #########519 if args.total_episodes is None: # allow the users to define episodes in terms of epochs.520 args.total_episodes = int(args.num_train_epochs * self.train_dataset_len)521 accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps)522 self.accelerator = accelerator523 args.world_size = accelerator.num_processes524 args.local_batch_size = (525 args.per_device_train_batch_size * args.gradient_accumulation_steps * args.num_mini_batches526 )527 args.micro_batch_size = int(args.per_device_train_batch_size * args.world_size)528 args.batch_size = int(args.local_batch_size * args.world_size)529 args.mini_batch_size = exact_div(530 args.batch_size, args.num_mini_batches, "`batch_size` must be a multiple of `num_mini_batches`"531 )532 args.local_mini_batch_size = exact_div(533 args.local_batch_size, args.num_mini_batches, "`local_batch_size` must be a multiple of `num_mini_batches`"534 )535 if args.whiten_rewards:536 assert (537 args.local_mini_batch_size >= 8538 ), f"Per-rank minibatch size {args.local_mini_batch_size} is insufficient for whitening"539 # `per_rank_rollout_batch_size` is our `args.local_batch_size`540 # `per_rank_minibatch_size` is our `args.local_mini_batch_size`541 args.num_total_batches = math.ceil(542 args.total_episodes / args.batch_size543 ) # we may train for more than `total_episodes`544 time_tensor = torch.tensor(int(time.time()), device=accelerator.device)545 time_int = broadcast(time_tensor, 0).item() # avoid different timestamps across processes546 args.run_name = f"{args.exp_name}__{args.seed}__{time_int}"547 self.local_seed = args.seed + accelerator.process_index * 100003 # Prime548 if args.num_sample_generations > 0:549 self.sample_generations_freq = max(1, args.num_total_batches // args.num_sample_generations)550 self.local_dataloader_batch_size = args.local_batch_size551 552 #########553 # setup model, optimizer, and others554 #########555 for module in [self.policy_model, self.ref_model, self.value_model, self.reward_model]:556 if module is not None:557 disable_dropout_in_model(module)558 self.model = PolicyAndValueWrapper(self.policy_model, self.value_model)559 self.model.config = self.policy_model.config # needed for pushing to hub560 self.create_optimizer_and_scheduler(561 num_training_steps=args.num_total_batches562 ) # note that we are calling `self.lr_scheduler.step()` manually only at the batch level563 564 #########565 ### trainer specifics566 #########567 default_callbacks = DEFAULT_CALLBACKS + get_reporting_integration_callbacks(self.args.report_to)568 self.callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks569 self.callback_handler = CallbackHandler(570 self.callbacks, self.model, self.processing_class, self.optimizer, self.lr_scheduler571 )572 self.add_callback(PrinterCallback if self.args.disable_tqdm else DEFAULT_PROGRESS_CALLBACK)573 self.control = TrainerControl()574 self.state = OnlineTrainerState(575 is_local_process_zero=self.is_local_process_zero(),576 is_world_process_zero=self.is_world_process_zero(),577 stateful_callbacks=[578 cb for cb in self.callback_handler.callbacks + [self.control] if isinstance(cb, ExportableState)579 ],580 )581 self.current_flos = 0582 self.hp_search_backend = None583 self.is_deepspeed_enabled = getattr(self.accelerator.state, "deepspeed_plugin", None) is not None584 self.is_fsdp_enabled = getattr(self.accelerator.state, "fsdp_plugin", None) is not None585 # Create distant repo and output directory if needed586 self.hub_model_id = None587 if self.args.push_to_hub:588 self.init_hf_repo()589 if self.args.should_save:590 os.makedirs(self.args.output_dir, exist_ok=True)591 592 # Add tags for models that have been loaded with the correct transformers version593 if hasattr(self.model, "add_model_tags"):594 self.model.add_model_tags(self._tag_names)595 596 #########597 ### setup dataloader598 #########599 self.dataloader = DataLoader(600 self.train_dataset,601 batch_size=self.local_dataloader_batch_size,602 shuffle=True,603 collate_fn=self.data_collator,604 drop_last=True, # needed; otherwise the last batch will be of ragged shape605 )606 # sync random states for DataLoader(shuffle=True) before `accelerator.prepare`607 # see https://gist.github.com/vwxyzjn/2581bff1e48e185e0b85b6dfe1def79c608 torch.manual_seed(args.seed)609 self.model, self.optimizer, self.dataloader = accelerator.prepare(self.model, self.optimizer, self.dataloader)610 torch.manual_seed(self.local_seed) # reset the local seed again611 612 self.eval_dataloader = DataLoader(613 self.eval_dataset,614 batch_size=args.per_device_eval_batch_size,615 collate_fn=self.data_collator,616 drop_last=True,617 ) # no need to shuffle eval dataset618 self.eval_dataloader = accelerator.prepare(self.eval_dataloader)619 620 if self.is_deepspeed_enabled:621 self.reward_model = prepare_deepspeed(622 self.reward_model, args.per_device_train_batch_size, args.fp16, args.bf16623 )624 625 if self.ref_model is None:626 if not self.is_peft_model:627 raise ValueError("No reference model and model is not a Peft model.")628 else:629 self.ref_model = prepare_deepspeed(630 self.ref_model, args.per_device_train_batch_size, args.fp16, args.bf16631 )632 else:633 if self.ref_model is None:634 if not self.is_peft_model:635 raise ValueError("No reference model and model is not a Peft model.")636 else:637 self.ref_model = self.ref_model.to(self.accelerator.device)638 self.reward_model = self.reward_model.to(self.accelerator.device)639 640 def get_train_dataloader(self) -> DataLoader:641 return self.dataloader642 643 def get_eval_dataloader(self) -> DataLoader:644 return self.eval_dataloader645 646 @contextmanager647 def null_ref_context(self):648 """Context manager for handling null reference model (that is, peft adapter manipulation)."""649 with (650 self.accelerator.unwrap_model(self.model.policy).disable_adapter()651 if self.is_peft_model and not self.ref_adapter_name652 else nullcontext()653 ):654 if self.ref_adapter_name:655 self.model.policy.set_adapter(self.ref_adapter_name)656 yield657 if self.ref_adapter_name:658 self.model.policy.set_adapter(self.model_adapter_name or "default")659 660 def save_model(self, output_dir: Optional[str] = None, _internal_call: bool = False):661 backup_model = self.model662 self.model = self.model.policy # save only the policy663 664 if self.is_deepspeed_enabled:665 backup_deepspeed = self.deepspeed666 self.deepspeed = self.model667 668 super().save_model(output_dir, _internal_call)669 670 self.model = backup_model671 672 if self.is_deepspeed_enabled:673 self.deepspeed = backup_deepspeed674 675 def train(self):676 args = self.args677 accelerator = self.accelerator678 optimizer = self.optimizer679 model = self.model680 ref_policy = self.ref_model681 reward_model = self.reward_model682 processing_class = self.processing_class683 dataloader = self.dataloader684 device = accelerator.device685 686 def repeat_generator():687 while True:688 yield from dataloader689 690 iter_dataloader = iter(repeat_generator())691 generation_config = GenerationConfig(692 max_new_tokens=args.response_length,693 temperature=(args.temperature + 1e-7),694 top_k=0.0,695 top_p=1.0,696 do_sample=True,697 )698 699 accelerator.print("===training policy===")700 start_time = time.time()701 stats_shape = (args.num_ppo_epochs, args.num_mini_batches, args.gradient_accumulation_steps)702 approxkl_stats = torch.zeros(stats_shape, device=device)703 pg_clipfrac_stats = torch.zeros(stats_shape, device=device)704 pg_loss_stats = torch.zeros(stats_shape, device=device)705 vf_loss_stats = torch.zeros(stats_shape, device=device)706 vf_clipfrac_stats = torch.zeros(stats_shape, device=device)707 entropy_stats = torch.zeros(stats_shape, device=device)708 ratio_stats = torch.zeros(stats_shape, device=device)709 model.train()710 711 # trainer state initialization712 self.state.global_step = 0713 self.state.episode = 0714 self.state.max_steps = args.num_total_batches * args.num_mini_batches715 self.state.num_train_epochs = args.total_episodes / self.train_dataset_len716 # Compute absolute values for logging, eval, and save if given as ratio717 if args.logging_steps is not None:718 if args.logging_steps < 1:719 self.state.logging_steps = math.ceil(self.state.max_steps * args.logging_steps)720 else:721 self.state.logging_steps = args.logging_steps722 if args.eval_steps is not None:723 if args.eval_steps < 1:724 self.state.eval_steps = math.ceil(self.state.max_steps * args.eval_steps)725 else:726 self.state.eval_steps = args.eval_steps727 if args.save_steps is not None:728 if args.save_steps < 1:729 self.state.save_steps = math.ceil(self.state.max_steps * args.save_steps)730 else:731 self.state.save_steps = args.save_steps732 self.control = self.callback_handler.on_train_begin(args, self.state, self.control)733 734 # backward compatibility735 if self.is_deepspeed_enabled:736 self.deepspeed = self.model737 self.model_wrapped = self.model738 739 for update in range(1, args.num_total_batches + 1):740 self.state.episode += 1 * args.batch_size741 data = next(iter_dataloader)742 with torch.no_grad():743 queries = data["input_ids"].to(device)744 context_length = queries.shape[1]745 responses = []746 postprocessed_responses = []747 logprobs = []748 ref_logprobs = []749 scores = []750 sequence_lengths = []751 values = []752 with unwrap_model_for_generation(753 self.model, self.accelerator, gather_deepspeed3_params=self.args.ds3_gather_for_generation754 ) as unwrapped_model:755 query_responses, logitss = batch_generation(756 unwrapped_model.policy,757 queries,758 args.local_rollout_forward_batch_size,759 processing_class.pad_token_id,760 generation_config,761 )762 763 for i in range(0, queries.shape[0], args.local_rollout_forward_batch_size):764 query = queries[i : i + args.local_rollout_forward_batch_size]765 query_response = query_responses[i : i + args.local_rollout_forward_batch_size]766 response = query_response[:, context_length:]767 logits = logitss[i : i + args.local_rollout_forward_batch_size]768 logprob = selective_log_softmax(logits, response)769 del logits770 torch.cuda.empty_cache()771 772 if ref_policy is None:773 with self.null_ref_context():774 ref_output = forward(model.policy, query_response, processing_class.pad_token_id)775 else:776 ref_output = forward(ref_policy, query_response, processing_class.pad_token_id)777 ref_logits = ref_output.logits[:, context_length - 1 : -1]778 ref_logits /= args.temperature + 1e-7779 ref_logprob = selective_log_softmax(ref_logits, response)780 del ref_output, ref_logits781 torch.cuda.empty_cache()782 783 # Response Processing 1. truncate response after the first occurrence of `stop_token_id`784 postprocessed_response = response785 if self.stop_token_id is not None: # handle the edge case when stop_token_id exists but is 0786 postprocessed_response = truncate_response(787 self.stop_token_id, processing_class.pad_token_id, response788 )789 790 # Response Processing 2. run reward model on the truncated responses791 postprocessed_query_response = torch.cat((query, postprocessed_response), 1)792 sequence_length = first_true_indices(postprocessed_response == processing_class.pad_token_id) - 1793 unwrapped_value_model = accelerator.unwrap_model(model).value_model794 full_value, _, _ = get_reward(795 unwrapped_value_model, query_response, processing_class.pad_token_id, context_length796 )797 value = full_value[:, context_length - 1 : -1].squeeze(-1)798 _, score, _ = get_reward(799 reward_model, postprocessed_query_response, processing_class.pad_token_id, context_length800 )801 802 responses.append(response)803 postprocessed_responses.append(postprocessed_response)804 logprobs.append(logprob)805 ref_logprobs.append(ref_logprob)806 sequence_lengths.append(sequence_length)807 scores.append(score)808 values.append(value)809 responses = torch.cat(responses, 0)810 postprocessed_responses = torch.cat(postprocessed_responses, 0)811 logprobs = torch.cat(logprobs, 0)812 ref_logprobs = torch.cat(ref_logprobs, 0)813 sequence_lengths = torch.cat(sequence_lengths, 0)814 scores = torch.cat(scores, 0)815 values = torch.cat(values, 0)816 del (logprob, ref_logprob, full_value, value, score, unwrapped_model)817 torch.cuda.empty_cache()818 gc.collect()819 820 # Response Processing 3. Filter completion. Ensure that the sample contains stop_token_id821 # Completions not passing that filter will receive a lower score.822 contain_eos_token = torch.any(postprocessed_responses == self.processing_class.eos_token_id, dim=-1)823 if self.args.missing_eos_penalty is not None:824 scores[~contain_eos_token] -= self.args.missing_eos_penalty825 # accelerator.print(f"{scores=}, {(contain_eos_token.sum() / len(contain_eos_token))=}")826 827 # be very careful with `padding_mask_p1`; see https://excalidraw.com/#json=LWnzG4w2k5DjF_EOL_xPt,e2w3a-hFJ_gX5vOfeyXGTw828 response_idxs = torch.arange(responses.shape[1], device=responses.device).repeat(responses.shape[0], 1)829 padding_mask = response_idxs > sequence_lengths.unsqueeze(1)830 logprobs = torch.masked_fill(logprobs, padding_mask, INVALID_LOGPROB)831 ref_logprobs = torch.masked_fill(ref_logprobs, padding_mask, INVALID_LOGPROB)832 sequence_lengths_p1 = sequence_lengths + 1833 padding_mask_p1 = response_idxs > (sequence_lengths_p1.unsqueeze(1))834 values = torch.masked_fill(values, padding_mask_p1, 0)835 836 # 4. compute rewards837 kl = logprobs - ref_logprobs838 non_score_reward = -args.kl_coef * kl839 rewards = non_score_reward.clone()840 actual_start = torch.arange(rewards.size(0), device=rewards.device)841 actual_end = torch.where(sequence_lengths_p1 < rewards.size(1), sequence_lengths_p1, sequence_lengths)842 rewards[[actual_start, actual_end]] += scores843 844 # 5. whiten rewards845 if args.whiten_rewards:846 rewards = masked_whiten(rewards, mask=~padding_mask_p1, shift_mean=False)847 rewards = torch.masked_fill(rewards, padding_mask_p1, 0)848 849 # 6. compute advantages and returns850 lastgaelam = 0851 advantages_reversed = []852 gen_length = responses.shape[1]853 for t in reversed(range(gen_length)):854 nextvalues = values[:, t + 1] if t < gen_length - 1 else 0.0855 delta = rewards[:, t] + args.gamma * nextvalues - values[:, t]856 lastgaelam = delta + args.gamma * args.lam * lastgaelam857 advantages_reversed.append(lastgaelam)858 advantages = torch.stack(advantages_reversed[::-1], axis=1)859 returns = advantages + values860 advantages = masked_whiten(advantages, ~padding_mask)861 advantages = torch.masked_fill(advantages, padding_mask, 0)862 torch.cuda.empty_cache()863 864 # Do multiple epochs of PPO training, with a fresh random shuffle in each epoch865 for ppo_epoch_idx in range(args.num_ppo_epochs):866 b_inds = np.random.permutation(args.local_batch_size)867 minibatch_idx = 0868 for mini_batch_start in range(0, args.local_batch_size, args.local_mini_batch_size):869 mini_batch_end = mini_batch_start + args.local_mini_batch_size870 mini_batch_inds = b_inds[mini_batch_start:mini_batch_end]871 gradient_accumulation_idx = 0872 for micro_batch_start in range(0, args.local_mini_batch_size, args.per_device_train_batch_size):873 with accelerator.accumulate(model):874 micro_batch_end = micro_batch_start + args.per_device_train_batch_size875 micro_batch_inds = mini_batch_inds[micro_batch_start:micro_batch_end]876 mb_advantage = advantages[micro_batch_inds]877 mb_responses = responses[micro_batch_inds]878 mb_query_responses = query_responses[micro_batch_inds]879 mb_logprobs = logprobs[micro_batch_inds]880 mb_return = returns[micro_batch_inds]881 mb_values = values[micro_batch_inds]882 883 output, vpred_temp = forward(model, mb_query_responses, processing_class.pad_token_id)884 logits = output.logits[:, context_length - 1 : -1]885 logits /= args.temperature + 1e-7886 new_logprobs = selective_log_softmax(logits, mb_responses)887 new_logprobs = torch.masked_fill(888 new_logprobs, padding_mask[micro_batch_inds], INVALID_LOGPROB889 )890 vpred = vpred_temp[:, context_length - 1 : -1].squeeze(-1)891 vpred = torch.masked_fill(vpred, padding_mask_p1[micro_batch_inds], 0)892 vpredclipped = torch.clamp(893 vpred,894 mb_values - args.cliprange_value,895 mb_values + args.cliprange_value,896 )897 vf_losses1 = torch.square(vpred - mb_return)898 vf_losses2 = torch.square(vpredclipped - mb_return)899 vf_loss_max = torch.max(vf_losses1, vf_losses2)900 vf_loss = 0.5 * masked_mean(vf_loss_max, ~padding_mask_p1[micro_batch_inds])901 vf_clipfrac = masked_mean(902 (vf_losses2 > vf_losses1).float(), ~padding_mask_p1[micro_batch_inds]903 )904 logprobs_diff = new_logprobs - mb_logprobs905 ratio = torch.exp(logprobs_diff)906 pg_losses = -mb_advantage * ratio907 pg_losses2 = -mb_advantage * torch.clamp(ratio, 1.0 - args.cliprange, 1.0 + args.cliprange)908 pg_loss_max = torch.max(pg_losses, pg_losses2)909 pg_loss = masked_mean(pg_loss_max, ~padding_mask[micro_batch_inds])910 loss = pg_loss + args.vf_coef * vf_loss911 accelerator.backward(loss)912 optimizer.step()913 optimizer.zero_grad()914 with torch.no_grad():915 pg_clipfrac = masked_mean(916 (pg_losses2 > pg_losses).float(), ~padding_mask[micro_batch_inds]917 )918 prob_dist = torch.nn.functional.softmax(logits, dim=-1)919 entropy = torch.logsumexp(logits, dim=-1) - torch.sum(prob_dist * logits, dim=-1)920 approxkl = 0.5 * (logprobs_diff**2).mean()921 approxkl_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = approxkl922 pg_clipfrac_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = (923 pg_clipfrac924 )925 pg_loss_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = pg_loss926 vf_loss_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = vf_loss927 vf_clipfrac_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = (928 vf_clipfrac929 )930 entropy_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = entropy.mean()931 ratio_stats[ppo_epoch_idx, minibatch_idx, gradient_accumulation_idx] = ratio.mean()932 gradient_accumulation_idx += 1933 minibatch_idx += 1934 # del everything and empty cache935 # fmt: off936 del (937 output, vpred_temp, logits, new_logprobs, vpred, vpredclipped,938 vf_losses1, vf_losses2, vf_loss, vf_clipfrac, logprobs_diff, ratio, pg_losses, pg_losses2, pg_loss_max,939 pg_loss, loss, pg_clipfrac, prob_dist, entropy, approxkl, mb_return,940 mb_advantage, mb_values, mb_responses, mb_query_responses, mb_logprobs,941 )942 # fmt: on943 torch.cuda.empty_cache()944 with torch.no_grad():945 mean_kl = kl.sum(1).mean()946 mean_entropy = (-logprobs).sum(1).mean()947 mean_non_score_reward = non_score_reward.sum(1).mean()948 rlhf_reward = mean_non_score_reward + scores.mean()949 eps = int(self.state.episode / (time.time() - start_time))950 metrics = {}951 metrics["eps"] = eps952 metrics["objective/kl"] = self.accelerator.gather_for_metrics(mean_kl).mean().item()953 metrics["objective/entropy"] = self.accelerator.gather_for_metrics(mean_entropy).mean().item()954 metrics["objective/non_score_reward"] = (955 self.accelerator.gather_for_metrics(mean_non_score_reward).mean().item()956 )957 metrics["objective/rlhf_reward"] = self.accelerator.gather_for_metrics(rlhf_reward).mean().item()958 metrics["objective/scores"] = self.accelerator.gather_for_metrics(scores.mean()).mean().item()959 metrics["policy/approxkl_avg"] = self.accelerator.gather_for_metrics(approxkl_stats).mean().item()960 metrics["policy/clipfrac_avg"] = self.accelerator.gather_for_metrics(pg_clipfrac_stats).mean().item()961 metrics["loss/policy_avg"] = self.accelerator.gather_for_metrics(pg_loss_stats).mean().item()962 metrics["loss/value_avg"] = self.accelerator.gather_for_metrics(vf_loss_stats).mean().item()963 metrics["val/clipfrac_avg"] = self.accelerator.gather_for_metrics(vf_clipfrac_stats).mean().item()964 metrics["policy/entropy_avg"] = self.accelerator.gather_for_metrics(entropy_stats).mean().item()965 metrics["val/ratio"] = self.accelerator.gather_for_metrics(ratio_stats).mean().item()966 metrics["val/ratio_var"] = self.accelerator.gather_for_metrics(ratio_stats).var().item()967 metrics["val/num_eos_tokens"] = (responses == processing_class.eos_token_id).sum().item()968 metrics["lr"] = self.lr_scheduler.get_last_lr()[0]969 metrics["episode"] = self.state.episode970 self.state.epoch = self.state.episode / self.train_dataset_len # used by self.log971 self.state.global_step += 1972 self.log(metrics)973 974 self.lr_scheduler.step()975 self.control = self.callback_handler.on_step_end(args, self.state, self.control)976 if self.control.should_save:977 self._save_checkpoint(model, trial=None)978 self.control = self.callback_handler.on_save(self.args, self.state, self.control)979 del kl, mean_kl, mean_entropy, mean_non_score_reward, scores, metrics, non_score_reward980 torch.cuda.empty_cache()981 gc.collect()982 983 if args.num_sample_generations > 0 and (update - 1) % self.sample_generations_freq == 0:984 self.generate_completions(sampling=True)985 torch.cuda.empty_cache()986 del (987 query_responses,988 responses,989 postprocessed_responses,990 logprobs,991 ref_logprobs,992 values,993 sequence_lengths,994 contain_eos_token,995 sequence_lengths_p1,996 response_idxs,997 padding_mask,998 padding_mask_p1,999 rewards,1000 actual_start,1001 actual_end,1002 advantages,1003 returns,1004 )1005 torch.cuda.empty_cache()1006 1007 # HF trainer specifics1008 self.control = self.callback_handler.on_train_end(args, self.state, self.control)1009 if self.control.should_save:1010 self._save_checkpoint(model, trial=None, metrics=None)1011 self.control = self.callback_handler.on_save(self.args, self.state, self.control)1012 1013 def generate_completions(self, sampling: bool = False):1014 args = self.args1015 processing_class = self.processing_class1016 generation_config = GenerationConfig(1017 max_new_tokens=self.args.response_length,1018 temperature=(0.01 + 1e-7),1019 top_k=0.0,1020 top_p=1.0,1021 do_sample=True,1022 )1023 1024 table = defaultdict(list)1025 with unwrap_model_for_generation(1026 self.model, self.accelerator, gather_deepspeed3_params=self.args.ds3_gather_for_generation1027 ) as unwrapped_model:1028 for batch in self.eval_dataloader:1029 query = batch["input_ids"]1030 with torch.no_grad():1031 context_length = query.shape[1]1032 query_response, _ = batch_generation(1033 unwrapped_model.policy,1034 query,1035 query.shape[0],1036 processing_class.pad_token_id,1037 generation_config,1038 )1039 response = query_response[:, context_length:]1040 postprocessed_response = response1041 if self.stop_token_id is not None: # handle the edge case when stop_token_id exists but is 01042 postprocessed_response = truncate_response(1043 self.stop_token_id, processing_class.pad_token_id, response1044 )1045 table["query"].extend(1046 gather_object(processing_class.batch_decode(query, skip_special_tokens=True))1047 )1048 table["model response"].extend(1049 gather_object(processing_class.batch_decode(postprocessed_response))1050 )1051 1052 postprocessed_query_response = torch.cat((query, postprocessed_response), 1)1053 _, score, _ = get_reward(1054 self.reward_model, postprocessed_query_response, processing_class.pad_token_id, context_length1055 )1056 table["score"].extend(self.accelerator.gather_for_metrics(score).float().cpu().numpy())1057 1058 if sampling:1059 break1060 df = pd.DataFrame(table)1061 1062 if self.accelerator.is_main_process:1063 print_rich_table(df.iloc[0 : 0 + 5])1064 if "wandb" in args.report_to:1065 import wandb1066 1067 if wandb.run is not None:1068 wandb.log({"completions": wandb.Table(dataframe=df)})1069 1070 if "comet_ml" in args.report_to:1071 log_table_to_comet_experiment(1072 name="completions.csv",1073 table=df,1074 )1075 1076 def create_model_card(1077 self,1078 model_name: Optional[str] = None,1079 dataset_name: Optional[str] = None,1080 tags: Union[str, list[str], None] = None,1081 ):1082 """1083 Creates a draft of a model card using the information available to the `Trainer`.1084 1085 Args:1086 model_name (`str` or `None`, *optional*, defaults to `None`):1087 Name of the model.1088 dataset_name (`str` or `None`, *optional*, defaults to `None`):1089 Name of the dataset used for training.1090 tags (`str`, `list[str]` or `None`, *optional*, defaults to `None`):1091 Tags to be associated with the model card.1092 """1093 if not self.is_world_process_zero():1094 return1095 1096 if hasattr(self.model.config, "_name_or_path") and not os.path.isdir(self.model.config._name_or_path):1097 base_model = self.model.config._name_or_path1098 else:1099 base_model = None1100 1101 tags = tags or []1102 if isinstance(tags, str):1103 tags = [tags]1104 1105 if hasattr(self.model.config, "unsloth_version"):1106 tags.append("unsloth")1107 1108 citation = textwrap.dedent("""\1109 @article{mziegler2019fine-tuning,1110 title = {{Fine-Tuning Language Models from Human Preferences}},1111 author = {Daniel M. Ziegler and Nisan Stiennon and Jeffrey Wu and Tom B. Brown and Alec Radford and Dario Amodei and Paul F. Christiano and Geoffrey Irving},1112 year = 2019,1113 eprint = {arXiv:1909.08593}1114 }""")1115 1116 model_card = generate_model_card(1117 base_model=base_model,1118 model_name=model_name,1119 hub_model_id=self.hub_model_id,1120 dataset_name=dataset_name,1121 tags=tags,1122 wandb_url=wandb.run.get_url() if is_wandb_available() and wandb.run is not None else None,1123 comet_url=get_comet_experiment_url(),1124 trainer_name="PPO",1125 trainer_citation=citation,1126 paper_title="Fine-Tuning Language Models from Human Preferences",1127 paper_id="1909.08593",1128 )1129 1130 model_card.save(os.path.join(self.args.output_dir, "README.md"))1131class UnslothPPOTrainer(_UnslothPPOTrainer):1132 """1133 1134 """1135 def __init__(1136 self,1137 args,1138 processing_class,1139 model,1140 ref_model,1141 reward_model,1142 train_dataset,1143 value_model = None,1144 data_collator = None,1145 eval_dataset = None,1146 callbacks = None,1147 peft_config = None,1148 **kwargs1149 ):1150 if args is None: args = UnslothPPOConfig()1151 use_bf16 = getattr(args, 'bf16', False)1152 use_fp16 = getattr(args, 'fp16', False)1153 force_float32 = False1154 if os.environ.get('UNSLOTH_FORCE_FLOAT32', '0') == '1':1155 print('Unsloth: Switching to float32 training since model cannot work with float16')1156 force_float32 = True1157 mixed_precision_dtype = os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32')1158 dtype = getattr(model.config, 'torch_dtype', None)1159 if dtype is None: dtype = model.get_input_embeddings().dtype1160 from unsloth_zoo.utils import _get_dtype1161 dtype = _get_dtype(dtype)1162 float16 = dtype == torch.float161163 if not force_float32 and (float16 and use_bf16): raise TypeError('Unsloth: Model is in float16 precision but you want to use bfloat16 precision. Set fp16 to `True` and bf16 to `False`')1164 if not force_float32 and (not float16 and use_fp16): raise TypeError('Unsloth: Model is in bfloat16 precision but you want to use float16 precision. Set fp16 to `False` and bf16 to `True`')1165 if force_float32:1166 args.fp16 = False1167 args.bf16 = False1168 os.environ['ACCELERATE_MIXED_PRECISION'] = 'no'1169 elif (not use_bf16 and not use_fp16) and mixed_precision_dtype == 'float32':1170 args.fp16 = float161171 args.bf16 = not float161172 os.environ['ACCELERATE_MIXED_PRECISION'] = 'fp16' if float16 else 'bf16'1173 if getattr(args, 'eval_dataset', None) is not None and getattr(args, 'eval_strategy', 'no') == 'no':1174 args.eval_strategy = 'steps'1175 if getattr(args, 'eval_steps', None) is None: args.eval_steps = 0.11176 ga_steps = getattr(args, 'gradient_accumulation_steps', None)1177 if ga_steps is not None and ga_steps > 1:1178 from transformers import __version__ as transformers_version1179 if Version(transformers_version) <= Version('4.45.2'):1180 print('**** Unsloth: Please use our fixed gradient_accumulation_steps by updating transformers, TRL and Unsloth!\n'1181 '`pip install --upgrade --no-cache-dir --force-reinstall --no-deps unsloth transformers trl unsloth_zoo`')1182 if getattr(args, 'eval_strategy', 'no') != 'no':1183 eval_bsz = getattr(args, 'per_device_eval_batch_size', 8)1184 if eval_bsz == 8 and args.per_device_train_batch_size < eval_bsz: args.per_device_eval_batch_size = args.per_device_train_batch_size1185 if getattr(args, 'eval_accumulation_steps', None) is None and ga_steps is not None: args.eval_accumulation_steps = ga_steps1186 fp16_full_eval = getattr(args, 'fp16_full_eval', False)1187 bf16_full_eval = getattr(args, 'bf16_full_eval', False)1188 if args.fp16 and bf16_full_eval: args.bf16_full_eval = False; args.fp16_full_eval = True1189 if args.bf16 and fp16_full_eval: args.bf16_full_eval = True; args.fp16_full_eval = False1190 if force_float32:1191 args.bf16_full_eval = False1192 args.fp16_full_eval = False1193 elif os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32') == 'bfloat16':1194 args.bf16_full_eval = True1195 args.fp16_full_eval = False1196 elif not bf16_full_eval and not fp16_full_eval:1197 args.bf16_full_eval = args.bf161198 args.fp16_full_eval = args.fp161199 _output_logits = False1200 if locals().get('compute_metrics', None) is not None: _output_logits = True