MLVLab/Human_Object_Interaction

1# ------------------------------------------------------------------------2# HOTR official code : engine/trainer.py3# Copyright (c) Kakao Brain, Inc. and its affiliates. All Rights Reserved4# ------------------------------------------------------------------------5# Modified from DETR (https://github.com/facebookresearch/detr)6# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved7# ------------------------------------------------------------------------8import math9import torch10import sys11import hotr.util.misc as utils12import hotr.util.logger as loggers13from hotr.util.ramp import *14from typing import Iterable15import wandb16 17def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module,18                    data_loader: Iterable, optimizer: torch.optim.Optimizer,19                    device: torch.device, epoch: int, max_epoch: int, ramp_up_epoch: int,rampdown_epoch: int,max_consis_coef: float=1.0,max_norm: float = 0,dataset_file: str = 'coco', log: bool = False):20    model.train()21    criterion.train()22    metric_logger = loggers.MetricLogger(mode="train", delimiter="  ")23    metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))24    space_fmt = str(len(str(max_epoch)))25    header = 'Epoch [{start_epoch: >{fill}}/{end_epoch}]'.format(start_epoch=epoch+1, end_epoch=max_epoch, fill=space_fmt)26    print_freq = int(len(data_loader)/5)27 28    if epoch<=rampdown_epoch:29        consis_coef=sigmoid_rampup(epoch,ramp_up_epoch,max_consis_coef)30    else:31        consis_coef=cosine_rampdown(epoch-rampdown_epoch,max_epoch-rampdown_epoch,max_consis_coef)32 33    print(f"\n>>> Epoch #{(epoch+1)}")34    for samples, targets in metric_logger.log_every(data_loader, print_freq, header):35        samples = samples.to(device)36        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]37 38        outputs = model(samples)39        loss_dict = criterion(outputs, targets, log)40        #print(loss_dict)41        weight_dict = criterion.weight_dict42 43        losses = sum(loss_dict[k] * weight_dict[k]*consis_coef if 'consistency' in k else loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)44 45        # reduce losses over all GPUs for logging purposes46        loss_dict_reduced = utils.reduce_dict(loss_dict)47        loss_dict_reduced_unscaled = {f'{k}_unscaled': v48                                      for k, v in loss_dict_reduced.items()}49        loss_dict_reduced_scaled = {k: v * weight_dict[k]*consis_coef if 'consistency' in k else v * weight_dict[k] for k, v in loss_dict_reduced.items() if k in weight_dict}50        losses_reduced_scaled = sum(loss_dict_reduced_scaled.values())51        loss_value = losses_reduced_scaled.item()52    53        54        if not math.isfinite(loss_value):55            print("Loss is {}, stopping training".format(loss_value))56            print(loss_dict_reduced)57            sys.exit(1)58 59        optimizer.zero_grad()60        losses.backward()61        if max_norm > 0:62            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)63        optimizer.step()64 65        metric_logger.update(loss=loss_value, **loss_dict_reduced_scaled)66        if "obj_class_error" in loss_dict:67            metric_logger.update(obj_class_error=loss_dict_reduced['obj_class_error'])68        metric_logger.update(lr=optimizer.param_groups[0]["lr"])69    # gather the stats from all processes70    metric_logger.synchronize_between_processes()71    if utils.get_rank() == 0 and log: wandb.log(loss_dict_reduced_scaled)72    print("Averaged stats:", metric_logger)73    return {k: meter.global_avg for k, meter in metric_logger.meters.items()}74