Shortcuts

Source code for torch.nn.utils.clip_grad

import warnings
import torch
from torch._six import inf


[docs]def clip_grad_norm_(parameters, max_norm, norm_type=2): r"""Clips gradient norm of an iterable of parameters. The norm is computed over all gradients together, as if they were concatenated into a single vector. Gradients are modified in-place. Arguments: parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a single Tensor that will have gradients normalized max_norm (float or int): max norm of the gradients norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for infinity norm. Returns: Total norm of the parameters (viewed as a single vector). """ if isinstance(parameters, torch.Tensor): parameters = [parameters] parameters = list(filter(lambda p: p.grad is not None, parameters)) max_norm = float(max_norm) norm_type = float(norm_type) if norm_type == inf: total_norm = max(p.grad.data.abs().max() for p in parameters) else: total_norm = 0 for p in parameters: param_norm = p.grad.data.norm(norm_type) total_norm += param_norm.item() ** norm_type total_norm = total_norm ** (1. / norm_type) clip_coef = max_norm / (total_norm + 1e-6) if clip_coef < 1: for p in parameters: p.grad.data.mul_(clip_coef) return total_norm
def clip_grad_norm(parameters, max_norm, norm_type=2): r"""Clips gradient norm of an iterable of parameters. .. warning:: This method is now deprecated in favor of :func:`torch.nn.utils.clip_grad_norm_`. """ warnings.warn("torch.nn.utils.clip_grad_norm is now deprecated in favor " "of torch.nn.utils.clip_grad_norm_.", stacklevel=2) return clip_grad_norm_(parameters, max_norm, norm_type)
[docs]def clip_grad_value_(parameters, clip_value): r"""Clips gradient of an iterable of parameters at specified value. Gradients are modified in-place. Arguments: parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a single Tensor that will have gradients normalized clip_value (float or int): maximum allowed value of the gradients The gradients are clipped in the range [-clip_value, clip_value] """ if isinstance(parameters, torch.Tensor): parameters = [parameters] clip_value = float(clip_value) for p in filter(lambda p: p.grad is not None, parameters): p.grad.data.clamp_(min=-clip_value, max=clip_value)

Docs

Access comprehensive developer documentation for PyTorch

View Docs

Tutorials

Get in-depth tutorials for beginners and advanced developers

View Tutorials

Resources

Find development resources and get your questions answered

View Resources