Module keras.api.keras.optimizers
Public API for tf.keras.optimizers namespace.
Expand source code
# This file is MACHINE GENERATED! Do not edit.
# Generated by: tensorflow/python/tools/api/generator/create_python_api.py script.
"""Public API for tf.keras.optimizers namespace.
"""
from __future__ import print_function as _print_function
import sys as _sys
from keras.api.keras.optimizers import schedules
from keras.optimizer_v2.adadelta import Adadelta
from keras.optimizer_v2.adagrad import Adagrad
from keras.optimizer_v2.adam import Adam
from keras.optimizer_v2.adamax import Adamax
from keras.optimizer_v2.ftrl import Ftrl
from keras.optimizer_v2.gradient_descent import SGD
from keras.optimizer_v2.nadam import Nadam
from keras.optimizer_v2.optimizer_v2 import OptimizerV2 as Optimizer
from keras.optimizer_v2.rmsprop import RMSprop
from keras.optimizers import deserialize
from keras.optimizers import get
from keras.optimizers import serialize
del _print_function
from tensorflow.python.util import module_wrapper as _module_wrapper
if not isinstance(_sys.modules[__name__], _module_wrapper.TFModuleWrapper):
_sys.modules[__name__] = _module_wrapper.TFModuleWrapper(
_sys.modules[__name__], "keras.optimizers", public_apis=None, deprecation=True,
has_lite=False)
Sub-modules
keras.api.keras.optimizers.schedules
-
Public API for tf.keras.optimizers.schedules namespace.
Functions
def deserialize(config, custom_objects=None)
-
Inverse of the
serialize()
function.Args
config
- Optimizer configuration dictionary.
custom_objects
- Optional dictionary mapping names (strings) to custom objects (classes and functions) to be considered during deserialization.
Returns
A Keras Optimizer instance.
Expand source code
@keras_export('keras.optimizers.deserialize') def deserialize(config, custom_objects=None): """Inverse of the `serialize` function. Args: config: Optimizer configuration dictionary. custom_objects: Optional dictionary mapping names (strings) to custom objects (classes and functions) to be considered during deserialization. Returns: A Keras Optimizer instance. """ # loss_scale_optimizer has a direct dependency of optimizer, import here # rather than top to avoid the cyclic dependency. from keras.mixed_precision import loss_scale_optimizer # pylint: disable=g-import-not-at-top all_classes = { 'adadelta': adadelta_v2.Adadelta, 'adagrad': adagrad_v2.Adagrad, 'adam': adam_v2.Adam, 'adamax': adamax_v2.Adamax, 'nadam': nadam_v2.Nadam, 'rmsprop': rmsprop_v2.RMSprop, 'sgd': gradient_descent_v2.SGD, 'ftrl': ftrl.Ftrl, 'lossscaleoptimizer': loss_scale_optimizer.LossScaleOptimizer, # LossScaleOptimizerV1 deserializes into LossScaleOptimizer, as # LossScaleOptimizerV1 will be removed soon but deserializing it will # still be supported. 'lossscaleoptimizerv1': loss_scale_optimizer.LossScaleOptimizer, } # Make deserialization case-insensitive for built-in optimizers. if config['class_name'].lower() in all_classes: config['class_name'] = config['class_name'].lower() return deserialize_keras_object( config, module_objects=all_classes, custom_objects=custom_objects, printable_module_name='optimizer')
def get(identifier)
-
Retrieves a Keras Optimizer instance.
Args
identifier
- Optimizer identifier, one of - String: name of an optimizer - Dictionary: configuration dictionary. - Keras Optimizer instance (it will be returned unchanged). - TensorFlow Optimizer instance (it will be wrapped as a Keras Optimizer).
Returns
A Keras Optimizer instance.
Raises
ValueError
- If
identifier
cannot be interpreted.
Expand source code
@keras_export('keras.optimizers.get') def get(identifier): """Retrieves a Keras Optimizer instance. Args: identifier: Optimizer identifier, one of - String: name of an optimizer - Dictionary: configuration dictionary. - Keras Optimizer instance (it will be returned unchanged). - TensorFlow Optimizer instance (it will be wrapped as a Keras Optimizer). Returns: A Keras Optimizer instance. Raises: ValueError: If `identifier` cannot be interpreted. """ if isinstance(identifier, (Optimizer, optimizer_v2.OptimizerV2)): return identifier # Wrap legacy TF optimizer instances elif isinstance(identifier, tf.compat.v1.train.Optimizer): opt = TFOptimizer(identifier) backend.track_tf_optimizer(opt) return opt elif isinstance(identifier, dict): return deserialize(identifier) elif isinstance(identifier, str): config = {'class_name': str(identifier), 'config': {}} return deserialize(config) else: raise ValueError( 'Could not interpret optimizer identifier: {}'.format(identifier))
def serialize(optimizer)
-
Serialize the optimizer configuration to JSON compatible python dict.
The configuration can be used for persistence and reconstruct the
OptimizerV2
instance again.>>> tf.keras.optimizers.serialize(tf.keras.optimizers.SGD()) {'class_name': 'SGD', 'config': {'name': 'SGD', 'learning_rate': 0.01, 'decay': 0.0, 'momentum': 0.0, 'nesterov': False}}
Args
optimizer
- An
OptimizerV2
instance to serialize.
Returns
Python dict which contains the configuration of the input optimizer.
Expand source code
@keras_export('keras.optimizers.serialize') def serialize(optimizer): """Serialize the optimizer configuration to JSON compatible python dict. The configuration can be used for persistence and reconstruct the `Optimizer` instance again. >>> tf.keras.optimizers.serialize(tf.keras.optimizers.SGD()) {'class_name': 'SGD', 'config': {'name': 'SGD', 'learning_rate': 0.01, 'decay': 0.0, 'momentum': 0.0, 'nesterov': False}} Args: optimizer: An `Optimizer` instance to serialize. Returns: Python dict which contains the configuration of the input optimizer. """ return serialize_keras_object(optimizer)
Classes
class Adadelta (learning_rate=0.001, rho=0.95, epsilon=1e-07, name='Adadelta', **kwargs)
-
Optimizer that implements the Adadelta algorithm.
Adadelta optimization is a stochastic gradient descent method that is based on adaptive learning rate per dimension to address two drawbacks:
- The continual decay of learning rates throughout training.
- The need for a manually selected global learning rate.
Adadelta is a more robust extension of Adagrad that adapts learning rates based on a moving window of gradient updates, instead of accumulating all past gradients. This way, Adadelta continues learning even when many updates have been done. Compared to Adagrad, in the original version of Adadelta you don't have to set an initial learning rate. In this version, the initial learning rate can be set, as in most other Keras optimizers.
Args
learning_rate
- Initial value for the learning rate:
either a floating point value,
or a
tf.keras.optimizers.schedules.LearningRateSchedule
instance. Defaults to 0.001. Note thatAdadelta
tends to benefit from higher initial learning rate values compared to other optimizers. To match the exact form in the original paper, use 1.0. rho
- A
Tensor
or a floating point value. The decay rate. epsilon
- Small floating point value used to maintain numerical stability.
name
- Optional name prefix for the operations created when applying
gradients.
Defaults to
"Adadelta"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm and represents the maximum norm of each parameter;"clipvalue"
(float) clips gradient by value and represents the maximum absolute value of each parameter.
Reference
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Adadelta(optimizer_v2.OptimizerV2): r"""Optimizer that implements the Adadelta algorithm. Adadelta optimization is a stochastic gradient descent method that is based on adaptive learning rate per dimension to address two drawbacks: - The continual decay of learning rates throughout training. - The need for a manually selected global learning rate. Adadelta is a more robust extension of Adagrad that adapts learning rates based on a moving window of gradient updates, instead of accumulating all past gradients. This way, Adadelta continues learning even when many updates have been done. Compared to Adagrad, in the original version of Adadelta you don't have to set an initial learning rate. In this version, the initial learning rate can be set, as in most other Keras optimizers. Args: learning_rate: Initial value for the learning rate: either a floating point value, or a `tf.keras.optimizers.schedules.LearningRateSchedule` instance. Defaults to 0.001. Note that `Adadelta` tends to benefit from higher initial learning rate values compared to other optimizers. To match the exact form in the original paper, use 1.0. rho: A `Tensor` or a floating point value. The decay rate. epsilon: Small floating point value used to maintain numerical stability. name: Optional name prefix for the operations created when applying gradients. Defaults to `"Adadelta"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm and represents the maximum norm of each parameter; `"clipvalue"` (float) clips gradient by value and represents the maximum absolute value of each parameter. Reference: - [Zeiler, 2012](http://arxiv.org/abs/1212.5701) """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, rho=0.95, epsilon=1e-7, name='Adadelta', **kwargs): super(Adadelta, self).__init__(name, **kwargs) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('decay', self._initial_decay) self._set_hyper('rho', rho) self.epsilon = epsilon or backend_config.epsilon() def _create_slots(self, var_list): # Separate for-loops to respect the ordering of slot variables from v1. for v in var_list: self.add_slot(v, 'accum_grad') for v in var_list: self.add_slot(v, 'accum_var') def _prepare_local(self, var_device, var_dtype, apply_state): super(Adadelta, self)._prepare_local(var_device, var_dtype, apply_state) apply_state[(var_device, var_dtype)].update( dict( epsilon=tf.convert_to_tensor( self.epsilon, var_dtype), rho=tf.identity(self._get_hyper('rho', var_dtype)))) def set_weights(self, weights): params = self.weights # Override set_weights for backward compatibility of Keras V1 optimizer # since it does not include iteration at head of the weight list. Set # iteration to 0. if len(params) == len(weights) + 1: weights = [np.array(0)] + weights super(Adadelta, self).set_weights(weights) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) accum_grad = self.get_slot(var, 'accum_grad') accum_var = self.get_slot(var, 'accum_var') return tf.raw_ops.ResourceApplyAdadelta( var=var.handle, accum=accum_grad.handle, accum_update=accum_var.handle, lr=coefficients['lr_t'], rho=coefficients['rho'], epsilon=coefficients['epsilon'], grad=grad, use_locking=self._use_locking) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) accum_grad = self.get_slot(var, 'accum_grad') accum_var = self.get_slot(var, 'accum_var') return tf.raw_ops.ResourceSparseApplyAdadelta( var=var.handle, accum=accum_grad.handle, accum_update=accum_var.handle, lr=coefficients['lr_t'], rho=coefficients['rho'], epsilon=coefficients['epsilon'], grad=grad, indices=indices, use_locking=self._use_locking) def get_config(self): config = super(Adadelta, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'rho': self._serialize_hyperparameter('rho'), 'epsilon': self.epsilon, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Adagrad (learning_rate=0.001, initial_accumulator_value=0.1, epsilon=1e-07, name='Adagrad', **kwargs)
-
Optimizer that implements the Adagrad algorithm.
Adagrad is an optimizer with parameter-specific learning rates, which are adapted relative to how frequently a parameter gets updated during training. The more updates a parameter receives, the smaller the updates.
Args
learning_rate
- Initial value for the learning rate:
either a floating point value,
or a
tf.keras.optimizers.schedules.LearningRateSchedule
instance. Defaults to 0.001. Note thatAdagrad
tends to benefit from higher initial learning rate values compared to other optimizers. To match the exact form in the original paper, use 1.0. initial_accumulator_value
- Floating point value. Starting value for the accumulators (per-parameter momentum values). Must be non-negative.
epsilon
- Small floating point value used to maintain numerical stability.
name
- Optional name prefix for the operations created when applying
gradients.
Defaults to
"Adagrad"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm and represents the maximum L2 norm of each weight variable;"clipvalue"
(float) clips gradient by value and represents the maximum absolute value of each weight variable.
Reference
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Adagrad(optimizer_v2.OptimizerV2): r"""Optimizer that implements the Adagrad algorithm. Adagrad is an optimizer with parameter-specific learning rates, which are adapted relative to how frequently a parameter gets updated during training. The more updates a parameter receives, the smaller the updates. Args: learning_rate: Initial value for the learning rate: either a floating point value, or a `tf.keras.optimizers.schedules.LearningRateSchedule` instance. Defaults to 0.001. Note that `Adagrad` tends to benefit from higher initial learning rate values compared to other optimizers. To match the exact form in the original paper, use 1.0. initial_accumulator_value: Floating point value. Starting value for the accumulators (per-parameter momentum values). Must be non-negative. epsilon: Small floating point value used to maintain numerical stability. name: Optional name prefix for the operations created when applying gradients. Defaults to `"Adagrad"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm and represents the maximum L2 norm of each weight variable; `"clipvalue"` (float) clips gradient by value and represents the maximum absolute value of each weight variable. Reference: - [Duchi et al., 2011]( http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf). """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, initial_accumulator_value=0.1, epsilon=1e-7, name='Adagrad', **kwargs): if initial_accumulator_value < 0.0: raise ValueError('initial_accumulator_value must be non-negative: %s' % initial_accumulator_value) if epsilon is None: epsilon = backend_config.epsilon() super(Adagrad, self).__init__(name, **kwargs) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('decay', self._initial_decay) self._initial_accumulator_value = initial_accumulator_value self.epsilon = epsilon or backend_config.epsilon() def _create_slots(self, var_list): for var in var_list: dtype = var.dtype.base_dtype init = tf.compat.v1.constant_initializer( self._initial_accumulator_value, dtype=dtype) self.add_slot(var, 'accumulator', init) def _prepare_local(self, var_device, var_dtype, apply_state): super(Adagrad, self)._prepare_local(var_device, var_dtype, apply_state) apply_state[(var_device, var_dtype)].update( dict( epsilon=tf.convert_to_tensor( self.epsilon, var_dtype), neg_lr_t=-apply_state[(var_device, var_dtype)]['lr_t'], zero=tf.zeros((), dtype=tf.int64))) def set_weights(self, weights): params = self.weights # Override set_weights for backward compatibility of Keras V1 optimizer # since it does not include iteration at head of the weight list. Set # iteration to 0. if len(params) == len(weights) + 1: weights = [np.array(0)] + weights super(Adagrad, self).set_weights(weights) @classmethod def from_config(cls, config, custom_objects=None): """Creates an optimizer from its config. This method is the reverse of `get_config`, capable of instantiating the same optimizer from the config dictionary. Args: config: A Python dictionary, typically the output of get_config. custom_objects: A Python dictionary mapping names to additional Python objects used to create this optimizer, such as a function used for a hyperparameter. Returns: An optimizer instance. """ if 'initial_accumulator_value' not in config: config['initial_accumulator_value'] = 0.1 if 'lr' in config: config['learning_rate'] = config.pop('lr') return cls(**config) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) acc = self.get_slot(var, 'accumulator') return tf.raw_ops.ResourceApplyAdagradV2( var=var.handle, accum=acc.handle, lr=coefficients['lr_t'], epsilon=coefficients['epsilon'], grad=grad, use_locking=self._use_locking) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) acc = self.get_slot(var, 'accumulator') return tf.raw_ops.ResourceSparseApplyAdagradV2( var=var.handle, accum=acc.handle, lr=coefficients['lr_t'], epsilon=coefficients['epsilon'], grad=grad, indices=indices, use_locking=self._use_locking) def get_config(self): config = super(Adagrad, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'initial_accumulator_value': self._initial_accumulator_value, 'epsilon': self.epsilon, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Adam (learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam', **kwargs)
-
Optimizer that implements the Adam algorithm.
Adam optimization is a stochastic gradient descent method that is based on adaptive estimation of first-order and second-order moments.
According to Kingma et al., 2014, the method is "computationally efficient, has little memory requirement, invariant to diagonal rescaling of gradients, and is well suited for problems that are large in terms of data/parameters".
Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
, or a callable that takes no arguments and returns the actual value to use, The learning rate. Defaults to 0.001. beta_1
- A float value or a constant float tensor, or a callable that takes no arguments and returns the actual value to use. The exponential decay rate for the 1st moment estimates. Defaults to 0.9.
beta_2
- A float value or a constant float tensor, or a callable that takes no arguments and returns the actual value to use, The exponential decay rate for the 2nd moment estimates. Defaults to 0.999.
epsilon
- A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7.
amsgrad
- Boolean. Whether to apply AMSGrad variant of this algorithm from
the paper "On the Convergence of Adam and beyond". Defaults to
False
. name
- Optional name for the operations created when applying gradients.
Defaults to
"Adam"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Usage:
>>> opt = tf.keras.optimizers.Adam(learning_rate=0.1) >>> var1 = tf.Variable(10.0) >>> loss = lambda: (var1 ** 2)/2.0 # d(loss)/d(var1) == var1 >>> step_count = opt.minimize(loss, [var1]).numpy() >>> # The first step is `-learning_rate*sign(grad)` >>> var1.numpy() 9.9
Reference
- Kingma et al., 2014
- Reddi et al., 2018 for
amsgrad
.
Notes:
The default value of 1e-7 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a current good choice is 1.0 or 0.1. Note that since Adam uses the formulation just before Section 2.1 of the Kingma and Ba paper rather than the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon hat" in the paper.
The sparse implementation of this algorithm (used when the gradient is an IndexedSlices object, typically because of
tf.gather
or an embedding lookup in the forward pass) does apply momentum to variable slices even if they were not used in the forward pass (meaning they have a gradient equal to zero). Momentum decay (beta1) is also applied to the entire momentum accumulator. This means that the sparse behavior is equivalent to the dense behavior (in contrast to some momentum implementations which ignore momentum unless a variable slice was actually used).Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Adam(optimizer_v2.OptimizerV2): r"""Optimizer that implements the Adam algorithm. Adam optimization is a stochastic gradient descent method that is based on adaptive estimation of first-order and second-order moments. According to [Kingma et al., 2014](http://arxiv.org/abs/1412.6980), the method is "*computationally efficient, has little memory requirement, invariant to diagonal rescaling of gradients, and is well suited for problems that are large in terms of data/parameters*". Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable that takes no arguments and returns the actual value to use, The learning rate. Defaults to 0.001. beta_1: A float value or a constant float tensor, or a callable that takes no arguments and returns the actual value to use. The exponential decay rate for the 1st moment estimates. Defaults to 0.9. beta_2: A float value or a constant float tensor, or a callable that takes no arguments and returns the actual value to use, The exponential decay rate for the 2nd moment estimates. Defaults to 0.999. epsilon: A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7. amsgrad: Boolean. Whether to apply AMSGrad variant of this algorithm from the paper "On the Convergence of Adam and beyond". Defaults to `False`. name: Optional name for the operations created when applying gradients. Defaults to `"Adam"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Usage: >>> opt = tf.keras.optimizers.Adam(learning_rate=0.1) >>> var1 = tf.Variable(10.0) >>> loss = lambda: (var1 ** 2)/2.0 # d(loss)/d(var1) == var1 >>> step_count = opt.minimize(loss, [var1]).numpy() >>> # The first step is `-learning_rate*sign(grad)` >>> var1.numpy() 9.9 Reference: - [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) - [Reddi et al., 2018]( https://openreview.net/pdf?id=ryQu7f-RZ) for `amsgrad`. Notes: The default value of 1e-7 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a current good choice is 1.0 or 0.1. Note that since Adam uses the formulation just before Section 2.1 of the Kingma and Ba paper rather than the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon hat" in the paper. The sparse implementation of this algorithm (used when the gradient is an IndexedSlices object, typically because of `tf.gather` or an embedding lookup in the forward pass) does apply momentum to variable slices even if they were not used in the forward pass (meaning they have a gradient equal to zero). Momentum decay (beta1) is also applied to the entire momentum accumulator. This means that the sparse behavior is equivalent to the dense behavior (in contrast to some momentum implementations which ignore momentum unless a variable slice was actually used). """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, amsgrad=False, name='Adam', **kwargs): super(Adam, self).__init__(name, **kwargs) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('decay', self._initial_decay) self._set_hyper('beta_1', beta_1) self._set_hyper('beta_2', beta_2) self.epsilon = epsilon or backend_config.epsilon() self.amsgrad = amsgrad def _create_slots(self, var_list): # Create slots for the first and second moments. # Separate for-loops to respect the ordering of slot variables from v1. for var in var_list: self.add_slot(var, 'm') for var in var_list: self.add_slot(var, 'v') if self.amsgrad: for var in var_list: self.add_slot(var, 'vhat') def _prepare_local(self, var_device, var_dtype, apply_state): super(Adam, self)._prepare_local(var_device, var_dtype, apply_state) local_step = tf.cast(self.iterations + 1, var_dtype) beta_1_t = tf.identity(self._get_hyper('beta_1', var_dtype)) beta_2_t = tf.identity(self._get_hyper('beta_2', var_dtype)) beta_1_power = tf.pow(beta_1_t, local_step) beta_2_power = tf.pow(beta_2_t, local_step) lr = (apply_state[(var_device, var_dtype)]['lr_t'] * (tf.sqrt(1 - beta_2_power) / (1 - beta_1_power))) apply_state[(var_device, var_dtype)].update( dict( lr=lr, epsilon=tf.convert_to_tensor( self.epsilon, var_dtype), beta_1_t=beta_1_t, beta_1_power=beta_1_power, one_minus_beta_1_t=1 - beta_1_t, beta_2_t=beta_2_t, beta_2_power=beta_2_power, one_minus_beta_2_t=1 - beta_2_t)) def set_weights(self, weights): params = self.weights # If the weights are generated by Keras V1 optimizer, it includes vhats # even without amsgrad, i.e, V1 optimizer has 3x + 1 variables, while V2 # optimizer has 2x + 1 variables. Filter vhats out for compatibility. num_vars = int((len(params) - 1) / 2) if len(weights) == 3 * num_vars + 1: weights = weights[:len(params)] super(Adam, self).set_weights(weights) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') if not self.amsgrad: return tf.raw_ops.ResourceApplyAdam( var=var.handle, m=m.handle, v=v.handle, beta1_power=coefficients['beta_1_power'], beta2_power=coefficients['beta_2_power'], lr=coefficients['lr_t'], beta1=coefficients['beta_1_t'], beta2=coefficients['beta_2_t'], epsilon=coefficients['epsilon'], grad=grad, use_locking=self._use_locking) else: vhat = self.get_slot(var, 'vhat') return tf.raw_ops.ResourceApplyAdamWithAmsgrad( var=var.handle, m=m.handle, v=v.handle, vhat=vhat.handle, beta1_power=coefficients['beta_1_power'], beta2_power=coefficients['beta_2_power'], lr=coefficients['lr_t'], beta1=coefficients['beta_1_t'], beta2=coefficients['beta_2_t'], epsilon=coefficients['epsilon'], grad=grad, use_locking=self._use_locking) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) # m_t = beta1 * m + (1 - beta1) * g_t m = self.get_slot(var, 'm') m_scaled_g_values = grad * coefficients['one_minus_beta_1_t'] m_t = tf.compat.v1.assign(m, m * coefficients['beta_1_t'], use_locking=self._use_locking) with tf.control_dependencies([m_t]): m_t = self._resource_scatter_add(m, indices, m_scaled_g_values) # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) v = self.get_slot(var, 'v') v_scaled_g_values = (grad * grad) * coefficients['one_minus_beta_2_t'] v_t = tf.compat.v1.assign(v, v * coefficients['beta_2_t'], use_locking=self._use_locking) with tf.control_dependencies([v_t]): v_t = self._resource_scatter_add(v, indices, v_scaled_g_values) if not self.amsgrad: v_sqrt = tf.sqrt(v_t) var_update = tf.compat.v1.assign_sub( var, coefficients['lr'] * m_t / (v_sqrt + coefficients['epsilon']), use_locking=self._use_locking) return tf.group(*[var_update, m_t, v_t]) else: v_hat = self.get_slot(var, 'vhat') v_hat_t = tf.maximum(v_hat, v_t) with tf.control_dependencies([v_hat_t]): v_hat_t = tf.compat.v1.assign( v_hat, v_hat_t, use_locking=self._use_locking) v_hat_sqrt = tf.sqrt(v_hat_t) var_update = tf.compat.v1.assign_sub( var, coefficients['lr'] * m_t / (v_hat_sqrt + coefficients['epsilon']), use_locking=self._use_locking) return tf.group(*[var_update, m_t, v_t, v_hat_t]) def get_config(self): config = super(Adam, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'beta_1': self._serialize_hyperparameter('beta_1'), 'beta_2': self._serialize_hyperparameter('beta_2'), 'epsilon': self.epsilon, 'amsgrad': self.amsgrad, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Adamax (learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax', **kwargs)
-
Optimizer that implements the Adamax algorithm.
It is a variant of Adam based on the infinity norm. Default parameters follow those provided in the paper. Adamax is sometimes superior to adam, specially in models with embeddings.
Initialization:
m = 0 # Initialize initial 1st moment vector v = 0 # Initialize the exponentially weighted infinity norm t = 0 # Initialize timestep
The update rule for parameter
w
with gradientg
is described at the end of section 7.1 of the paper:t += 1 m = beta1 * m + (1 - beta) * g v = max(beta2 * v, abs(g)) current_lr = learning_rate / (1 - beta1 ** t) w = w - current_lr * m / (v + epsilon)
Similarly to
Adam
, the epsilon is added for numerical stability (especially to get rid of division by zero whenv_t == 0
).In contrast to
Adam
, the sparse implementation of this algorithm (used when the gradient is an IndexedSlices object, typically because oftf.gather
or an embedding lookup in the forward pass) only updates variable slices and correspondingm_t
,v_t
terms when that part of the variable was used in the forward pass. This means that the sparse behavior is contrast to the dense behavior (similar to some momentum implementations which ignore momentum unless a variable slice was actually used).Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
. The learning rate. beta_1
- A float value or a constant float tensor. The exponential decay rate for the 1st moment estimates.
beta_2
- A float value or a constant float tensor. The exponential decay rate for the exponentially weighted infinity norm.
epsilon
- A small constant for numerical stability.
name
- Optional name for the operations created when applying gradients.
Defaults to
"Adamax"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Reference
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Adamax(optimizer_v2.OptimizerV2): """Optimizer that implements the Adamax algorithm. It is a variant of Adam based on the infinity norm. Default parameters follow those provided in the paper. Adamax is sometimes superior to adam, specially in models with embeddings. Initialization: ```python m = 0 # Initialize initial 1st moment vector v = 0 # Initialize the exponentially weighted infinity norm t = 0 # Initialize timestep ``` The update rule for parameter `w` with gradient `g` is described at the end of section 7.1 of the paper: ```python t += 1 m = beta1 * m + (1 - beta) * g v = max(beta2 * v, abs(g)) current_lr = learning_rate / (1 - beta1 ** t) w = w - current_lr * m / (v + epsilon) ``` Similarly to `Adam`, the epsilon is added for numerical stability (especially to get rid of division by zero when `v_t == 0`). In contrast to `Adam`, the sparse implementation of this algorithm (used when the gradient is an IndexedSlices object, typically because of `tf.gather` or an embedding lookup in the forward pass) only updates variable slices and corresponding `m_t`, `v_t` terms when that part of the variable was used in the forward pass. This means that the sparse behavior is contrast to the dense behavior (similar to some momentum implementations which ignore momentum unless a variable slice was actually used). Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`. The learning rate. beta_1: A float value or a constant float tensor. The exponential decay rate for the 1st moment estimates. beta_2: A float value or a constant float tensor. The exponential decay rate for the exponentially weighted infinity norm. epsilon: A small constant for numerical stability. name: Optional name for the operations created when applying gradients. Defaults to `"Adamax"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Reference: - [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, name='Adamax', **kwargs): super(Adamax, self).__init__(name, **kwargs) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('decay', self._initial_decay) self._set_hyper('beta_1', beta_1) self._set_hyper('beta_2', beta_2) self.epsilon = epsilon or backend_config.epsilon() def _create_slots(self, var_list): # Separate for-loops to respect the ordering of slot variables from v1. for var in var_list: self.add_slot(var, 'm') # Create slots for the first moments. for var in var_list: self.add_slot(var, 'v') # Create slots for the second moments. def _prepare_local(self, var_device, var_dtype, apply_state): super(Adamax, self)._prepare_local(var_device, var_dtype, apply_state) local_step = tf.cast(self.iterations + 1, var_dtype) beta_1_t = tf.identity(self._get_hyper('beta_1', var_dtype)) beta_2_t = tf.identity(self._get_hyper('beta_2', var_dtype)) beta_1_power = tf.pow(beta_1_t, local_step) lr_t = apply_state[(var_device, var_dtype)]['lr_t'] apply_state[(var_device, var_dtype)].update( dict( neg_scaled_lr=-lr_t / (1 - beta_1_power), epsilon=tf.convert_to_tensor( self.epsilon, var_dtype), beta_1_t=beta_1_t, beta_1_power=beta_1_power, one_minus_beta_1_t=1 - beta_1_t, beta_2_t=beta_2_t, zero=tf.zeros((), dtype=tf.int64))) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') return tf.raw_ops.ResourceApplyAdaMax( var=var.handle, m=m.handle, v=v.handle, beta1_power=coefficients['beta_1_power'], lr=coefficients['lr_t'], beta1=coefficients['beta_1_t'], beta2=coefficients['beta_2_t'], epsilon=coefficients['epsilon'], grad=grad, use_locking=self._use_locking) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) # m_t = beta1 * m + (1 - beta1) * g_t m = self.get_slot(var, 'm') m_slice = tf.gather(m, indices, axis=coefficients['zero']) m_t_slice = (m_slice * coefficients['beta_1_t'] + grad * coefficients['one_minus_beta_1_t']) with tf.control_dependencies([m_t_slice]): m_t = self._resource_scatter_update(m, indices, m_t_slice) # u_t = max(beta2 * u, abs(g_t)) v = self.get_slot(var, 'v') v_slice = tf.gather(v, indices, axis=coefficients['zero']) v_t_slice = tf.maximum(v_slice * coefficients['beta_2_t'], tf.abs(grad)) with tf.control_dependencies([v_t_slice]): v_t = self._resource_scatter_update(v, indices, v_t_slice) # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t var_slice = coefficients['neg_scaled_lr'] * ( m_t_slice / (v_t_slice + coefficients['epsilon'])) with tf.control_dependencies([var_slice]): var_update = self._resource_scatter_add(var, indices, var_slice) return tf.group(*[var_update, m_t, v_t]) def get_config(self): config = super(Adamax, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'beta_1': self._serialize_hyperparameter('beta_1'), 'beta_2': self._serialize_hyperparameter('beta_2'), 'epsilon': self.epsilon, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Ftrl (learning_rate=0.001, learning_rate_power=-0.5, initial_accumulator_value=0.1, l1_regularization_strength=0.0, l2_regularization_strength=0.0, name='Ftrl', l2_shrinkage_regularization_strength=0.0, beta=0.0, **kwargs)
-
Optimizer that implements the FTRL algorithm.
"Follow The Regularized Leader" (FTRL) is an optimization algorithm developed at Google for click-through rate prediction in the early 2010s. It is most suitable for shallow models with large and sparse feature spaces. The algorithm is described by McMahan et al., 2013. The Keras version has support for both online L2 regularization (the L2 regularization described in the paper above) and shrinkage-type L2 regularization (which is the addition of an L2 penalty to the loss function).
Initialization:
n = 0 sigma = 0 z = 0
Update rule for one variable
w
:prev_n = n n = n + g ** 2 sigma = (sqrt(n) - sqrt(prev_n)) / lr z = z + g - sigma * w if abs(z) < lambda_1: w = 0 else: w = (sgn(z) * lambda_1 - z) / ((beta + sqrt(n)) / alpha + lambda_2)
Notation:
lr
is the learning rateg
is the gradient for the variablelambda_1
is the L1 regularization strengthlambda_2
is the L2 regularization strength
Check the documentation for the
l2_shrinkage_regularization_strength
parameter for more details when shrinkage is enabled, in which case gradient is replaced with a gradient with shrinkage.Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
. The learning rate. learning_rate_power
- A float value, must be less or equal to zero. Controls how the learning rate decreases during training. Use zero for a fixed learning rate.
initial_accumulator_value
- The starting value for accumulators. Only zero or positive values are allowed.
l1_regularization_strength
- A float value, must be greater than or equal to zero. Defaults to 0.0.
l2_regularization_strength
- A float value, must be greater than or equal to zero. Defaults to 0.0.
name
- Optional name prefix for the operations created when applying
gradients.
Defaults to
"Ftrl"
. l2_shrinkage_regularization_strength
- A float value, must be greater than or equal to zero. This differs from L2 above in that the L2 above is a stabilization penalty, whereas this L2 shrinkage is a magnitude penalty. When input is sparse shrinkage will only happen on the active weights.
beta
- A float value, representing the beta value from the paper. Defaults to 0.0.
**kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Reference
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Ftrl(optimizer_v2.OptimizerV2): r"""Optimizer that implements the FTRL algorithm. "Follow The Regularized Leader" (FTRL) is an optimization algorithm developed at Google for click-through rate prediction in the early 2010s. It is most suitable for shallow models with large and sparse feature spaces. The algorithm is described by [McMahan et al., 2013](https://research.google.com/pubs/archive/41159.pdf). The Keras version has support for both online L2 regularization (the L2 regularization described in the paper above) and shrinkage-type L2 regularization (which is the addition of an L2 penalty to the loss function). Initialization: ```python n = 0 sigma = 0 z = 0 ``` Update rule for one variable `w`: ```python prev_n = n n = n + g ** 2 sigma = (sqrt(n) - sqrt(prev_n)) / lr z = z + g - sigma * w if abs(z) < lambda_1: w = 0 else: w = (sgn(z) * lambda_1 - z) / ((beta + sqrt(n)) / alpha + lambda_2) ``` Notation: - `lr` is the learning rate - `g` is the gradient for the variable - `lambda_1` is the L1 regularization strength - `lambda_2` is the L2 regularization strength Check the documentation for the `l2_shrinkage_regularization_strength` parameter for more details when shrinkage is enabled, in which case gradient is replaced with a gradient with shrinkage. Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`. The learning rate. learning_rate_power: A float value, must be less or equal to zero. Controls how the learning rate decreases during training. Use zero for a fixed learning rate. initial_accumulator_value: The starting value for accumulators. Only zero or positive values are allowed. l1_regularization_strength: A float value, must be greater than or equal to zero. Defaults to 0.0. l2_regularization_strength: A float value, must be greater than or equal to zero. Defaults to 0.0. name: Optional name prefix for the operations created when applying gradients. Defaults to `"Ftrl"`. l2_shrinkage_regularization_strength: A float value, must be greater than or equal to zero. This differs from L2 above in that the L2 above is a stabilization penalty, whereas this L2 shrinkage is a magnitude penalty. When input is sparse shrinkage will only happen on the active weights. beta: A float value, representing the beta value from the paper. Defaults to 0.0. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Reference: - [McMahan et al., 2013]( https://research.google.com/pubs/archive/41159.pdf) """ def __init__(self, learning_rate=0.001, learning_rate_power=-0.5, initial_accumulator_value=0.1, l1_regularization_strength=0.0, l2_regularization_strength=0.0, name='Ftrl', l2_shrinkage_regularization_strength=0.0, beta=0.0, **kwargs): super(Ftrl, self).__init__(name, **kwargs) if initial_accumulator_value < 0.0: raise ValueError( 'initial_accumulator_value %f needs to be positive or zero' % initial_accumulator_value) if learning_rate_power > 0.0: raise ValueError('learning_rate_power %f needs to be negative or zero' % learning_rate_power) if l1_regularization_strength < 0.0: raise ValueError( 'l1_regularization_strength %f needs to be positive or zero' % l1_regularization_strength) if l2_regularization_strength < 0.0: raise ValueError( 'l2_regularization_strength %f needs to be positive or zero' % l2_regularization_strength) if l2_shrinkage_regularization_strength < 0.0: raise ValueError( 'l2_shrinkage_regularization_strength %f needs to be positive' ' or zero' % l2_shrinkage_regularization_strength) self._set_hyper('learning_rate', learning_rate) self._set_hyper('decay', self._initial_decay) self._set_hyper('learning_rate_power', learning_rate_power) self._set_hyper('l1_regularization_strength', l1_regularization_strength) self._set_hyper('l2_regularization_strength', l2_regularization_strength) self._set_hyper('beta', beta) self._initial_accumulator_value = initial_accumulator_value self._l2_shrinkage_regularization_strength = ( l2_shrinkage_regularization_strength) def _create_slots(self, var_list): # Create the "accum" and "linear" slots. for var in var_list: dtype = var.dtype.base_dtype init = tf.compat.v1.constant_initializer( self._initial_accumulator_value, dtype=dtype) self.add_slot(var, 'accumulator', init) self.add_slot(var, 'linear') def _prepare_local(self, var_device, var_dtype, apply_state): super(Ftrl, self)._prepare_local(var_device, var_dtype, apply_state) apply_state[(var_device, var_dtype)].update( dict( learning_rate_power=tf.identity( self._get_hyper('learning_rate_power', var_dtype)), l1_regularization_strength=tf.identity( self._get_hyper('l1_regularization_strength', var_dtype)), l2_regularization_strength=tf.identity( self._get_hyper('l2_regularization_strength', var_dtype)), beta=tf.identity(self._get_hyper('beta', var_dtype)), l2_shrinkage_regularization_strength=tf.cast( self._l2_shrinkage_regularization_strength, var_dtype))) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) # Adjust L2 regularization strength to include beta to avoid the underlying # TensorFlow ops needing to include it. adjusted_l2_regularization_strength = ( coefficients['l2_regularization_strength'] + coefficients['beta'] / (2. * coefficients['lr_t'])) accum = self.get_slot(var, 'accumulator') linear = self.get_slot(var, 'linear') if self._l2_shrinkage_regularization_strength <= 0.0: return tf.raw_ops.ResourceApplyFtrl( var=var.handle, accum=accum.handle, linear=linear.handle, grad=grad, lr=coefficients['lr_t'], l1=coefficients['l1_regularization_strength'], l2=adjusted_l2_regularization_strength, lr_power=coefficients['learning_rate_power'], use_locking=self._use_locking) else: return tf.raw_ops.ResourceApplyFtrlV2( var=var.handle, accum=accum.handle, linear=linear.handle, grad=grad, lr=coefficients['lr_t'], l1=coefficients['l1_regularization_strength'], l2=adjusted_l2_regularization_strength, l2_shrinkage=coefficients['l2_shrinkage_regularization_strength'], lr_power=coefficients['learning_rate_power'], use_locking=self._use_locking) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) # Adjust L2 regularization strength to include beta to avoid the underlying # TensorFlow ops needing to include it. adjusted_l2_regularization_strength = ( coefficients['l2_regularization_strength'] + coefficients['beta'] / (2. * coefficients['lr_t'])) accum = self.get_slot(var, 'accumulator') linear = self.get_slot(var, 'linear') if self._l2_shrinkage_regularization_strength <= 0.0: return tf.raw_ops.ResourceSparseApplyFtrl( var=var.handle, accum=accum.handle, linear=linear.handle, grad=grad, indices=indices, lr=coefficients['lr_t'], l1=coefficients['l1_regularization_strength'], l2=adjusted_l2_regularization_strength, lr_power=coefficients['learning_rate_power'], use_locking=self._use_locking) else: return tf.raw_ops.ResourceSparseApplyFtrlV2( var=var.handle, accum=accum.handle, linear=linear.handle, grad=grad, indices=indices, lr=coefficients['lr_t'], l1=coefficients['l1_regularization_strength'], l2=adjusted_l2_regularization_strength, l2_shrinkage=coefficients['l2_shrinkage_regularization_strength'], lr_power=coefficients['learning_rate_power'], use_locking=self._use_locking) def get_config(self): config = super(Ftrl, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'initial_accumulator_value': self._initial_accumulator_value, 'learning_rate_power': self._serialize_hyperparameter('learning_rate_power'), 'l1_regularization_strength': self._serialize_hyperparameter('l1_regularization_strength'), 'l2_regularization_strength': self._serialize_hyperparameter('l2_regularization_strength'), 'beta': self._serialize_hyperparameter('beta'), 'l2_shrinkage_regularization_strength': self._l2_shrinkage_regularization_strength, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Nadam (learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam', **kwargs)
-
Optimizer that implements the NAdam algorithm. Much like Adam is essentially RMSprop with momentum, Nadam is Adam with Nesterov momentum.
Args
learning_rate
- A Tensor or a floating point value. The learning rate.
beta_1
- A float value or a constant float tensor. The exponential decay rate for the 1st moment estimates.
beta_2
- A float value or a constant float tensor. The exponential decay rate for the exponentially weighted infinity norm.
epsilon
- A small constant for numerical stability.
name
- Optional name for the operations created when applying gradients.
Defaults to
"Nadam"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Usage Example:
opt = tf.keras.optimizers.Nadam(learning_rate=0.2) var1 = tf.Variable(10.0) loss = lambda: (var1 ** 2) / 2.0 step_count = opt.minimize(loss, [var1]).numpy() "{:.1f}".format(var1.numpy()) 9.8
Reference
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class Nadam(optimizer_v2.OptimizerV2): r"""Optimizer that implements the NAdam algorithm. Much like Adam is essentially RMSprop with momentum, Nadam is Adam with Nesterov momentum. Args: learning_rate: A Tensor or a floating point value. The learning rate. beta_1: A float value or a constant float tensor. The exponential decay rate for the 1st moment estimates. beta_2: A float value or a constant float tensor. The exponential decay rate for the exponentially weighted infinity norm. epsilon: A small constant for numerical stability. name: Optional name for the operations created when applying gradients. Defaults to `"Nadam"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Usage Example: >>> opt = tf.keras.optimizers.Nadam(learning_rate=0.2) >>> var1 = tf.Variable(10.0) >>> loss = lambda: (var1 ** 2) / 2.0 >>> step_count = opt.minimize(loss, [var1]).numpy() >>> "{:.1f}".format(var1.numpy()) 9.8 Reference: - [Dozat, 2015](http://cs229.stanford.edu/proj2015/054_report.pdf). """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, name='Nadam', **kwargs): # Backwards compatibility with keras NAdam optimizer. kwargs['decay'] = kwargs.pop('schedule_decay', 0.004) learning_rate = kwargs.get('lr', learning_rate) if isinstance(learning_rate, learning_rate_schedule.LearningRateSchedule): raise ValueError('The Nadam optimizer does not support ' 'tf.keras.optimizers.LearningRateSchedules as the ' 'learning rate.') super(Nadam, self).__init__(name, **kwargs) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('decay', self._initial_decay) self._set_hyper('beta_1', beta_1) self._set_hyper('beta_2', beta_2) self.epsilon = epsilon or backend_config.epsilon() self._m_cache = None def _create_slots(self, var_list): var_dtype = var_list[0].dtype.base_dtype if self._m_cache is None: self._m_cache = self.add_weight( 'momentum_cache', shape=[], dtype=var_dtype, initializer='ones', trainable=False, aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) self._weights.append(self._m_cache) # Separate for-loops to respect the ordering of slot variables from v1. for var in var_list: # Create slots for the first moments. self.add_slot(var, 'm') for var in var_list: # Create slots for the second moments. self.add_slot(var, 'v') def _prepare_local(self, var_device, var_dtype, apply_state): lr_t = tf.identity(self._get_hyper('learning_rate', var_dtype)) beta_1_t = tf.identity(self._get_hyper('beta_1', var_dtype)) beta_2_t = tf.identity(self._get_hyper('beta_2', var_dtype)) local_step = tf.cast(self.iterations + 1, var_dtype) next_step = tf.cast(self.iterations + 2, var_dtype) decay_base = tf.cast(0.96, var_dtype) m_t = beta_1_t * (1. - 0.5 * ( tf.pow(decay_base, self._initial_decay * local_step))) m_t_1 = beta_1_t * (1. - 0.5 * ( tf.pow(decay_base, self._initial_decay * next_step))) m_schedule_new = tf.cast(self._m_cache_read, var_dtype) * m_t if var_dtype is self._m_cache.dtype: m_schedule_new = tf.identity(tf.compat.v1.assign( self._m_cache, m_schedule_new, use_locking=self._use_locking)) m_schedule_next = m_schedule_new * m_t_1 apply_state[(var_device, var_dtype)] = dict( lr_t=lr_t, neg_lr_t=-lr_t, # pylint: disable=invalid-unary-operand-type epsilon=tf.convert_to_tensor(self.epsilon, var_dtype), beta_1_t=beta_1_t, beta_2_t=beta_2_t, m_t=m_t, m_t_1=m_t_1, one_minus_beta_1_t=1 - beta_1_t, one_minus_beta_2_t=1 - beta_2_t, one_minus_m_t=1. - m_t, one_minus_m_schedule_new=1. - m_schedule_new, one_minus_m_schedule_next=1. - m_schedule_next, v_t_prime_denominator=1. - tf.pow(beta_2_t, local_step), ) def _prepare(self, var_list): # Get the value of the momentum cache before starting to apply gradients. self._m_cache_read = tf.identity(self._m_cache) return super(Nadam, self)._prepare(var_list) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') g_prime = grad / coefficients['one_minus_m_schedule_new'] m_t = (coefficients['beta_1_t'] * m + coefficients['one_minus_beta_1_t'] * grad) m_t = tf.compat.v1.assign(m, m_t, use_locking=self._use_locking) m_t_prime = m_t / coefficients['one_minus_m_schedule_next'] v_t = (coefficients['beta_2_t'] * v + coefficients['one_minus_beta_2_t'] * tf.square(grad)) v_t = tf.compat.v1.assign(v, v_t, use_locking=self._use_locking) v_t_prime = v_t / coefficients['v_t_prime_denominator'] m_t_bar = (coefficients['one_minus_m_t'] * g_prime + coefficients['m_t_1'] * m_t_prime) var_t = var - coefficients['lr_t'] * m_t_bar / ( tf.sqrt(v_t_prime) + coefficients['epsilon']) return tf.compat.v1.assign(var, var_t, use_locking=self._use_locking).op def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') g_prime = grad / coefficients['one_minus_m_schedule_new'] # m_t = beta1 * m + (1 - beta1) * g_t m_scaled_g_values = grad * coefficients['one_minus_beta_1_t'] m_t = tf.compat.v1.assign(m, m * coefficients['beta_1_t'], use_locking=self._use_locking) with tf.control_dependencies([m_t]): m_t = self._resource_scatter_add(m, indices, m_scaled_g_values) m_t_slice = tf.gather(m_t, indices) m_t_prime = m_t_slice / coefficients['one_minus_m_schedule_next'] m_t_bar = (coefficients['one_minus_m_t'] * g_prime + coefficients['m_t_1'] * m_t_prime) # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) v_scaled_g_values = (grad * grad) * coefficients['one_minus_beta_2_t'] v_t = tf.compat.v1.assign(v, v * coefficients['beta_2_t'], use_locking=self._use_locking) with tf.control_dependencies([v_t]): v_t = self._resource_scatter_add(v, indices, v_scaled_g_values) v_t_slice = tf.gather(v_t, indices) v_t_prime = v_t_slice / coefficients['v_t_prime_denominator'] v_prime_sqrt_plus_eps = tf.sqrt(v_t_prime) + coefficients['epsilon'] var_update = self._resource_scatter_add( var, indices, coefficients['neg_lr_t'] * m_t_bar / v_prime_sqrt_plus_eps) return tf.group(*[var_update, m_t_bar, v_t]) def get_config(self): config = super(Nadam, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), 'decay': self._initial_decay, 'beta_1': self._serialize_hyperparameter('beta_1'), 'beta_2': self._serialize_hyperparameter('beta_2'), 'epsilon': self.epsilon, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class Optimizer (name, gradient_aggregator=None, gradient_transformers=None, **kwargs)
-
Base class for Keras optimizers.
You should not use this class directly, but instead instantiate one of its subclasses such as
tf.keras.optimizers.SGD
,tf.keras.optimizers.Adam
, etc.Usage
# Create an optimizer with the desired parameters. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # `loss` is a callable that takes no argument and returns the value # to minimize. loss = lambda: 3 * var1 * var1 + 2 * var2 * var2 # In graph mode, returns op that minimizes the loss by updating the listed # variables. opt_op = opt.minimize(loss, var_list=[var1, var2]) opt_op.run() # In eager mode, simply call minimize to update the list of variables. opt.minimize(loss, var_list=[var1, var2])
Usage in custom training loops
In Keras models, sometimes variables are created when the model is first called, instead of construction time. Examples include 1) sequential models without input shape pre-defined, or 2) subclassed models. Pass var_list as callable in these cases.
Example:
opt = tf.keras.optimizers.SGD(learning_rate=0.1) model = tf.keras.Sequential() model.add(tf.keras.layers.Dense(num_hidden, activation='relu')) model.add(tf.keras.layers.Dense(num_classes, activation='sigmoid')) loss_fn = lambda: tf.keras.losses.mse(model(input), output) var_list_fn = lambda: model.trainable_weights for input, output in data: opt.minimize(loss_fn, var_list_fn)
Processing gradients before applying them
Calling
minimize()
takes care of both computing the gradients and applying them to the variables. If you want to process the gradients before applying them you can instead use the optimizer in three steps:- Compute the gradients with
tf.GradientTape
. - Process the gradients as you wish.
- Apply the processed gradients with
apply_gradients()
.
Example:
# Create an optimizer. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # Compute the gradients for a list of variables. with tf.GradientTape() as tape: loss = <call_loss_function> vars = <list_of_variables> grads = tape.gradient(loss, vars) # Process the gradients, for example cap them, etc. # capped_grads = [MyCapper(g) for g in grads] processed_grads = [process_gradient(g) for g in grads] # Ask the optimizer to apply the processed gradients. opt.apply_gradients(zip(processed_grads, var_list))
Use with
tf.distribute.Strategy
This optimizer class is
tf.distribute.Strategy
aware, which means it automatically sums gradients across all replicas. To average gradients, you divide your loss by the global batch size, which is done automatically if you usetf.keras
built-in training or evaluation loops. See thereduction
argument of your loss which should be set totf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE
for averaging ortf.keras.losses.Reduction.SUM
for not.To aggregate gradients yourself, call
apply_gradients
withexperimental_aggregate_gradients
set to False. This is useful if you need to process aggregated gradients.If you are not using these and you want to average gradients, you should use
tf.math.reduce_sum
to add up your per-example losses and then divide by the global batch size. Note that when usingtf.distribute.Strategy
, the first component of a tensor's shape is the replica-local batch size, which is off by a factor equal to the number of replicas being used to compute a single step. As a result, usingtf.math.reduce_mean
will give the wrong answer, resulting in gradients that can be many times too big.Variable Constraints
All Keras optimizers respect variable constraints. If constraint function is passed to any variable, the constraint will be applied to the variable after the gradient has been applied to the variable. Important: If gradient is sparse tensor, variable constraint is not supported.
Thread Compatibility
The entire optimizer is currently thread compatible, not thread-safe. The user needs to perform synchronization if necessary.
Slots
Many optimizer subclasses, such as
Adam
andAdagrad
allocate and manage additional variables associated with the variables to train. These are called Slots. Slots have names and you can ask the optimizer for the names of the slots that it uses. Once you have a slot name you can ask the optimizer for the variable it created to hold the slot value.This can be useful if you want to log debug a training algorithm, report stats about the slots, etc.
Hyperparameters
These are arguments passed to the optimizer subclass constructor (the
__init__
method), and then passed toself._set_hyper()
. They can be either regular Python values (like 1.0), tensors, or callables. If they are callable, the callable will be called duringapply_gradients()
to get the value for the hyper parameter.Hyperparameters can be overwritten through user code:
Example:
# Create an optimizer with the desired parameters. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # `loss` is a callable that takes no argument and returns the value # to minimize. loss = lambda: 3 * var1 + 2 * var2 # In eager mode, simply call minimize to update the list of variables. opt.minimize(loss, var_list=[var1, var2]) # update learning rate opt.learning_rate = 0.05 opt.minimize(loss, var_list=[var1, var2])
Callable learning rate
Optimizer accepts a callable learning rate in two ways. The first way is through built-in or customized
tf.keras.optimizers.schedules.LearningRateSchedule
. The schedule will be called on each iteration withschedule(iteration)
, atf.Variable
owned by the optimizer.Example:
>>> var = tf.Variable(np.random.random(size=(1,))) >>> learning_rate = tf.keras.optimizers.schedules.ExponentialDecay( ... initial_learning_rate=.01, decay_steps=20, decay_rate=.1) >>> opt = tf.keras.optimizers.SGD(learning_rate=learning_rate) >>> loss = lambda: 3 * var >>> opt.minimize(loss, var_list=[var]) <tf.Variable...
The second way is through a callable function that does not accept any arguments.
Example:
>>> var = tf.Variable(np.random.random(size=(1,))) >>> def lr_callable(): ... return .1 >>> opt = tf.keras.optimizers.SGD(learning_rate=lr_callable) >>> loss = lambda: 3 * var >>> opt.minimize(loss, var_list=[var]) <tf.Variable...
Creating a custom optimizer
If you intend to create your own optimization algorithm, simply inherit from this class and override the following methods:
_resource_apply_dense
(update variable given gradient tensor is a densetf.Tensor
)_resource_apply_sparse
(update variable given gradient tensor is a sparsetf.IndexedSlices
. The most common way for this to happen is if you are taking the gradient through atf.gather
.)_create_slots
(if your optimizer algorithm requires additional variables)get_config
(serialization of the optimizer, include all hyper parameters)
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class OptimizerV2(tf.__internal__.tracking.Trackable): """Base class for Keras optimizers. You should not use this class directly, but instead instantiate one of its subclasses such as `tf.keras.optimizers.SGD`, `tf.keras.optimizers.Adam`, etc. ### Usage ```python # Create an optimizer with the desired parameters. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # `loss` is a callable that takes no argument and returns the value # to minimize. loss = lambda: 3 * var1 * var1 + 2 * var2 * var2 # In graph mode, returns op that minimizes the loss by updating the listed # variables. opt_op = opt.minimize(loss, var_list=[var1, var2]) opt_op.run() # In eager mode, simply call minimize to update the list of variables. opt.minimize(loss, var_list=[var1, var2]) ``` ### Usage in custom training loops In Keras models, sometimes variables are created when the model is first called, instead of construction time. Examples include 1) sequential models without input shape pre-defined, or 2) subclassed models. Pass var_list as callable in these cases. Example: ```python opt = tf.keras.optimizers.SGD(learning_rate=0.1) model = tf.keras.Sequential() model.add(tf.keras.layers.Dense(num_hidden, activation='relu')) model.add(tf.keras.layers.Dense(num_classes, activation='sigmoid')) loss_fn = lambda: tf.keras.losses.mse(model(input), output) var_list_fn = lambda: model.trainable_weights for input, output in data: opt.minimize(loss_fn, var_list_fn) ``` ### Processing gradients before applying them Calling `minimize()` takes care of both computing the gradients and applying them to the variables. If you want to process the gradients before applying them you can instead use the optimizer in three steps: 1. Compute the gradients with `tf.GradientTape`. 2. Process the gradients as you wish. 3. Apply the processed gradients with `apply_gradients()`. Example: ```python # Create an optimizer. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # Compute the gradients for a list of variables. with tf.GradientTape() as tape: loss = <call_loss_function> vars = <list_of_variables> grads = tape.gradient(loss, vars) # Process the gradients, for example cap them, etc. # capped_grads = [MyCapper(g) for g in grads] processed_grads = [process_gradient(g) for g in grads] # Ask the optimizer to apply the processed gradients. opt.apply_gradients(zip(processed_grads, var_list)) ``` ### Use with `tf.distribute.Strategy` This optimizer class is `tf.distribute.Strategy` aware, which means it automatically sums gradients across all replicas. To average gradients, you divide your loss by the global batch size, which is done automatically if you use `tf.keras` built-in training or evaluation loops. See the `reduction` argument of your loss which should be set to `tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE` for averaging or `tf.keras.losses.Reduction.SUM` for not. To aggregate gradients yourself, call `apply_gradients` with `experimental_aggregate_gradients` set to False. This is useful if you need to process aggregated gradients. If you are not using these and you want to average gradients, you should use `tf.math.reduce_sum` to add up your per-example losses and then divide by the global batch size. Note that when using `tf.distribute.Strategy`, the first component of a tensor's shape is the *replica-local* batch size, which is off by a factor equal to the number of replicas being used to compute a single step. As a result, using `tf.math.reduce_mean` will give the wrong answer, resulting in gradients that can be many times too big. ### Variable Constraints All Keras optimizers respect variable constraints. If constraint function is passed to any variable, the constraint will be applied to the variable after the gradient has been applied to the variable. Important: If gradient is sparse tensor, variable constraint is not supported. ### Thread Compatibility The entire optimizer is currently thread compatible, not thread-safe. The user needs to perform synchronization if necessary. ### Slots Many optimizer subclasses, such as `Adam` and `Adagrad` allocate and manage additional variables associated with the variables to train. These are called <i>Slots</i>. Slots have names and you can ask the optimizer for the names of the slots that it uses. Once you have a slot name you can ask the optimizer for the variable it created to hold the slot value. This can be useful if you want to log debug a training algorithm, report stats about the slots, etc. ### Hyperparameters These are arguments passed to the optimizer subclass constructor (the `__init__` method), and then passed to `self._set_hyper()`. They can be either regular Python values (like 1.0), tensors, or callables. If they are callable, the callable will be called during `apply_gradients()` to get the value for the hyper parameter. Hyperparameters can be overwritten through user code: Example: ```python # Create an optimizer with the desired parameters. opt = tf.keras.optimizers.SGD(learning_rate=0.1) # `loss` is a callable that takes no argument and returns the value # to minimize. loss = lambda: 3 * var1 + 2 * var2 # In eager mode, simply call minimize to update the list of variables. opt.minimize(loss, var_list=[var1, var2]) # update learning rate opt.learning_rate = 0.05 opt.minimize(loss, var_list=[var1, var2]) ``` ### Callable learning rate Optimizer accepts a callable learning rate in two ways. The first way is through built-in or customized `tf.keras.optimizers.schedules.LearningRateSchedule`. The schedule will be called on each iteration with `schedule(iteration)`, a `tf.Variable` owned by the optimizer. Example: >>> var = tf.Variable(np.random.random(size=(1,))) >>> learning_rate = tf.keras.optimizers.schedules.ExponentialDecay( ... initial_learning_rate=.01, decay_steps=20, decay_rate=.1) >>> opt = tf.keras.optimizers.SGD(learning_rate=learning_rate) >>> loss = lambda: 3 * var >>> opt.minimize(loss, var_list=[var]) <tf.Variable... The second way is through a callable function that does not accept any arguments. Example: >>> var = tf.Variable(np.random.random(size=(1,))) >>> def lr_callable(): ... return .1 >>> opt = tf.keras.optimizers.SGD(learning_rate=lr_callable) >>> loss = lambda: 3 * var >>> opt.minimize(loss, var_list=[var]) <tf.Variable... ### Creating a custom optimizer If you intend to create your own optimization algorithm, simply inherit from this class and override the following methods: - `_resource_apply_dense` (update variable given gradient tensor is a dense `tf.Tensor`) - `_resource_apply_sparse` (update variable given gradient tensor is a sparse `tf.IndexedSlices`. The most common way for this to happen is if you are taking the gradient through a `tf.gather`.) - `_create_slots` (if your optimizer algorithm requires additional variables) - `get_config` (serialization of the optimizer, include all hyper parameters) """ # Subclasses should set this to True unless they override `apply_gradients` # with a version that does not have the `experimental_aggregate_gradients` # argument. Older versions of Keras did not have this argument so custom # optimizers may have overridden `apply_gradients` without the # `experimental_aggregate_gradients` argument. Keras only passes # `experimental_aggregate_gradients` if this attribute is True. # Note: This attribute will likely be removed in an upcoming release. _HAS_AGGREGATE_GRAD = False def __init__(self, name, gradient_aggregator=None, gradient_transformers=None, **kwargs): """Create a new Optimizer. This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead. This class is stateful and thread-compatible. Example of custom gradient transformations: ```python def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer]) ``` Args: name: String. The name to use for momentum accumulator weights created by the optimizer. gradient_aggregator: The function to use to aggregate gradients across devices (when using `tf.distribute.Strategy`). If `None`, defaults to summing the gradients across devices. The function should accept and return a list of `(gradient, variable)` tuples. gradient_transformers: Optional. List of functions to use to transform gradients before applying updates to Variables. The functions are applied after `gradient_aggregator`. The functions should accept and return a list of `(gradient, variable)` tuples. **kwargs: keyword arguments. Allowed arguments are `clipvalue`, `clipnorm`, `global_clipnorm`. If `clipvalue` (float) is set, the gradient of each weight is clipped to be no higher than this value. If `clipnorm` (float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. If `global_clipnorm` (float) is set the gradient of all weights is clipped so that their global norm is no higher than this value. Raises: ValueError: in case of any invalid argument. """ # Instrument optimizer usages keras_optimizers_gauge.get_cell(self.__class__.__name__).set(True) allowed_kwargs = {"clipnorm", "clipvalue", "lr", "decay", "global_clipnorm"} for k in kwargs: if k not in allowed_kwargs: raise TypeError("Unexpected keyword argument " "passed to optimizer: " + str(k)) # checks that all keyword arguments are non-negative. if kwargs[k] is not None and kwargs[k] < 0: raise ValueError("Expected {} >= 0, received: {}".format(k, kwargs[k])) if k == "lr": warnings.warn( "The `lr` argument is deprecated, use `learning_rate` instead.") self._use_locking = True self._init_set_name(name) self._hyper = {} # dict: {variable name : {slot name : variable}} self._slots = {} self._slot_names = [] self._weights = [] self._iterations = None # For implementing Trackable. Stores information about how to restore # slot variables which have not yet been created # (trackable._CheckpointPosition objects). # {slot_name : # {_var_key(variable_to_train): [checkpoint_position, ... ], ... }, # ... } self._deferred_slot_restorations = {} decay = kwargs.pop("decay", 0.0) if decay < 0.: raise ValueError("decay cannot be less than 0: {}".format(decay)) self._initial_decay = decay self._hypers_created = False # Store the distribution strategy object if the optimizer is created inside # strategy scope, so it could be used to create variables later. if tf.distribute.has_strategy(): self._distribution_strategy = tf.distribute.get_strategy() else: self._distribution_strategy = None # Configure gradient transformations. if gradient_aggregator is None: gradient_aggregator = optimizer_utils.all_reduce_sum_gradients self.gradient_aggregator = gradient_aggregator if gradient_transformers is None: gradient_transformers = [] self.gradient_transformers = gradient_transformers self.clipnorm = kwargs.pop("clipnorm", None) self.global_clipnorm = kwargs.pop("global_clipnorm", None) if self.clipnorm is not None and self.global_clipnorm is not None: raise ValueError("Cannot accept both `clipnorm` and `global_clipnorm`, " "passed `clipnorm` {}, `global_clipnorm` {}".format( self.clipnorm, self.global_clipnorm)) self.clipvalue = kwargs.pop("clipvalue", None) @property def clipnorm(self): """`float` or `None`. If set, clips gradients to a maximum norm.""" return self._clipnorm @property def global_clipnorm(self): """`float` or `None`. If set, clips gradients to a maximum norm.""" return self._global_clipnorm @clipnorm.setter def clipnorm(self, val): if val is not None and self.gradient_transformers: raise ValueError("`clipnorm` cannot be set when `gradient_transformers` " "is set. Instead, use the `gradient_transformers` to " "specify clipping and other transformations.") self._clipnorm = val self._clipnorm_fn = optimizer_utils.make_gradient_clipnorm_fn( self._clipnorm) @global_clipnorm.setter def global_clipnorm(self, val): if val is not None and self.gradient_transformers: raise ValueError("`clipnorm` cannot be set when `gradient_transformers` " "is set. Instead, use the `gradient_transformers` to " "specify clipping and other transformations.") self._global_clipnorm = val self._global_clipnorm_fn = optimizer_utils.make_global_gradient_clipnorm_fn( self._global_clipnorm) @property def clipvalue(self): """`float` or `None`. If set, clips gradients to a maximum value.""" return self._clipvalue @clipvalue.setter def clipvalue(self, val): if val is not None and self.gradient_transformers: raise ValueError("`clipvalue` cannot be set when `gradient_transformers` " "is set. Instead, use the `gradient_transformers` to " "specify clipping and other transformations.") self._clipvalue = val self._clipvalue_fn = optimizer_utils.make_gradient_clipvalue_fn( self._clipvalue) def _transform_loss(self, loss): """Called in `.minimize` to transform loss before computing gradients.""" return loss def _get_gradients(self, tape, loss, var_list, grad_loss=None): """Called in `minimize` to compute gradients from loss.""" grads = tape.gradient(loss, var_list, grad_loss) return list(zip(grads, var_list)) def _transform_unaggregated_gradients(self, grads_and_vars): """Called in `apply_gradients` before gradient aggregation.""" return grads_and_vars def _aggregate_gradients(self, grads_and_vars): """Called in `apply_gradients` to aggregate gradients across devices. Note that user subclasses may override this, so the interface should not be changed. Args: grads_and_vars: List of (gradient, variable) pairs. Returns: A list of (aggregrated_gradient, variable) pairs. By default, this calls `self.gradient_aggregator`. """ return self.gradient_aggregator(grads_and_vars) def _transform_gradients(self, grads_and_vars): """Called in `apply_gradients` after aggregation.""" if self._clipvalue is not None: grads_and_vars = self._clipvalue_fn(grads_and_vars) if self._clipnorm is not None: grads_and_vars = self._clipnorm_fn(grads_and_vars) if self._global_clipnorm is not None: grads_and_vars = self._global_clipnorm_fn(grads_and_vars) for fn in self.gradient_transformers: grads_and_vars = fn(grads_and_vars) return grads_and_vars def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None): """Minimize `loss` by updating `var_list`. This method simply computes gradient using `tf.GradientTape` and calls `apply_gradients()`. If you want to process the gradient before applying then call `tf.GradientTape` and `apply_gradients()` explicitly instead of using this function. Args: loss: `Tensor` or callable. If a callable, `loss` should take no arguments and return the value to minimize. If a `Tensor`, the `tape` argument must be passed. var_list: list or tuple of `Variable` objects to update to minimize `loss`, or a callable returning the list or tuple of `Variable` objects. Use callable when the variable list would otherwise be incomplete before `minimize` since the variables are created at the first time `loss` is called. grad_loss: (Optional). A `Tensor` holding the gradient computed for `loss`. name: (Optional) str. Name for the returned operation. tape: (Optional) `tf.GradientTape`. If `loss` is provided as a `Tensor`, the tape that computed the `loss` must be provided. Returns: An `Operation` that updates the variables in `var_list`. The `iterations` will be automatically increased by 1. Raises: ValueError: If some of the variables are not `Variable` objects. """ grads_and_vars = self._compute_gradients( loss, var_list=var_list, grad_loss=grad_loss, tape=tape) return self.apply_gradients(grads_and_vars, name=name) def _compute_gradients(self, loss, var_list, grad_loss=None, tape=None): """Compute gradients of `loss` for the variables in `var_list`. This is the first part of `minimize()`. It returns a list of (gradient, variable) pairs where "gradient" is the gradient for "variable". Note that "gradient" can be a `Tensor`, an `IndexedSlices`, or `None` if there is no gradient for the given variable. Args: loss: `Tensor` or callable. If a callable, `loss` should take no arguments and return the value to minimize. If a `Tensor`, the `tape` argument must be passed. var_list: list or tuple of `Variable` objects to update to minimize `loss`, or a callable returning the list or tuple of `Variable` objects. Use callable when the variable list would otherwise be incomplete before `minimize` and the variables are created at the first time when `loss` is called. grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. tape: (Optional) `tf.GradientTape`. If `loss` is provided as a `Tensor`, the tape that computed the `loss` must be provided. Returns: A list of (gradient, variable) pairs. Variable is always present, but gradient can be `None`. Raises: TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid, or var_list is None. """ # TODO(joshl): Test that we handle weight decay in a reasonable way. if not callable(loss) and tape is None: raise ValueError("`tape` is required when a `Tensor` loss is passed.") tape = tape if tape is not None else tf.GradientTape() if callable(loss): with tape: if not callable(var_list): tape.watch(var_list) loss = loss() if callable(var_list): var_list = var_list() with tape: loss = self._transform_loss(loss) var_list = tf.nest.flatten(var_list) with tf.name_scope(self._name + "/gradients"): grads_and_vars = self._get_gradients(tape, loss, var_list, grad_loss) self._assert_valid_dtypes([ v for g, v in grads_and_vars if g is not None and v.dtype != tf.resource ]) return grads_and_vars def apply_gradients(self, grads_and_vars, name=None, experimental_aggregate_gradients=True): """Apply gradients to variables. This is the second part of `minimize()`. It returns an `Operation` that applies gradients. The method sums gradients from all replicas in the presence of `tf.distribute.Strategy` by default. You can aggregate gradients yourself by passing `experimental_aggregate_gradients=False`. Example: ```python grads = tape.gradient(loss, vars) grads = tf.distribute.get_replica_context().all_reduce('sum', grads) # Processing aggregated gradients. optimizer.apply_gradients(zip(grads, vars), experimental_aggregate_gradients=False) ``` Args: grads_and_vars: List of (gradient, variable) pairs. name: Optional name for the returned operation. Default to the name passed to the `Optimizer` constructor. experimental_aggregate_gradients: Whether to sum gradients from different replicas in the presense of `tf.distribute.Strategy`. If False, it's user responsibility to aggregate the gradients. Default to True. Returns: An `Operation` that applies the specified gradients. The `iterations` will be automatically increased by 1. Raises: TypeError: If `grads_and_vars` is malformed. ValueError: If none of the variables have gradients. RuntimeError: If called in a cross-replica context. """ grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) var_list = [v for (_, v) in grads_and_vars] with tf.name_scope(self._name): # Create iteration if necessary. with tf.init_scope(): self._create_all_weights(var_list) if not grads_and_vars: # Distribution strategy does not support reducing an empty list of # gradients return tf.no_op() if tf.distribute.in_cross_replica_context(): raise RuntimeError( "`apply_gradients() cannot be called in cross-replica context. " "Use `tf.distribute.Strategy.run` to enter replica " "context.") strategy = tf.distribute.get_strategy() if (not experimental_aggregate_gradients and strategy and isinstance(strategy, (tf.compat.v1.distribute.experimental.ParameterServerStrategy, tf.distribute.experimental.ParameterServerStrategy, tf.distribute.experimental.CentralStorageStrategy, tf.compat.v1.distribute.experimental.CentralStorageStrategy))): raise NotImplementedError( "`experimental_aggregate_gradients=False is not supported for " "ParameterServerStrategy and CentralStorageStrategy") apply_state = self._prepare(var_list) if experimental_aggregate_gradients: grads_and_vars = self._transform_unaggregated_gradients(grads_and_vars) grads_and_vars = self._aggregate_gradients(grads_and_vars) grads_and_vars = self._transform_gradients(grads_and_vars) if optimizer_utils.strategy_supports_no_merge_call(): return self._distributed_apply(strategy, grads_and_vars, name, apply_state) else: return tf.distribute.get_replica_context().merge_call( functools.partial(self._distributed_apply, apply_state=apply_state), args=(grads_and_vars,), kwargs={ "name": name, }) def _distributed_apply(self, distribution, grads_and_vars, name, apply_state): """`apply_gradients` using a `DistributionStrategy`.""" def apply_grad_to_update_var(var, grad): """Apply gradient to variable.""" if isinstance(var, tf.Tensor): raise NotImplementedError("Trying to update a Tensor ", var) apply_kwargs = {} if isinstance(grad, tf.IndexedSlices): if var.constraint is not None: raise RuntimeError( "Cannot use a constraint function on a sparse variable.") if "apply_state" in self._sparse_apply_args: apply_kwargs["apply_state"] = apply_state return self._resource_apply_sparse_duplicate_indices( grad.values, var, grad.indices, **apply_kwargs) if "apply_state" in self._dense_apply_args: apply_kwargs["apply_state"] = apply_state update_op = self._resource_apply_dense(grad, var, **apply_kwargs) if var.constraint is not None: with tf.control_dependencies([update_op]): return var.assign(var.constraint(var)) else: return update_op eagerly_outside_functions = tf.compat.v1.executing_eagerly_outside_functions() update_ops = [] with name_scope_only_in_function_or_graph(name or self._name): for grad, var in grads_and_vars: # Colocate the update with variables to avoid unnecessary communication # delays. See b/136304694. with distribution.extended.colocate_vars_with(var): with name_scope_only_in_function_or_graph( "update" if eagerly_outside_functions else "update_" + var.op.name): update_op = distribution.extended.update( var, apply_grad_to_update_var, args=(grad,), group=False) if tf.distribute.in_cross_replica_context(): # In cross-replica context, extended.update returns a list of # update ops from all replicas (group=False). update_ops.extend(update_op) else: # In replica context, extended.update return the single update op # of current replica. update_ops.append(update_op) any_symbolic = any(isinstance(i, tf.Operation) or tf_utils.is_symbolic_tensor(i) for i in update_ops) if not tf.executing_eagerly() or any_symbolic: # If the current context is graph mode or any of the update ops are # symbolic then the step update should be carried out under a graph # context. (eager updates execute immediately) with backend._current_graph(update_ops).as_default(): # pylint: disable=protected-access with tf.control_dependencies([tf.group(update_ops)]): return self._iterations.assign_add(1, read_value=False) return self._iterations.assign_add(1) def get_gradients(self, loss, params): """Returns gradients of `loss` with respect to `params`. Should be used only in legacy v1 graph mode. Args: loss: Loss tensor. params: List of variables. Returns: List of gradient tensors. Raises: ValueError: In case any gradient cannot be computed (e.g. if gradient function not implemented). """ params = tf.nest.flatten(params) with backend.get_graph().as_default(), backend.name_scope(self._name + "/gradients"): grads = tf.compat.v1.gradients(loss, params) for grad, param in zip(grads, params): if grad is None: raise ValueError("Variable {} has `None` for gradient. " "Please make sure that all of your ops have a " "gradient defined (i.e. are differentiable). " "Common ops without gradient: " "K.argmax, K.round, K.eval.".format(param)) return grads def get_updates(self, loss, params): grads = self.get_gradients(loss, params) grads_and_vars = list(zip(grads, params)) self._assert_valid_dtypes([ v for g, v in grads_and_vars if g is not None and v.dtype != tf.resource ]) return [self.apply_gradients(grads_and_vars)] def _set_hyper(self, name, value): """set hyper `name` to value. value can be callable, tensor, numeric.""" if isinstance(value, tf.__internal__.tracking.Trackable): self._track_trackable(value, name, overwrite=True) if name not in self._hyper: self._hyper[name] = value else: prev_value = self._hyper[name] if (callable(prev_value) or isinstance(prev_value, (tf.Tensor, int, float, learning_rate_schedule.LearningRateSchedule)) or isinstance(value, learning_rate_schedule.LearningRateSchedule)): self._hyper[name] = value else: backend.set_value(self._hyper[name], value) def _get_hyper(self, name, dtype=None): if not self._hypers_created: self._create_hypers() value = self._hyper[name] if isinstance(value, learning_rate_schedule.LearningRateSchedule): return value if callable(value): value = value() if dtype: return tf.cast(value, dtype) else: return value def _create_slots(self, var_list): pass def _create_all_weights(self, var_list): """Creates all weights, including iterations, hyperparameters and slot vars. This will add newly created variables to `optimizer.weights`. New variables are only created when this method is called the first time, or when called with different variables in the var_list. Args: var_list: list or tuple of `Variable` objects that will be minimized using this optimizer. """ _ = self.iterations self._create_hypers() self._create_slots(var_list) def __getattribute__(self, name): """Overridden to support hyperparameter access.""" try: return super(OptimizerV2, self).__getattribute__(name) except AttributeError as e: # Needed to avoid infinite recursion with __setattr__. if name == "_hyper": raise e # Backwards compatibility with Keras optimizers. if name == "lr": name = "learning_rate" if name in self._hyper: return self._get_hyper(name) raise e def __dir__(self): result = set(super(OptimizerV2, self).__dir__()) if "_hyper" in result: result |= self._hyper.keys() if "learning_rate" in self._hyper.keys(): result.add("lr") return list(result) def __setattr__(self, name, value): """Override setattr to support dynamic hyperparameter setting.""" # Backwards compatibility with Keras optimizers. if name == "lr": name = "learning_rate" if hasattr(self, "_hyper") and name in self._hyper: self._set_hyper(name, value) else: super(OptimizerV2, self).__setattr__(name, value) def get_slot_names(self): """A list of names for this optimizer's slots.""" return self._slot_names def add_slot(self, var, slot_name, initializer="zeros", shape=None): """Add a new slot variable for `var`. A slot variable is an additional variable associated with `var` to train. It is allocated and managed by optimizers, e.g. `Adam`. Args: var: a `Variable` object. slot_name: name of the slot variable. initializer: initializer of the slot variable shape: (Optional) shape of the slot variable. If not set, it will default to the shape of `var`. Returns: A slot variable. """ if slot_name not in self._slot_names: self._slot_names.append(slot_name) var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) weight = slot_dict.get(slot_name, None) if weight is None: if isinstance(initializer, str) or callable(initializer): initializer = initializers.get(initializer) if isinstance( initializer, tf.__internal__.tracking.CheckpointInitialValueCallable) or (shape is not None): slot_shape = shape else: slot_shape = var.shape initial_value = functools.partial( initializer, shape=slot_shape, dtype=var.dtype) else: initial_value = initializer with self._distribution_strategy_scope(): strategy = tf.distribute.get_strategy() if not strategy.extended.variable_created_in_scope(var): raise ValueError( "Trying to create optimizer slot variable under the scope for " "tf.distribute.Strategy ({}), which is different from the scope " "used for the original variable ({}). Make sure the slot " "variables are created under the same strategy scope. This may " "happen if you're restoring from a checkpoint outside the scope" .format(strategy, var)) with strategy.extended.colocate_vars_with(var): weight = tf.Variable( name="%s/%s" % (var._shared_name, slot_name), # pylint: disable=protected-access dtype=var.dtype, trainable=False, initial_value=initial_value) backend.track_variable(weight) slot_dict[slot_name] = weight self._restore_slot_variable( slot_name=slot_name, variable=var, slot_variable=weight) self._weights.append(weight) return weight def get_slot(self, var, slot_name): var_key = _var_key(var) slot_dict = self._slots[var_key] return slot_dict[slot_name] def _prepare(self, var_list): keys = set() for var in var_list: if isinstance(var, tf.distribute.DistributedValues): var_devices = var._devices # pylint: disable=protected-access else: var_devices = [var.device] var_dtype = var.dtype.base_dtype for var_device in var_devices: keys.add((var_device, var_dtype)) apply_state = {} for var_device, var_dtype in keys: apply_state[(var_device, var_dtype)] = {} with tf.device(var_device): self._prepare_local(var_device, var_dtype, apply_state) return apply_state def _prepare_local(self, var_device, var_dtype, apply_state): if "learning_rate" in self._hyper: lr_t = tf.identity(self._decayed_lr(var_dtype)) apply_state[(var_device, var_dtype)]["lr_t"] = lr_t def _fallback_apply_state(self, var_device, var_dtype): """Compatibility for subclasses that don't pass apply_state through.""" apply_state = {(var_device, var_dtype): {}} self._prepare_local(var_device, var_dtype, apply_state) return apply_state[(var_device, var_dtype)] def _create_hypers(self): if self._hypers_created: return with self._distribution_strategy_scope(): # Iterate hyper values deterministically. for name, value in sorted(self._hyper.items()): if isinstance(value, (tf.Tensor, tf.Variable)) or callable(value): # The check for `callable` covers the usage when `value` is a # `LearningRateSchedule`, in which case it does not need to create a # variable. continue else: self._hyper[name] = self.add_weight( name, shape=[], trainable=False, initializer=value, aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) self._hypers_created = True @property def iterations(self): """Variable. The number of training steps this Optimizer has run.""" if self._iterations is None: with self._distribution_strategy_scope(): self._iterations = self.add_weight( "iter", shape=[], dtype=tf.int64, trainable=False, aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) self._weights.append(self._iterations) return self._iterations @iterations.setter def iterations(self, variable): if self._iterations is not None: raise RuntimeError("Cannot set `iterations` to a new Variable after " "the Optimizer weights have been created") self._iterations = variable self._weights.append(self._iterations) def _decayed_lr(self, var_dtype): """Get decayed learning rate as a Tensor with dtype=var_dtype.""" lr_t = self._get_hyper("learning_rate", var_dtype) if isinstance(lr_t, learning_rate_schedule.LearningRateSchedule): local_step = tf.cast(self.iterations, var_dtype) lr_t = tf.cast(lr_t(local_step), var_dtype) if self._initial_decay > 0.: local_step = tf.cast(self.iterations, var_dtype) decay_t = tf.cast(self._initial_decay, var_dtype) lr_t = lr_t / (1. + decay_t * local_step) return lr_t @abc.abstractmethod def get_config(self): """Returns the config of the optimizer. An optimizer config is a Python dictionary (serializable) containing the configuration of an optimizer. The same optimizer can be reinstantiated later (without any saved state) from this configuration. Returns: Python dictionary. """ config = {"name": self._name} if self.clipnorm is not None: config["clipnorm"] = self.clipnorm if self.clipvalue is not None: config["clipvalue"] = self.clipvalue if self.global_clipnorm is not None: config["global_clipnorm"] = self.global_clipnorm return config @classmethod def from_config(cls, config, custom_objects=None): """Creates an optimizer from its config. This method is the reverse of `get_config`, capable of instantiating the same optimizer from the config dictionary. Args: config: A Python dictionary, typically the output of get_config. custom_objects: A Python dictionary mapping names to additional Python objects used to create this optimizer, such as a function used for a hyperparameter. Returns: An optimizer instance. """ if "lr" in config: config["learning_rate"] = config.pop("lr") if "learning_rate" in config: if isinstance(config["learning_rate"], dict): config["learning_rate"] = learning_rate_schedule.deserialize( config["learning_rate"], custom_objects=custom_objects) return cls(**config) def _serialize_hyperparameter(self, hyperparameter_name): """Serialize a hyperparameter that can be a float, callable, or Tensor.""" value = self._hyper[hyperparameter_name] if isinstance(value, learning_rate_schedule.LearningRateSchedule): return learning_rate_schedule.serialize(value) if callable(value): return value() if tf.is_tensor(value): return backend.get_value(value) return value def variables(self): """Returns variables of this Optimizer based on the order created.""" return self._weights @property def weights(self): """Returns variables of this Optimizer based on the order created.""" return self._weights def get_weights(self): """Returns the current weights of the optimizer. The weights of an optimizer are its state (ie, variables). This function returns the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they were created. The returned list can in turn be used to load state into similarly parameterized optimizers. For example, the RMSprop optimizer for this simple model returns a list of three values-- the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer: >>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> len(opt.get_weights()) 3 Returns: Weights values as a list of numpy arrays. """ params = self.weights return backend.batch_get_value(params) # TODO(tanzheny): Maybe share this logic with base_layer. def set_weights(self, weights): """Set the weights of the optimizer. The weights of an optimizer are its state (ie, variables). This function takes the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they are created. The passed values are used to set the new state of the optimizer. For example, the RMSprop optimizer for this simple model takes a list of three values-- the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer: >>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> new_weights = [np.array(10), np.ones([20, 10]), np.zeros([10])] >>> opt.set_weights(new_weights) >>> opt.iterations <tf.Variable 'RMSprop/iter:0' shape=() dtype=int64, numpy=10> Args: weights: weight values as a list of numpy arrays. """ params = self.weights if len(params) != len(weights): raise ValueError( "You called `set_weights(weights)` on optimizer " + self._name + " with a weight list of length " + str(len(weights)) + ", but the optimizer was expecting " + str(len(params)) + " weights. Provided weights: " + str(weights)[:50] + "...") if not params: return weight_value_tuples = [] param_values = backend.batch_get_value(params) for pv, p, w in zip(param_values, params, weights): if pv.shape != w.shape: raise ValueError("Optimizer weight shape " + str(pv.shape) + " not compatible with " "provided weight shape " + str(w.shape)) weight_value_tuples.append((p, w)) backend.batch_set_value(weight_value_tuples) def add_weight(self, name, shape, dtype=None, initializer="zeros", trainable=None, synchronization=tf.VariableSynchronization.AUTO, aggregation=tf.VariableAggregation.NONE): if dtype is None: dtype = tf.float32 if isinstance(initializer, str) or callable(initializer): initializer = initializers.get(initializer) if synchronization == tf.VariableSynchronization.ON_READ: if trainable: raise ValueError( "Synchronization value can be set to " "VariableSynchronization.ON_READ only for non-trainable variables. " "You have specified trainable=True and " "synchronization=VariableSynchronization.ON_READ.") else: # Set trainable to be false when variable is to be synced on read. trainable = False elif trainable is None: trainable = True variable = self._add_variable_with_custom_getter( name=name, shape=shape, getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, trainable=trainable, use_resource=True, synchronization=synchronization, aggregation=aggregation) backend.track_variable(variable) return variable def _init_set_name(self, name, zero_based=True): if not name: self._name = backend.unique_object_name( generic_utils.to_snake_case(self.__class__.__name__), zero_based=zero_based) else: self._name = name def _assert_valid_dtypes(self, tensors): """Asserts tensors are all valid types (see `_valid_dtypes`). Args: tensors: Tensors to check. Raises: ValueError: If any tensor is not a valid type. """ valid_dtypes = self._valid_dtypes() for t in tensors: dtype = t.dtype.base_dtype if dtype not in valid_dtypes: raise ValueError("Invalid type %r for %s, expected: %s." % (dtype, t.name, [v for v in valid_dtypes])) def _valid_dtypes(self): """Valid types for loss, variables and gradients. Subclasses should override to allow other float types. Returns: Valid types for loss, variables and gradients. """ return _DEFAULT_VALID_DTYPES def _call_if_callable(self, param): """Call the function if param is callable.""" return param() if callable(param) else param def _resource_apply_dense(self, grad, handle, apply_state): """Add ops to apply dense gradients to the variable `handle`. Args: grad: a `Tensor` representing the gradient. handle: a `Tensor` of dtype `resource` which points to the variable to be updated. apply_state: A dict which is used across multiple apply calls. Returns: An `Operation` which updates the value of the variable. """ raise NotImplementedError("Must be implemented in subclasses.") def _resource_apply_sparse_duplicate_indices(self, grad, handle, indices, **kwargs): """Add ops to apply sparse gradients to `handle`, with repeated indices. Optimizers which override this method must deal with repeated indices. See the docstring of `_apply_sparse_duplicate_indices` for details. By default the correct behavior, to sum non-unique indices and their associated gradients, is enforced by first pre-processing `grad` and `indices` and passing them on to `_resource_apply_sparse`. Optimizers which deal correctly with duplicate indices may instead override this method to avoid the overhead of summing. Args: grad: a `Tensor` representing the gradient for the affected indices. handle: a `Tensor` of dtype `resource` which points to the variable to be updated. indices: a `Tensor` of integral type representing the indices for which the gradient is nonzero. Indices may be repeated. **kwargs: May optionally contain `apply_state` Returns: An `Operation` which updates the value of the variable. """ summed_grad, unique_indices = _deduplicate_indexed_slices( values=grad, indices=indices) return self._resource_apply_sparse(summed_grad, handle, unique_indices, **kwargs) def _resource_apply_sparse(self, grad, handle, indices, apply_state): """Add ops to apply sparse gradients to the variable `handle`. Similar to `_apply_sparse`, the `indices` argument to this method has been de-duplicated. Optimizers which deal correctly with non-unique indices may instead override `_resource_apply_sparse_duplicate_indices` to avoid this overhead. Args: grad: a `Tensor` representing the gradient for the affected indices. handle: a `Tensor` of dtype `resource` which points to the variable to be updated. indices: a `Tensor` of integral type representing the indices for which the gradient is nonzero. Indices are unique. apply_state: A dict which is used across multiple apply calls. Returns: An `Operation` which updates the value of the variable. """ raise NotImplementedError("Must be implemented in subclasses.") def _resource_scatter_add(self, x, i, v): with tf.control_dependencies([ tf.raw_ops.ResourceScatterAdd( resource=x.handle, indices=i, updates=v) ]): return x.value() def _resource_scatter_update(self, x, i, v): with tf.control_dependencies( [tf.raw_ops.ResourceScatterUpdate( resource=x.handle, indices=i, updates=v)]): return x.value() @property @layer_utils.cached_per_instance def _dense_apply_args(self): return tf_inspect.getfullargspec(self._resource_apply_dense).args @property @layer_utils.cached_per_instance def _sparse_apply_args(self): return tf_inspect.getfullargspec(self._resource_apply_sparse).args # --------------- # For implementing the trackable interface # --------------- def _restore_slot_variable(self, slot_name, variable, slot_variable): """Restore a newly created slot variable's value.""" variable_key = _var_key(variable) deferred_restorations = self._deferred_slot_restorations.get( slot_name, {}).pop(variable_key, []) # Iterate over restores, highest restore UID first to minimize the number # of assignments. deferred_restorations.sort(key=lambda position: position.restore_uid, reverse=True) for checkpoint_position in deferred_restorations: checkpoint_position.restore(slot_variable) def _create_or_restore_slot_variable( self, slot_variable_position, slot_name, variable): """Restore a slot variable's value, possibly creating it. Called when a variable which has an associated slot variable is created or restored. When executing eagerly, we create the slot variable with a restoring initializer. No new variables are created when graph building. Instead, _restore_slot_variable catches these after normal creation and adds restore ops to the graph. This method is nonetheless important when graph building for the case when a slot variable has already been created but `variable` has just been added to a dependency graph (causing us to realize that the slot variable needs to be restored). Args: slot_variable_position: A `trackable._CheckpointPosition` object indicating the slot variable `Trackable` object to be restored. slot_name: The name of this `Optimizer`'s slot to restore into. variable: The variable object this slot is being created for. """ variable_key = _var_key(variable) slot_dict = self._slots.get(variable_key, {}) slot_variable = slot_dict.get(slot_name, None) if (slot_variable is None and tf.executing_eagerly() and slot_variable_position.is_simple_variable() # Defer slot variable creation if there is an active variable creator # scope. Generally we'd like to eagerly create/restore slot variables # when possible, but this may mean that scopes intended to catch # `variable` also catch its eagerly created slot variable # unintentionally (specifically make_template would add a dependency on # a slot variable if not for this case). Deferring is mostly harmless # (aside from double initialization), and makes variable creator scopes # behave the same way they do when graph building. # # One notable case is with distribution strategy, which uses variable # creator scope but always desires the `variable` and the slot to use # the same scope, thus we can safely eagerly create/restore slot # variables. and (not tf.compat.v1.get_default_graph()._variable_creator_stack or # pylint: disable=protected-access self._distribution_strategy)): initializer = tf.__internal__.tracking.CheckpointInitialValueCallable( checkpoint_position=slot_variable_position) slot_variable = self.add_slot( var=variable, initializer=initializer, slot_name=slot_name, shape=slot_variable_position.value_shape()) # Slot variables are not owned by any one object (because we don't want to # save the slot variable if the optimizer is saved without the non-slot # variable, or if the non-slot variable is saved without the optimizer; # it's a dependency hypergraph with edges of the form (optimizer, non-slot # variable, variable)). So we don't _track_ slot variables anywhere, and # instead special-case this dependency and otherwise pretend it's a normal # graph. if slot_variable is not None: # If we've either made this slot variable, or if we've pulled out an # existing slot variable, we should restore it. slot_variable_position.restore(slot_variable) else: # We didn't make the slot variable. Defer restoring until it gets created # normally. We keep a list rather than the one with the highest restore # UID in case slot variables have their own dependencies, in which case # those could differ between restores. self._deferred_slot_restorations.setdefault( slot_name, {}).setdefault(variable_key, []).append( slot_variable_position) @contextlib.contextmanager def _distribution_strategy_scope(self): """Returns the `tf.distribute.Strategy` this optimizer was created under.""" if self._distribution_strategy and not tf.distribute.has_strategy(): with self._distribution_strategy.scope(): yield self._distribution_strategy.scope() else: yield
Ancestors
- tensorflow.python.training.tracking.base.Trackable
Subclasses
- LossScaleOptimizer
- Adadelta
- Adagrad
- Adam
- NonFusedAdam
- Adamax
- Ftrl
- SGD
- Nadam
- RestoredOptimizer
- RMSprop
Static methods
def from_config(config, custom_objects=None)
-
Creates an optimizer from its config.
This method is the reverse of
get_config
, capable of instantiating the same optimizer from the config dictionary.Args
config
- A Python dictionary, typically the output of get_config.
custom_objects
- A Python dictionary mapping names to additional Python objects used to create this optimizer, such as a function used for a hyperparameter.
Returns
An optimizer instance.
Expand source code
@classmethod def from_config(cls, config, custom_objects=None): """Creates an optimizer from its config. This method is the reverse of `get_config`, capable of instantiating the same optimizer from the config dictionary. Args: config: A Python dictionary, typically the output of get_config. custom_objects: A Python dictionary mapping names to additional Python objects used to create this optimizer, such as a function used for a hyperparameter. Returns: An optimizer instance. """ if "lr" in config: config["learning_rate"] = config.pop("lr") if "learning_rate" in config: if isinstance(config["learning_rate"], dict): config["learning_rate"] = learning_rate_schedule.deserialize( config["learning_rate"], custom_objects=custom_objects) return cls(**config)
Instance variables
var clipnorm
-
float
orNone
. If set, clips gradients to a maximum norm.Expand source code
@property def clipnorm(self): """`float` or `None`. If set, clips gradients to a maximum norm.""" return self._clipnorm
var clipvalue
-
float
orNone
. If set, clips gradients to a maximum value.Expand source code
@property def clipvalue(self): """`float` or `None`. If set, clips gradients to a maximum value.""" return self._clipvalue
var global_clipnorm
-
float
orNone
. If set, clips gradients to a maximum norm.Expand source code
@property def global_clipnorm(self): """`float` or `None`. If set, clips gradients to a maximum norm.""" return self._global_clipnorm
var iterations
-
Variable. The number of training steps this Optimizer has run.
Expand source code
@property def iterations(self): """Variable. The number of training steps this Optimizer has run.""" if self._iterations is None: with self._distribution_strategy_scope(): self._iterations = self.add_weight( "iter", shape=[], dtype=tf.int64, trainable=False, aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) self._weights.append(self._iterations) return self._iterations
var weights
-
Returns variables of this Optimizer based on the order created.
Expand source code
@property def weights(self): """Returns variables of this Optimizer based on the order created.""" return self._weights
Methods
def add_slot(self, var, slot_name, initializer='zeros', shape=None)
-
Add a new slot variable for
var
.A slot variable is an additional variable associated with
var
to train. It is allocated and managed by optimizers, e.g.Adam
.Args
var
- a
Variable
object. slot_name
- name of the slot variable.
initializer
- initializer of the slot variable
shape
- (Optional) shape of the slot variable. If not set, it will default
to the shape of
var
.Returns
A slot variable.
Expand source code
def add_slot(self, var, slot_name, initializer="zeros", shape=None): """Add a new slot variable for `var`. A slot variable is an additional variable associated with `var` to train. It is allocated and managed by optimizers, e.g. `Adam`. Args: var: a `Variable` object. slot_name: name of the slot variable. initializer: initializer of the slot variable shape: (Optional) shape of the slot variable. If not set, it will default to the shape of `var`. Returns: A slot variable. """ if slot_name not in self._slot_names: self._slot_names.append(slot_name) var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) weight = slot_dict.get(slot_name, None) if weight is None: if isinstance(initializer, str) or callable(initializer): initializer = initializers.get(initializer) if isinstance( initializer, tf.__internal__.tracking.CheckpointInitialValueCallable) or (shape is not None): slot_shape = shape else: slot_shape = var.shape initial_value = functools.partial( initializer, shape=slot_shape, dtype=var.dtype) else: initial_value = initializer with self._distribution_strategy_scope(): strategy = tf.distribute.get_strategy() if not strategy.extended.variable_created_in_scope(var): raise ValueError( "Trying to create optimizer slot variable under the scope for " "tf.distribute.Strategy ({}), which is different from the scope " "used for the original variable ({}). Make sure the slot " "variables are created under the same strategy scope. This may " "happen if you're restoring from a checkpoint outside the scope" .format(strategy, var)) with strategy.extended.colocate_vars_with(var): weight = tf.Variable( name="%s/%s" % (var._shared_name, slot_name), # pylint: disable=protected-access dtype=var.dtype, trainable=False, initial_value=initial_value) backend.track_variable(weight) slot_dict[slot_name] = weight self._restore_slot_variable( slot_name=slot_name, variable=var, slot_variable=weight) self._weights.append(weight) return weight
def add_weight(self, name, shape, dtype=None, initializer='zeros', trainable=None, synchronization=VariableSynchronization.AUTO, aggregation=VariableAggregationV2.NONE)
-
Expand source code
def add_weight(self, name, shape, dtype=None, initializer="zeros", trainable=None, synchronization=tf.VariableSynchronization.AUTO, aggregation=tf.VariableAggregation.NONE): if dtype is None: dtype = tf.float32 if isinstance(initializer, str) or callable(initializer): initializer = initializers.get(initializer) if synchronization == tf.VariableSynchronization.ON_READ: if trainable: raise ValueError( "Synchronization value can be set to " "VariableSynchronization.ON_READ only for non-trainable variables. " "You have specified trainable=True and " "synchronization=VariableSynchronization.ON_READ.") else: # Set trainable to be false when variable is to be synced on read. trainable = False elif trainable is None: trainable = True variable = self._add_variable_with_custom_getter( name=name, shape=shape, getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, trainable=trainable, use_resource=True, synchronization=synchronization, aggregation=aggregation) backend.track_variable(variable) return variable
def apply_gradients(self, grads_and_vars, name=None, experimental_aggregate_gradients=True)
-
Apply gradients to variables.
This is the second part of
minimize()
. It returns anOperation
that applies gradients.The method sums gradients from all replicas in the presence of
tf.distribute.Strategy
by default. You can aggregate gradients yourself by passingexperimental_aggregate_gradients=False
.Example:
grads = tape.gradient(loss, vars) grads = tf.distribute.get_replica_context().all_reduce('sum', grads) # Processing aggregated gradients. optimizer.apply_gradients(zip(grads, vars), experimental_aggregate_gradients=False)
Args
grads_and_vars
- List of (gradient, variable) pairs.
name
- Optional name for the returned operation. Default to the name passed
to the
OptimizerV2
constructor. experimental_aggregate_gradients
- Whether to sum gradients from different
replicas in the presense of
tf.distribute.Strategy
. If False, it's user responsibility to aggregate the gradients. Default to True.
Returns
An
Operation
that applies the specified gradients. Theiterations
will be automatically increased by 1.Raises
TypeError
- If
grads_and_vars
is malformed. ValueError
- If none of the variables have gradients.
RuntimeError
- If called in a cross-replica context.
Expand source code
def apply_gradients(self, grads_and_vars, name=None, experimental_aggregate_gradients=True): """Apply gradients to variables. This is the second part of `minimize()`. It returns an `Operation` that applies gradients. The method sums gradients from all replicas in the presence of `tf.distribute.Strategy` by default. You can aggregate gradients yourself by passing `experimental_aggregate_gradients=False`. Example: ```python grads = tape.gradient(loss, vars) grads = tf.distribute.get_replica_context().all_reduce('sum', grads) # Processing aggregated gradients. optimizer.apply_gradients(zip(grads, vars), experimental_aggregate_gradients=False) ``` Args: grads_and_vars: List of (gradient, variable) pairs. name: Optional name for the returned operation. Default to the name passed to the `Optimizer` constructor. experimental_aggregate_gradients: Whether to sum gradients from different replicas in the presense of `tf.distribute.Strategy`. If False, it's user responsibility to aggregate the gradients. Default to True. Returns: An `Operation` that applies the specified gradients. The `iterations` will be automatically increased by 1. Raises: TypeError: If `grads_and_vars` is malformed. ValueError: If none of the variables have gradients. RuntimeError: If called in a cross-replica context. """ grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) var_list = [v for (_, v) in grads_and_vars] with tf.name_scope(self._name): # Create iteration if necessary. with tf.init_scope(): self._create_all_weights(var_list) if not grads_and_vars: # Distribution strategy does not support reducing an empty list of # gradients return tf.no_op() if tf.distribute.in_cross_replica_context(): raise RuntimeError( "`apply_gradients() cannot be called in cross-replica context. " "Use `tf.distribute.Strategy.run` to enter replica " "context.") strategy = tf.distribute.get_strategy() if (not experimental_aggregate_gradients and strategy and isinstance(strategy, (tf.compat.v1.distribute.experimental.ParameterServerStrategy, tf.distribute.experimental.ParameterServerStrategy, tf.distribute.experimental.CentralStorageStrategy, tf.compat.v1.distribute.experimental.CentralStorageStrategy))): raise NotImplementedError( "`experimental_aggregate_gradients=False is not supported for " "ParameterServerStrategy and CentralStorageStrategy") apply_state = self._prepare(var_list) if experimental_aggregate_gradients: grads_and_vars = self._transform_unaggregated_gradients(grads_and_vars) grads_and_vars = self._aggregate_gradients(grads_and_vars) grads_and_vars = self._transform_gradients(grads_and_vars) if optimizer_utils.strategy_supports_no_merge_call(): return self._distributed_apply(strategy, grads_and_vars, name, apply_state) else: return tf.distribute.get_replica_context().merge_call( functools.partial(self._distributed_apply, apply_state=apply_state), args=(grads_and_vars,), kwargs={ "name": name, })
def get_config(self)
-
Returns the config of the optimizer.
An optimizer config is a Python dictionary (serializable) containing the configuration of an optimizer. The same optimizer can be reinstantiated later (without any saved state) from this configuration.
Returns
Python dictionary.
Expand source code
@abc.abstractmethod def get_config(self): """Returns the config of the optimizer. An optimizer config is a Python dictionary (serializable) containing the configuration of an optimizer. The same optimizer can be reinstantiated later (without any saved state) from this configuration. Returns: Python dictionary. """ config = {"name": self._name} if self.clipnorm is not None: config["clipnorm"] = self.clipnorm if self.clipvalue is not None: config["clipvalue"] = self.clipvalue if self.global_clipnorm is not None: config["global_clipnorm"] = self.global_clipnorm return config
def get_gradients(self, loss, params)
-
Returns gradients of
loss
with respect toparams
.Should be used only in legacy v1 graph mode.
Args
loss
- Loss tensor.
params
- List of variables.
Returns
List of gradient tensors.
Raises
ValueError
- In case any gradient cannot be computed (e.g. if gradient function not implemented).
Expand source code
def get_gradients(self, loss, params): """Returns gradients of `loss` with respect to `params`. Should be used only in legacy v1 graph mode. Args: loss: Loss tensor. params: List of variables. Returns: List of gradient tensors. Raises: ValueError: In case any gradient cannot be computed (e.g. if gradient function not implemented). """ params = tf.nest.flatten(params) with backend.get_graph().as_default(), backend.name_scope(self._name + "/gradients"): grads = tf.compat.v1.gradients(loss, params) for grad, param in zip(grads, params): if grad is None: raise ValueError("Variable {} has `None` for gradient. " "Please make sure that all of your ops have a " "gradient defined (i.e. are differentiable). " "Common ops without gradient: " "K.argmax, K.round, K.eval.".format(param)) return grads
def get_slot(self, var, slot_name)
-
Expand source code
def get_slot(self, var, slot_name): var_key = _var_key(var) slot_dict = self._slots[var_key] return slot_dict[slot_name]
def get_slot_names(self)
-
A list of names for this optimizer's slots.
Expand source code
def get_slot_names(self): """A list of names for this optimizer's slots.""" return self._slot_names
def get_updates(self, loss, params)
-
Expand source code
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) grads_and_vars = list(zip(grads, params)) self._assert_valid_dtypes([ v for g, v in grads_and_vars if g is not None and v.dtype != tf.resource ]) return [self.apply_gradients(grads_and_vars)]
def get_weights(self)
-
Returns the current weights of the optimizer.
The weights of an optimizer are its state (ie, variables). This function returns the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they were created. The returned list can in turn be used to load state into similarly parameterized optimizers.
For example, the RMSprop optimizer for this simple model returns a list of three values– the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer:
>>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> len(opt.get_weights()) 3
Returns
Weights values as a list of numpy arrays.
Expand source code
def get_weights(self): """Returns the current weights of the optimizer. The weights of an optimizer are its state (ie, variables). This function returns the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they were created. The returned list can in turn be used to load state into similarly parameterized optimizers. For example, the RMSprop optimizer for this simple model returns a list of three values-- the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer: >>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> len(opt.get_weights()) 3 Returns: Weights values as a list of numpy arrays. """ params = self.weights return backend.batch_get_value(params)
def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None)
-
Minimize
loss
by updatingvar_list
.This method simply computes gradient using
tf.GradientTape
and callsapply_gradients()
. If you want to process the gradient before applying then calltf.GradientTape
andapply_gradients()
explicitly instead of using this function.Args
loss
Tensor
or callable. If a callable,loss
should take no arguments and return the value to minimize. If aTensor
, thetape
argument must be passed.var_list
- list or tuple of
Variable
objects to update to minimizeloss
, or a callable returning the list or tuple ofVariable
objects. Use callable when the variable list would otherwise be incomplete beforeminimize
since the variables are created at the first timeloss
is called. grad_loss
- (Optional). A
Tensor
holding the gradient computed forloss
. name
- (Optional) str. Name for the returned operation.
tape
- (Optional)
tf.GradientTape
. Ifloss
is provided as aTensor
, the tape that computed theloss
must be provided.
Returns
An
Operation
that updates the variables invar_list
. Theiterations
will be automatically increased by 1.Raises
ValueError
- If some of the variables are not
Variable
objects.
Expand source code
def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None): """Minimize `loss` by updating `var_list`. This method simply computes gradient using `tf.GradientTape` and calls `apply_gradients()`. If you want to process the gradient before applying then call `tf.GradientTape` and `apply_gradients()` explicitly instead of using this function. Args: loss: `Tensor` or callable. If a callable, `loss` should take no arguments and return the value to minimize. If a `Tensor`, the `tape` argument must be passed. var_list: list or tuple of `Variable` objects to update to minimize `loss`, or a callable returning the list or tuple of `Variable` objects. Use callable when the variable list would otherwise be incomplete before `minimize` since the variables are created at the first time `loss` is called. grad_loss: (Optional). A `Tensor` holding the gradient computed for `loss`. name: (Optional) str. Name for the returned operation. tape: (Optional) `tf.GradientTape`. If `loss` is provided as a `Tensor`, the tape that computed the `loss` must be provided. Returns: An `Operation` that updates the variables in `var_list`. The `iterations` will be automatically increased by 1. Raises: ValueError: If some of the variables are not `Variable` objects. """ grads_and_vars = self._compute_gradients( loss, var_list=var_list, grad_loss=grad_loss, tape=tape) return self.apply_gradients(grads_and_vars, name=name)
def set_weights(self, weights)
-
Set the weights of the optimizer.
The weights of an optimizer are its state (ie, variables). This function takes the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they are created. The passed values are used to set the new state of the optimizer.
For example, the RMSprop optimizer for this simple model takes a list of three values– the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer:
>>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> new_weights = [np.array(10), np.ones([20, 10]), np.zeros([10])] >>> opt.set_weights(new_weights) >>> opt.iterations <tf.Variable 'RMSprop/iter:0' shape=() dtype=int64, numpy=10>
Args
weights
- weight values as a list of numpy arrays.
Expand source code
def set_weights(self, weights): """Set the weights of the optimizer. The weights of an optimizer are its state (ie, variables). This function takes the weight values associated with this optimizer as a list of Numpy arrays. The first value is always the iterations count of the optimizer, followed by the optimizer's state variables in the order they are created. The passed values are used to set the new state of the optimizer. For example, the RMSprop optimizer for this simple model takes a list of three values-- the iteration count, followed by the root-mean-square value of the kernel and bias of the single Dense layer: >>> opt = tf.keras.optimizers.RMSprop() >>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)]) >>> m.compile(opt, loss='mse') >>> data = np.arange(100).reshape(5, 20) >>> labels = np.zeros(5) >>> print('Training'); results = m.fit(data, labels) Training ... >>> new_weights = [np.array(10), np.ones([20, 10]), np.zeros([10])] >>> opt.set_weights(new_weights) >>> opt.iterations <tf.Variable 'RMSprop/iter:0' shape=() dtype=int64, numpy=10> Args: weights: weight values as a list of numpy arrays. """ params = self.weights if len(params) != len(weights): raise ValueError( "You called `set_weights(weights)` on optimizer " + self._name + " with a weight list of length " + str(len(weights)) + ", but the optimizer was expecting " + str(len(params)) + " weights. Provided weights: " + str(weights)[:50] + "...") if not params: return weight_value_tuples = [] param_values = backend.batch_get_value(params) for pv, p, w in zip(param_values, params, weights): if pv.shape != w.shape: raise ValueError("Optimizer weight shape " + str(pv.shape) + " not compatible with " "provided weight shape " + str(w.shape)) weight_value_tuples.append((p, w)) backend.batch_set_value(weight_value_tuples)
def variables(self)
-
Returns variables of this Optimizer based on the order created.
Expand source code
def variables(self): """Returns variables of this Optimizer based on the order created.""" return self._weights
- Compute the gradients with
class RMSprop (learning_rate=0.001, rho=0.9, momentum=0.0, epsilon=1e-07, centered=False, name='RMSprop', **kwargs)
-
Optimizer that implements the RMSprop algorithm.
The gist of RMSprop is to:
- Maintain a moving (discounted) average of the square of gradients
- Divide the gradient by the root of this average
This implementation of RMSprop uses plain momentum, not Nesterov momentum.
The centered version additionally maintains a moving average of the gradients, and uses that average to estimate the variance.
Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.001. rho
- Discounting factor for the history/coming gradient. Defaults to 0.9.
momentum
- A scalar or a scalar
Tensor
. Defaults to 0.0. epsilon
- A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7.
centered
- Boolean. If
True
, gradients are normalized by the estimated variance of the gradient; if False, by the uncentered second moment. Setting this toTrue
may help with training, but is slightly more expensive in terms of computation and memory. Defaults toFalse
. name
- Optional name prefix for the operations created when applying
gradients. Defaults to
"RMSprop"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Note that in the dense implementation of this algorithm, variables and their corresponding accumulators (momentum, gradient moving average, square gradient moving average) will be updated even if the gradient is zero (i.e. accumulators will decay, momentum will be applied). The sparse implementation (used when the gradient is an
IndexedSlices
object, typically because oftf.gather
or an embedding lookup in the forward pass) will not update variable slices or their accumulators unless those slices were used in the forward pass (nor is there an "eventual" correction to account for these omitted updates). This leads to more efficient updates for large embedding lookup tables (where most of the slices are not accessed in a particular graph execution), but differs from the published algorithm.Usage:
>>> opt = tf.keras.optimizers.RMSprop(learning_rate=0.1) >>> var1 = tf.Variable(10.0) >>> loss = lambda: (var1 ** 2) / 2.0 # d(loss) / d(var1) = var1 >>> step_count = opt.minimize(loss, [var1]).numpy() >>> var1.numpy() 9.683772
Reference
Construct a new RMSprop optimizer.
Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.001. rho
- Discounting factor for the history/coming gradient. Defaults to 0.9.
momentum
- A scalar or a scalar
Tensor
. Defaults to 0.0. epsilon
- A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7.
centered
- Boolean. If
True
, gradients are normalized by the estimated variance of the gradient; if False, by the uncentered second moment. Setting this toTrue
may help with training, but is slightly more expensive in terms of computation and memory. Defaults toFalse
. name
- Optional name prefix for the operations created when applying gradients. Defaults to "RMSprop".
**kwargs
- keyword arguments. Allowed to be {
clipnorm
,clipvalue
,lr
,decay
}.clipnorm
is clip gradients by norm;clipvalue
is clip gradients by value,decay
is included for backward compatibility to allow time inverse decay of learning rate.lr
is included for backward compatibility, recommended to uselearning_rate
instead.
@compatibility(eager) When eager execution is enabled,
learning_rate
,decay
,momentum
, andepsilon
can each be a callable that takes no arguments and returns the actual value to use. This can be useful for changing these values across different invocations of optimizer functions. @end_compatibilityExpand source code
class RMSprop(optimizer_v2.OptimizerV2): r"""Optimizer that implements the RMSprop algorithm. The gist of RMSprop is to: - Maintain a moving (discounted) average of the square of gradients - Divide the gradient by the root of this average This implementation of RMSprop uses plain momentum, not Nesterov momentum. The centered version additionally maintains a moving average of the gradients, and uses that average to estimate the variance. Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.001. rho: Discounting factor for the history/coming gradient. Defaults to 0.9. momentum: A scalar or a scalar `Tensor`. Defaults to 0.0. epsilon: A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7. centered: Boolean. If `True`, gradients are normalized by the estimated variance of the gradient; if False, by the uncentered second moment. Setting this to `True` may help with training, but is slightly more expensive in terms of computation and memory. Defaults to `False`. name: Optional name prefix for the operations created when applying gradients. Defaults to `"RMSprop"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Note that in the dense implementation of this algorithm, variables and their corresponding accumulators (momentum, gradient moving average, square gradient moving average) will be updated even if the gradient is zero (i.e. accumulators will decay, momentum will be applied). The sparse implementation (used when the gradient is an `IndexedSlices` object, typically because of `tf.gather` or an embedding lookup in the forward pass) will not update variable slices or their accumulators unless those slices were used in the forward pass (nor is there an "eventual" correction to account for these omitted updates). This leads to more efficient updates for large embedding lookup tables (where most of the slices are not accessed in a particular graph execution), but differs from the published algorithm. Usage: >>> opt = tf.keras.optimizers.RMSprop(learning_rate=0.1) >>> var1 = tf.Variable(10.0) >>> loss = lambda: (var1 ** 2) / 2.0 # d(loss) / d(var1) = var1 >>> step_count = opt.minimize(loss, [var1]).numpy() >>> var1.numpy() 9.683772 Reference: - [Hinton, 2012]( http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.001, rho=0.9, momentum=0.0, epsilon=1e-7, centered=False, name="RMSprop", **kwargs): """Construct a new RMSprop optimizer. Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.001. rho: Discounting factor for the history/coming gradient. Defaults to 0.9. momentum: A scalar or a scalar `Tensor`. Defaults to 0.0. epsilon: A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. Defaults to 1e-7. centered: Boolean. If `True`, gradients are normalized by the estimated variance of the gradient; if False, by the uncentered second moment. Setting this to `True` may help with training, but is slightly more expensive in terms of computation and memory. Defaults to `False`. name: Optional name prefix for the operations created when applying gradients. Defaults to "RMSprop". **kwargs: keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`, `decay`}. `clipnorm` is clip gradients by norm; `clipvalue` is clip gradients by value, `decay` is included for backward compatibility to allow time inverse decay of learning rate. `lr` is included for backward compatibility, recommended to use `learning_rate` instead. @compatibility(eager) When eager execution is enabled, `learning_rate`, `decay`, `momentum`, and `epsilon` can each be a callable that takes no arguments and returns the actual value to use. This can be useful for changing these values across different invocations of optimizer functions. @end_compatibility """ super(RMSprop, self).__init__(name, **kwargs) self._set_hyper("learning_rate", kwargs.get("lr", learning_rate)) self._set_hyper("decay", self._initial_decay) self._set_hyper("rho", rho) self._momentum = False if isinstance(momentum, tf.Tensor) or callable(momentum) or momentum > 0: self._momentum = True if isinstance(momentum, (int, float)) and (momentum < 0 or momentum > 1): raise ValueError("`momentum` must be between [0, 1].") self._set_hyper("momentum", momentum) self.epsilon = epsilon or backend_config.epsilon() self.centered = centered def _create_slots(self, var_list): for var in var_list: self.add_slot(var, "rms") if self._momentum: for var in var_list: self.add_slot(var, "momentum") if self.centered: for var in var_list: self.add_slot(var, "mg") def _prepare_local(self, var_device, var_dtype, apply_state): super(RMSprop, self)._prepare_local(var_device, var_dtype, apply_state) rho = tf.identity(self._get_hyper("rho", var_dtype)) apply_state[(var_device, var_dtype)].update( dict( neg_lr_t=-apply_state[(var_device, var_dtype)]["lr_t"], epsilon=tf.convert_to_tensor( self.epsilon, var_dtype), rho=rho, momentum=tf.identity(self._get_hyper("momentum", var_dtype)), one_minus_rho=1. - rho)) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) rms = self.get_slot(var, "rms") if self._momentum: mom = self.get_slot(var, "momentum") if self.centered: mg = self.get_slot(var, "mg") return tf.raw_ops.ResourceApplyCenteredRMSProp( var=var.handle, mg=mg.handle, ms=rms.handle, mom=mom.handle, lr=coefficients["lr_t"], rho=coefficients["rho"], momentum=coefficients["momentum"], epsilon=coefficients["epsilon"], grad=grad, use_locking=self._use_locking) else: return tf.raw_ops.ResourceApplyRMSProp( var=var.handle, ms=rms.handle, mom=mom.handle, lr=coefficients["lr_t"], rho=coefficients["rho"], momentum=coefficients["momentum"], epsilon=coefficients["epsilon"], grad=grad, use_locking=self._use_locking) else: rms_t = (coefficients["rho"] * rms + coefficients["one_minus_rho"] * tf.square(grad)) rms_t = tf.compat.v1.assign(rms, rms_t, use_locking=self._use_locking) denom_t = rms_t if self.centered: mg = self.get_slot(var, "mg") mg_t = coefficients["rho"] * mg + coefficients["one_minus_rho"] * grad mg_t = tf.compat.v1.assign(mg, mg_t, use_locking=self._use_locking) denom_t = rms_t - tf.square(mg_t) var_t = var - coefficients["lr_t"] * grad / ( tf.sqrt(denom_t) + coefficients["epsilon"]) return tf.compat.v1.assign(var, var_t, use_locking=self._use_locking).op def _resource_apply_sparse(self, grad, var, indices, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) rms = self.get_slot(var, "rms") if self._momentum: mom = self.get_slot(var, "momentum") if self.centered: mg = self.get_slot(var, "mg") return tf.raw_ops.ResourceSparseApplyCenteredRMSProp( var=var.handle, mg=mg.handle, ms=rms.handle, mom=mom.handle, lr=coefficients["lr_t"], rho=coefficients["rho"], momentum=coefficients["momentum"], epsilon=coefficients["epsilon"], grad=grad, indices=indices, use_locking=self._use_locking) else: return tf.raw_ops.ResourceSparseApplyRMSProp( var=var.handle, ms=rms.handle, mom=mom.handle, lr=coefficients["lr_t"], rho=coefficients["rho"], momentum=coefficients["momentum"], epsilon=coefficients["epsilon"], grad=grad, indices=indices, use_locking=self._use_locking) else: rms_scaled_g_values = (grad * grad) * coefficients["one_minus_rho"] rms_t = tf.compat.v1.assign(rms, rms * coefficients["rho"], use_locking=self._use_locking) with tf.control_dependencies([rms_t]): rms_t = self._resource_scatter_add(rms, indices, rms_scaled_g_values) rms_slice = tf.gather(rms_t, indices) denom_slice = rms_slice if self.centered: mg = self.get_slot(var, "mg") mg_scaled_g_values = grad * coefficients["one_minus_rho"] mg_t = tf.compat.v1.assign(mg, mg * coefficients["rho"], use_locking=self._use_locking) with tf.control_dependencies([mg_t]): mg_t = self._resource_scatter_add(mg, indices, mg_scaled_g_values) mg_slice = tf.gather(mg_t, indices) denom_slice = rms_slice - tf.square(mg_slice) var_update = self._resource_scatter_add( var, indices, coefficients["neg_lr_t"] * grad / ( tf.sqrt(denom_slice) + coefficients["epsilon"])) if self.centered: return tf.group(*[var_update, rms_t, mg_t]) return tf.group(*[var_update, rms_t]) def set_weights(self, weights): params = self.weights # Override set_weights for backward compatibility of Keras V1 optimizer # since it does not include iteration at head of the weight list. Set # iteration to 0. if len(params) == len(weights) + 1: weights = [np.array(0)] + weights super(RMSprop, self).set_weights(weights) def get_config(self): config = super(RMSprop, self).get_config() config.update({ "learning_rate": self._serialize_hyperparameter("learning_rate"), "decay": self._initial_decay, "rho": self._serialize_hyperparameter("rho"), "momentum": self._serialize_hyperparameter("momentum"), "epsilon": self.epsilon, "centered": self.centered, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members
class SGD (learning_rate=0.01, momentum=0.0, nesterov=False, name='SGD', **kwargs)
-
Gradient descent (with momentum) optimizer.
Update rule for parameter
w
with gradientg
whenmomentum
is 0:w = w - learning_rate * g
Update rule when
momentum
is larger than 0:velocity = momentum * velocity - learning_rate * g w = w + velocity
When
nesterov=True
, this rule becomes:velocity = momentum * velocity - learning_rate * g w = w + momentum * velocity - learning_rate * g
Args
learning_rate
- A
Tensor
, floating point value, or a schedule that is atf.keras.optimizers.schedules.LearningRateSchedule
, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.01. momentum
- float hyperparameter >= 0 that accelerates gradient descent in the relevant direction and dampens oscillations. Defaults to 0, i.e., vanilla gradient descent.
nesterov
- boolean. Whether to apply Nesterov momentum.
Defaults to
False
. name
- Optional name prefix for the operations created when applying
gradients.
Defaults to
"SGD"
. **kwargs
- Keyword arguments. Allowed to be one of
"clipnorm"
or"clipvalue"
."clipnorm"
(float) clips gradients by norm;"clipvalue"
(float) clips gradients by value.
Usage:
>>> opt = tf.keras.optimizers.SGD(learning_rate=0.1) >>> var = tf.Variable(1.0) >>> loss = lambda: (var ** 2)/2.0 # d(loss)/d(var1) = var1 >>> step_count = opt.minimize(loss, [var]).numpy() >>> # Step is `- learning_rate * grad` >>> var.numpy() 0.9
>>> opt = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9) >>> var = tf.Variable(1.0) >>> val0 = var.value() >>> loss = lambda: (var ** 2)/2.0 # d(loss)/d(var1) = var1 >>> # First step is `- learning_rate * grad` >>> step_count = opt.minimize(loss, [var]).numpy() >>> val1 = var.value() >>> (val0 - val1).numpy() 0.1 >>> # On later steps, step-size increases because of momentum >>> step_count = opt.minimize(loss, [var]).numpy() >>> val2 = var.value() >>> (val1 - val2).numpy() 0.18
Reference
- For
nesterov=True
, See Sutskever et al., 2013.
Create a new Optimizer.
This must be called by the constructors of subclasses. Note that Optimizer instances should not bind to a single graph, and so shouldn't keep Tensors as member variables. Generally you should be able to use the _set_hyper()/state.get_hyper() facility instead.
This class is stateful and thread-compatible.
Example of custom gradient transformations:
def my_gradient_transformer(grads_and_vars): # Simple example, double the gradients. return [(2. * g, v) for g, v in grads_and_vars] optimizer = tf.keras.optimizers.SGD( 1e-3, gradient_transformers=[my_gradient_transformer])
Args
name
- String. The name to use for momentum accumulator weights created by the optimizer.
gradient_aggregator
- The function to use to aggregate gradients across
devices (when using
tf.distribute.Strategy
). IfNone
, defaults to summing the gradients across devices. The function should accept and return a list of(gradient, variable)
tuples. gradient_transformers
- Optional. List of functions to use to transform
gradients before applying updates to Variables. The functions are
applied after
gradient_aggregator
. The functions should accept and return a list of(gradient, variable)
tuples. **kwargs
- keyword arguments. Allowed arguments are
clipvalue
,clipnorm
,global_clipnorm
. Ifclipvalue
(float) is set, the gradient of each weight is clipped to be no higher than this value. Ifclipnorm
(float) is set, the gradient of each weight is individually clipped so that its norm is no higher than this value. Ifglobal_clipnorm
(float) is set the gradient of all weights is clipped so that their global norm is no higher than this value.
Raises
ValueError
- in case of any invalid argument.
Expand source code
class SGD(optimizer_v2.OptimizerV2): r"""Gradient descent (with momentum) optimizer. Update rule for parameter `w` with gradient `g` when `momentum` is 0: ```python w = w - learning_rate * g ``` Update rule when `momentum` is larger than 0: ```python velocity = momentum * velocity - learning_rate * g w = w + velocity ``` When `nesterov=True`, this rule becomes: ```python velocity = momentum * velocity - learning_rate * g w = w + momentum * velocity - learning_rate * g ``` Args: learning_rate: A `Tensor`, floating point value, or a schedule that is a `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable that takes no arguments and returns the actual value to use. The learning rate. Defaults to 0.01. momentum: float hyperparameter >= 0 that accelerates gradient descent in the relevant direction and dampens oscillations. Defaults to 0, i.e., vanilla gradient descent. nesterov: boolean. Whether to apply Nesterov momentum. Defaults to `False`. name: Optional name prefix for the operations created when applying gradients. Defaults to `"SGD"`. **kwargs: Keyword arguments. Allowed to be one of `"clipnorm"` or `"clipvalue"`. `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips gradients by value. Usage: >>> opt = tf.keras.optimizers.SGD(learning_rate=0.1) >>> var = tf.Variable(1.0) >>> loss = lambda: (var ** 2)/2.0 # d(loss)/d(var1) = var1 >>> step_count = opt.minimize(loss, [var]).numpy() >>> # Step is `- learning_rate * grad` >>> var.numpy() 0.9 >>> opt = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9) >>> var = tf.Variable(1.0) >>> val0 = var.value() >>> loss = lambda: (var ** 2)/2.0 # d(loss)/d(var1) = var1 >>> # First step is `- learning_rate * grad` >>> step_count = opt.minimize(loss, [var]).numpy() >>> val1 = var.value() >>> (val0 - val1).numpy() 0.1 >>> # On later steps, step-size increases because of momentum >>> step_count = opt.minimize(loss, [var]).numpy() >>> val2 = var.value() >>> (val1 - val2).numpy() 0.18 Reference: - For `nesterov=True`, See [Sutskever et al., 2013]( http://jmlr.org/proceedings/papers/v28/sutskever13.pdf). """ _HAS_AGGREGATE_GRAD = True def __init__(self, learning_rate=0.01, momentum=0.0, nesterov=False, name="SGD", **kwargs): super(SGD, self).__init__(name, **kwargs) self._set_hyper("learning_rate", kwargs.get("lr", learning_rate)) self._set_hyper("decay", self._initial_decay) self._momentum = False if isinstance(momentum, tf.Tensor) or callable(momentum) or momentum > 0: self._momentum = True if isinstance(momentum, (int, float)) and (momentum < 0 or momentum > 1): raise ValueError("`momentum` must be between [0, 1].") self._set_hyper("momentum", momentum) self.nesterov = nesterov def _create_slots(self, var_list): if self._momentum: for var in var_list: self.add_slot(var, "momentum") def _prepare_local(self, var_device, var_dtype, apply_state): super(SGD, self)._prepare_local(var_device, var_dtype, apply_state) apply_state[(var_device, var_dtype)]["momentum"] = tf.identity( self._get_hyper("momentum", var_dtype)) def _resource_apply_dense(self, grad, var, apply_state=None): var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) if self._momentum: momentum_var = self.get_slot(var, "momentum") return tf.raw_ops.ResourceApplyKerasMomentum( var=var.handle, accum=momentum_var.handle, lr=coefficients["lr_t"], grad=grad, momentum=coefficients["momentum"], use_locking=self._use_locking, use_nesterov=self.nesterov) else: return tf.raw_ops.ResourceApplyGradientDescent( var=var.handle, alpha=coefficients["lr_t"], delta=grad, use_locking=self._use_locking) def _resource_apply_sparse_duplicate_indices(self, grad, var, indices, **kwargs): if self._momentum: return super(SGD, self)._resource_apply_sparse_duplicate_indices( grad, var, indices, **kwargs) else: var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = (kwargs.get("apply_state", {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) return tf.raw_ops.ResourceScatterAdd( resource=var.handle, indices=indices, updates=-grad * coefficients["lr_t"]) def _resource_apply_sparse(self, grad, var, indices, apply_state=None): # This method is only needed for momentum optimization. var_device, var_dtype = var.device, var.dtype.base_dtype coefficients = ((apply_state or {}).get((var_device, var_dtype)) or self._fallback_apply_state(var_device, var_dtype)) momentum_var = self.get_slot(var, "momentum") return tf.raw_ops.ResourceSparseApplyKerasMomentum( var=var.handle, accum=momentum_var.handle, lr=coefficients["lr_t"], grad=grad, indices=indices, momentum=coefficients["momentum"], use_locking=self._use_locking, use_nesterov=self.nesterov) def get_config(self): config = super(SGD, self).get_config() config.update({ "learning_rate": self._serialize_hyperparameter("learning_rate"), "decay": self._initial_decay, "momentum": self._serialize_hyperparameter("momentum"), "nesterov": self.nesterov, }) return config
Ancestors
- OptimizerV2
- tensorflow.python.training.tracking.base.Trackable
Inherited members