Module keras.regularizers

Built-in regularizers.

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Built-in regularizers."""

import tensorflow.compat.v2 as tf
# pylint: disable=invalid-name

import math

from keras import backend
from keras.utils.generic_utils import deserialize_keras_object
from keras.utils.generic_utils import serialize_keras_object
from tensorflow.python.util.tf_export import keras_export


def _check_penalty_number(x):
  """check penalty number availability, raise ValueError if failed."""
  if not isinstance(x, (float, int)):
    raise ValueError(('Value: {} is not a valid regularization penalty number, '
                      'expected an int or float value').format(x))

  if math.isinf(x) or math.isnan(x):
    raise ValueError(
        ('Value: {} is not a valid regularization penalty number, '
         'a positive/negative infinity or NaN is not a property value'
        ).format(x))


def _none_to_default(inputs, default):
  return default if inputs is None else default


@keras_export('keras.regularizers.Regularizer')
class Regularizer(object):
  """Regularizer base class.

  Regularizers allow you to apply penalties on layer parameters or layer
  activity during optimization. These penalties are summed into the loss
  function that the network optimizes.

  Regularization penalties are applied on a per-layer basis. The exact API will
  depend on the layer, but many layers (e.g. `Dense`, `Conv1D`, `Conv2D` and
  `Conv3D`) have a unified API.

  These layers expose 3 keyword arguments:

  - `kernel_regularizer`: Regularizer to apply a penalty on the layer's kernel
  - `bias_regularizer`: Regularizer to apply a penalty on the layer's bias
  - `activity_regularizer`: Regularizer to apply a penalty on the layer's output

  All layers (including custom layers) expose `activity_regularizer` as a
  settable property, whether or not it is in the constructor arguments.

  The value returned by the `activity_regularizer` is divided by the input
  batch size so that the relative weighting between the weight regularizers and
  the activity regularizers does not change with the batch size.

  You can access a layer's regularization penalties by calling `layer.losses`
  after calling the layer on inputs.

  ## Example

  >>> layer = tf.keras.layers.Dense(
  ...     5, input_dim=5,
  ...     kernel_initializer='ones',
  ...     kernel_regularizer=tf.keras.regularizers.L1(0.01),
  ...     activity_regularizer=tf.keras.regularizers.L2(0.01))
  >>> tensor = tf.ones(shape=(5, 5)) * 2.0
  >>> out = layer(tensor)

  >>> # The kernel regularization term is 0.25
  >>> # The activity regularization term (after dividing by the batch size) is 5
  >>> tf.math.reduce_sum(layer.losses)
  <tf.Tensor: shape=(), dtype=float32, numpy=5.25>

  ## Available penalties

  ```python
  tf.keras.regularizers.L1(0.3)  # L1 Regularization Penalty
  tf.keras.regularizers.L2(0.1)  # L2 Regularization Penalty
  tf.keras.regularizers.L1L2(l1=0.01, l2=0.01)  # L1 + L2 penalties
  ```

  ## Directly calling a regularizer

  Compute a regularization loss on a tensor by directly calling a regularizer
  as if it is a one-argument function.

  E.g.
  >>> regularizer = tf.keras.regularizers.L2(2.)
  >>> tensor = tf.ones(shape=(5, 5))
  >>> regularizer(tensor)
  <tf.Tensor: shape=(), dtype=float32, numpy=50.0>


  ## Developing new regularizers

  Any function that takes in a weight matrix and returns a scalar
  tensor can be used as a regularizer, e.g.:

  >>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l1')
  ... def l1_reg(weight_matrix):
  ...    return 0.01 * tf.math.reduce_sum(tf.math.abs(weight_matrix))
  ...
  >>> layer = tf.keras.layers.Dense(5, input_dim=5,
  ...     kernel_initializer='ones', kernel_regularizer=l1_reg)
  >>> tensor = tf.ones(shape=(5, 5))
  >>> out = layer(tensor)
  >>> layer.losses
  [<tf.Tensor: shape=(), dtype=float32, numpy=0.25>]

  Alternatively, you can write your custom regularizers in an
  object-oriented way by extending this regularizer base class, e.g.:

  >>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l2')
  ... class L2Regularizer(tf.keras.regularizers.Regularizer):
  ...   def __init__(self, l2=0.):  # pylint: disable=redefined-outer-name
  ...     self.l2 = l2
  ...
  ...   def __call__(self, x):
  ...     return self.l2 * tf.math.reduce_sum(tf.math.square(x))
  ...
  ...   def get_config(self):
  ...     return {'l2': float(self.l2)}
  ...
  >>> layer = tf.keras.layers.Dense(
  ...   5, input_dim=5, kernel_initializer='ones',
  ...   kernel_regularizer=L2Regularizer(l2=0.5))

  >>> tensor = tf.ones(shape=(5, 5))
  >>> out = layer(tensor)
  >>> layer.losses
  [<tf.Tensor: shape=(), dtype=float32, numpy=12.5>]

  ### A note on serialization and deserialization:

  Registering the regularizers as serializable is optional if you are just
  training and executing models, exporting to and from SavedModels, or saving
  and loading weight checkpoints.

  Registration is required for Keras `model_to_estimator`, saving and
  loading models to HDF5 formats, Keras model cloning, some visualization
  utilities, and exporting models to and from JSON. If using this functionality,
  you must make sure any python process running your model has also defined
  and registered your custom regularizer.

  `tf.keras.utils.register_keras_serializable` is only available in TF 2.1 and
  beyond. In earlier versions of TensorFlow you must pass your custom
  regularizer to the `custom_objects` argument of methods that expect custom
  regularizers to be registered as serializable.
  """

  def __call__(self, x):
    """Compute a regularization penalty from an input tensor."""
    return 0.

  @classmethod
  def from_config(cls, config):
    """Creates a regularizer from its config.

    This method is the reverse of `get_config`,
    capable of instantiating the same regularizer from the config
    dictionary.

    This method is used by Keras `model_to_estimator`, saving and
    loading models to HDF5 formats, Keras model cloning, some visualization
    utilities, and exporting models to and from JSON.

    Args:
        config: A Python dictionary, typically the output of get_config.

    Returns:
        A regularizer instance.
    """
    return cls(**config)

  def get_config(self):
    """Returns the config of the regularizer.

    An regularizer config is a Python dictionary (serializable)
    containing all configuration parameters of the regularizer.
    The same regularizer can be reinstantiated later
    (without any saved state) from this configuration.

    This method is optional if you are just training and executing models,
    exporting to and from SavedModels, or using weight checkpoints.

    This method is required for Keras `model_to_estimator`, saving and
    loading models to HDF5 formats, Keras model cloning, some visualization
    utilities, and exporting models to and from JSON.

    Returns:
        Python dictionary.
    """
    raise NotImplementedError(str(self) + ' does not implement get_config()')


@keras_export('keras.regularizers.L1L2')
class L1L2(Regularizer):
  """A regularizer that applies both L1 and L2 regularization penalties.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  The L2 regularization penalty is computed as
  `loss = l2 * reduce_sum(square(x))`

  L1L2 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1_l2')

  In this case, the default values used are `l1=0.01` and `l2=0.01`.

  Attributes:
      l1: Float; L1 regularization factor.
      l2: Float; L2 regularization factor.
  """

  def __init__(self, l1=0., l2=0.):  # pylint: disable=redefined-outer-name
    # The default value for l1 and l2 are different from the value in l1_l2
    # for backward compatibility reason. Eg, L1L2(l2=0.1) will only have l2
    # and no l1 penalty.
    l1 = 0. if l1 is None else l1
    l2 = 0. if l2 is None else l2
    _check_penalty_number(l1)
    _check_penalty_number(l2)

    self.l1 = backend.cast_to_floatx(l1)
    self.l2 = backend.cast_to_floatx(l2)

  def __call__(self, x):
    regularization = backend.constant(0., dtype=x.dtype)
    if self.l1:
      regularization += self.l1 * tf.reduce_sum(tf.abs(x))
    if self.l2:
      regularization += self.l2 * tf.reduce_sum(tf.square(x))
    return regularization

  def get_config(self):
    return {'l1': float(self.l1), 'l2': float(self.l2)}


@keras_export('keras.regularizers.L1', 'keras.regularizers.l1')
class L1(Regularizer):
  """A regularizer that applies a L1 regularization penalty.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  L1 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1')

  In this case, the default value used is `l1=0.01`.

  Attributes:
      l1: Float; L1 regularization factor.
  """

  def __init__(self, l1=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l1 = kwargs.pop('l', l1)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l1 = 0.01 if l1 is None else l1
    _check_penalty_number(l1)

    self.l1 = backend.cast_to_floatx(l1)

  def __call__(self, x):
    return self.l1 * tf.reduce_sum(tf.abs(x))

  def get_config(self):
    return {'l1': float(self.l1)}


@keras_export('keras.regularizers.L2', 'keras.regularizers.l2')
class L2(Regularizer):
  """A regularizer that applies a L2 regularization penalty.

  The L2 regularization penalty is computed as:
  `loss = l2 * reduce_sum(square(x))`

  L2 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l2')

  In this case, the default value used is `l2=0.01`.

  Attributes:
      l2: Float; L2 regularization factor.
  """

  def __init__(self, l2=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l2 = kwargs.pop('l', l2)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l2 = 0.01 if l2 is None else l2
    _check_penalty_number(l2)

    self.l2 = backend.cast_to_floatx(l2)

  def __call__(self, x):
    return self.l2 * tf.reduce_sum(tf.square(x))

  def get_config(self):
    return {'l2': float(self.l2)}


@keras_export('keras.regularizers.l1_l2')
def l1_l2(l1=0.01, l2=0.01):  # pylint: disable=redefined-outer-name
  r"""Create a regularizer that applies both L1 and L2 penalties.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  The L2 regularization penalty is computed as:
  `loss = l2 * reduce_sum(square(x))`

  Args:
      l1: Float; L1 regularization factor.
      l2: Float; L2 regularization factor.

  Returns:
    An L1L2 Regularizer with the given regularization factors.
  """
  return L1L2(l1=l1, l2=l2)


# Deserialization aliases.
l1 = L1
l2 = L2


@keras_export('keras.regularizers.serialize')
def serialize(regularizer):
  return serialize_keras_object(regularizer)


@keras_export('keras.regularizers.deserialize')
def deserialize(config, custom_objects=None):
  if config == 'l1_l2':
    # Special case necessary since the defaults used for "l1_l2" (string)
    # differ from those of the L1L2 class.
    return L1L2(l1=0.01, l2=0.01)
  return deserialize_keras_object(
      config,
      module_objects=globals(),
      custom_objects=custom_objects,
      printable_module_name='regularizer')


@keras_export('keras.regularizers.get')
def get(identifier):
  """Retrieve a regularizer instance from a config or identifier."""
  if identifier is None:
    return None
  if isinstance(identifier, dict):
    return deserialize(identifier)
  elif isinstance(identifier, str):
    return deserialize(str(identifier))
  elif callable(identifier):
    return identifier
  else:
    raise ValueError(
        'Could not interpret regularizer identifier: {}'.format(identifier))

Functions

def deserialize(config, custom_objects=None)

Expand source code

@keras_export('keras.regularizers.deserialize')
def deserialize(config, custom_objects=None):
  if config == 'l1_l2':
    # Special case necessary since the defaults used for "l1_l2" (string)
    # differ from those of the L1L2 class.
    return L1L2(l1=0.01, l2=0.01)
  return deserialize_keras_object(
      config,
      module_objects=globals(),
      custom_objects=custom_objects,
      printable_module_name='regularizer')

def get(identifier)

Retrieve a regularizer instance from a config or identifier.

Expand source code

@keras_export('keras.regularizers.get')
def get(identifier):
  """Retrieve a regularizer instance from a config or identifier."""
  if identifier is None:
    return None
  if isinstance(identifier, dict):
    return deserialize(identifier)
  elif isinstance(identifier, str):
    return deserialize(str(identifier))
  elif callable(identifier):
    return identifier
  else:
    raise ValueError(
        'Could not interpret regularizer identifier: {}'.format(identifier))

def l1_l2(l1=0.01, l2=0.01)

Create a regularizer that applies both L1 and L2 penalties.

The L1 regularization penalty is computed as: loss = l1 * reduce_sum(abs(x))

The L2 regularization penalty is computed as: loss = l2 * reduce_sum(square(x))

Args

l1

Float; L1 regularization factor.

l2

Float; L2 regularization factor.

Returns

An L1L2 Regularizer with the given regularization factors.

Expand source code

@keras_export('keras.regularizers.l1_l2')
def l1_l2(l1=0.01, l2=0.01):  # pylint: disable=redefined-outer-name
  r"""Create a regularizer that applies both L1 and L2 penalties.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  The L2 regularization penalty is computed as:
  `loss = l2 * reduce_sum(square(x))`

  Args:
      l1: Float; L1 regularization factor.
      l2: Float; L2 regularization factor.

  Returns:
    An L1L2 Regularizer with the given regularization factors.
  """
  return L1L2(l1=l1, l2=l2)

def serialize(regularizer)

Expand source code

@keras_export('keras.regularizers.serialize')
def serialize(regularizer):
  return serialize_keras_object(regularizer)

Classes

class L1 (l1=0.01, **kwargs)

A regularizer that applies a L1 regularization penalty.

The L1 regularization penalty is computed as: loss = l1 * reduce_sum(abs(x))

L1 may be passed to a layer as a string identifier:

>>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1')

In this case, the default value used is l1=0.01.

Attributes

l1

Float; L1 regularization factor.

Expand source code

class L1(Regularizer):
  """A regularizer that applies a L1 regularization penalty.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  L1 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1')

  In this case, the default value used is `l1=0.01`.

  Attributes:
      l1: Float; L1 regularization factor.
  """

  def __init__(self, l1=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l1 = kwargs.pop('l', l1)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l1 = 0.01 if l1 is None else l1
    _check_penalty_number(l1)

    self.l1 = backend.cast_to_floatx(l1)

  def __call__(self, x):
    return self.l1 * tf.reduce_sum(tf.abs(x))

  def get_config(self):
    return {'l1': float(self.l1)}

Ancestors

• Regularizer

Methods

def get_config(self)

Returns the config of the regularizer.

An regularizer config is a Python dictionary (serializable) containing all configuration parameters of the regularizer. The same regularizer can be reinstantiated later (without any saved state) from this configuration.

This method is optional if you are just training and executing models, exporting to and from SavedModels, or using weight checkpoints.

This method is required for Keras model_to_estimator, saving and loading models to HDF5 formats, Keras model cloning, some visualization utilities, and exporting models to and from JSON.

Returns

Python dictionary.

Expand source code

def get_config(self):
  return {'l1': float(self.l1)}

class l1 (l1=0.01, **kwargs)

A regularizer that applies a L1 regularization penalty.

The L1 regularization penalty is computed as: loss = l1 * reduce_sum(abs(x))

L1 may be passed to a layer as a string identifier:

>>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1')

In this case, the default value used is l1=0.01.

Attributes

l1

Float; L1 regularization factor.

Expand source code

class L1(Regularizer):
  """A regularizer that applies a L1 regularization penalty.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  L1 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1')

  In this case, the default value used is `l1=0.01`.

  Attributes:
      l1: Float; L1 regularization factor.
  """

  def __init__(self, l1=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l1 = kwargs.pop('l', l1)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l1 = 0.01 if l1 is None else l1
    _check_penalty_number(l1)

    self.l1 = backend.cast_to_floatx(l1)

  def __call__(self, x):
    return self.l1 * tf.reduce_sum(tf.abs(x))

  def get_config(self):
    return {'l1': float(self.l1)}

Ancestors

• Regularizer

Inherited members

• Regularizer:
  • from_config
  • get_config

class L1L2 (l1=0.0, l2=0.0)

A regularizer that applies both L1 and L2 regularization penalties.

The L1 regularization penalty is computed as: loss = l1 * reduce_sum(abs(x))

The L2 regularization penalty is computed as loss = l2 * reduce_sum(square(x))

L1L2 may be passed to a layer as a string identifier:

>>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1_l2')

In this case, the default values used are l1=0.01 and l2=0.01.

Attributes

l1

Float; L1 regularization factor.

l2

Float; L2 regularization factor.

Expand source code

class L1L2(Regularizer):
  """A regularizer that applies both L1 and L2 regularization penalties.

  The L1 regularization penalty is computed as:
  `loss = l1 * reduce_sum(abs(x))`

  The L2 regularization penalty is computed as
  `loss = l2 * reduce_sum(square(x))`

  L1L2 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l1_l2')

  In this case, the default values used are `l1=0.01` and `l2=0.01`.

  Attributes:
      l1: Float; L1 regularization factor.
      l2: Float; L2 regularization factor.
  """

  def __init__(self, l1=0., l2=0.):  # pylint: disable=redefined-outer-name
    # The default value for l1 and l2 are different from the value in l1_l2
    # for backward compatibility reason. Eg, L1L2(l2=0.1) will only have l2
    # and no l1 penalty.
    l1 = 0. if l1 is None else l1
    l2 = 0. if l2 is None else l2
    _check_penalty_number(l1)
    _check_penalty_number(l2)

    self.l1 = backend.cast_to_floatx(l1)
    self.l2 = backend.cast_to_floatx(l2)

  def __call__(self, x):
    regularization = backend.constant(0., dtype=x.dtype)
    if self.l1:
      regularization += self.l1 * tf.reduce_sum(tf.abs(x))
    if self.l2:
      regularization += self.l2 * tf.reduce_sum(tf.square(x))
    return regularization

  def get_config(self):
    return {'l1': float(self.l1), 'l2': float(self.l2)}

Ancestors

• Regularizer

Inherited members

• Regularizer:
  • from_config
  • get_config

class L2 (l2=0.01, **kwargs)

A regularizer that applies a L2 regularization penalty.

The L2 regularization penalty is computed as: loss = l2 * reduce_sum(square(x))

L2 may be passed to a layer as a string identifier:

>>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l2')

In this case, the default value used is l2=0.01.

Attributes

l2

Float; L2 regularization factor.

Expand source code

class L2(Regularizer):
  """A regularizer that applies a L2 regularization penalty.

  The L2 regularization penalty is computed as:
  `loss = l2 * reduce_sum(square(x))`

  L2 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l2')

  In this case, the default value used is `l2=0.01`.

  Attributes:
      l2: Float; L2 regularization factor.
  """

  def __init__(self, l2=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l2 = kwargs.pop('l', l2)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l2 = 0.01 if l2 is None else l2
    _check_penalty_number(l2)

    self.l2 = backend.cast_to_floatx(l2)

  def __call__(self, x):
    return self.l2 * tf.reduce_sum(tf.square(x))

  def get_config(self):
    return {'l2': float(self.l2)}

Ancestors

• Regularizer

Methods

def get_config(self)

Returns the config of the regularizer.

An regularizer config is a Python dictionary (serializable) containing all configuration parameters of the regularizer. The same regularizer can be reinstantiated later (without any saved state) from this configuration.

This method is optional if you are just training and executing models, exporting to and from SavedModels, or using weight checkpoints.

This method is required for Keras model_to_estimator, saving and loading models to HDF5 formats, Keras model cloning, some visualization utilities, and exporting models to and from JSON.

Returns

Python dictionary.

Expand source code

def get_config(self):
  return {'l2': float(self.l2)}

class l2 (l2=0.01, **kwargs)

A regularizer that applies a L2 regularization penalty.

The L2 regularization penalty is computed as: loss = l2 * reduce_sum(square(x))

L2 may be passed to a layer as a string identifier:

>>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l2')

In this case, the default value used is l2=0.01.

Attributes

l2

Float; L2 regularization factor.

Expand source code

class L2(Regularizer):
  """A regularizer that applies a L2 regularization penalty.

  The L2 regularization penalty is computed as:
  `loss = l2 * reduce_sum(square(x))`

  L2 may be passed to a layer as a string identifier:

  >>> dense = tf.keras.layers.Dense(3, kernel_regularizer='l2')

  In this case, the default value used is `l2=0.01`.

  Attributes:
      l2: Float; L2 regularization factor.
  """

  def __init__(self, l2=0.01, **kwargs):  # pylint: disable=redefined-outer-name
    l2 = kwargs.pop('l', l2)  # Backwards compatibility
    if kwargs:
      raise TypeError('Argument(s) not recognized: %s' % (kwargs,))

    l2 = 0.01 if l2 is None else l2
    _check_penalty_number(l2)

    self.l2 = backend.cast_to_floatx(l2)

  def __call__(self, x):
    return self.l2 * tf.reduce_sum(tf.square(x))

  def get_config(self):
    return {'l2': float(self.l2)}

Ancestors

• Regularizer

Inherited members

• Regularizer:
  • from_config
  • get_config

class Regularizer

Regularizer base class.

Regularizers allow you to apply penalties on layer parameters or layer activity during optimization. These penalties are summed into the loss function that the network optimizes.

Regularization penalties are applied on a per-layer basis. The exact API will depend on the layer, but many layers (e.g. Dense, Conv1D, Conv2D and Conv3D) have a unified API.

These layers expose 3 keyword arguments:

• kernel_regularizer: Regularizer to apply a penalty on the layer's kernel
• bias_regularizer: Regularizer to apply a penalty on the layer's bias
• activity_regularizer: Regularizer to apply a penalty on the layer's output

All layers (including custom layers) expose activity_regularizer as a settable property, whether or not it is in the constructor arguments.

The value returned by the activity_regularizer is divided by the input batch size so that the relative weighting between the weight regularizers and the activity regularizers does not change with the batch size.

You can access a layer's regularization penalties by calling layer.losses after calling the layer on inputs.

Example

>>> layer = tf.keras.layers.Dense(
...     5, input_dim=5,
...     kernel_initializer='ones',
...     kernel_regularizer=tf.keras.regularizers.L1(0.01),
...     activity_regularizer=tf.keras.regularizers.L2(0.01))
>>> tensor = tf.ones(shape=(5, 5)) * 2.0
>>> out = layer(tensor)

>>> # The kernel regularization term is 0.25
>>> # The activity regularization term (after dividing by the batch size) is 5
>>> tf.math.reduce_sum(layer.losses)
<tf.Tensor: shape=(), dtype=float32, numpy=5.25>

Available penalties

tf.keras.regularizers.L1(0.3)  # L1 Regularization Penalty
tf.keras.regularizers.L2(0.1)  # L2 Regularization Penalty
tf.keras.regularizers.L1L2(l1=0.01, l2=0.01)  # L1 + L2 penalties

Directly calling a regularizer

Compute a regularization loss on a tensor by directly calling a regularizer as if it is a one-argument function.

E.g.

>>> regularizer = tf.keras.regularizers.L2(2.)
>>> tensor = tf.ones(shape=(5, 5))
>>> regularizer(tensor)
<tf.Tensor: shape=(), dtype=float32, numpy=50.0>

Developing new regularizers

Any function that takes in a weight matrix and returns a scalar tensor can be used as a regularizer, e.g.:

>>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l1')
... def l1_reg(weight_matrix):
...    return 0.01 * tf.math.reduce_sum(tf.math.abs(weight_matrix))
...
>>> layer = tf.keras.layers.Dense(5, input_dim=5,
...     kernel_initializer='ones', kernel_regularizer=l1_reg)
>>> tensor = tf.ones(shape=(5, 5))
>>> out = layer(tensor)
>>> layer.losses
[<tf.Tensor: shape=(), dtype=float32, numpy=0.25>]

Alternatively, you can write your custom regularizers in an object-oriented way by extending this regularizer base class, e.g.:

>>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l2')
... class L2Regularizer(tf.keras.regularizers.Regularizer):
...   def __init__(self, l2=0.):  # pylint: disable=redefined-outer-name
...     self.l2 = l2
...
...   def __call__(self, x):
...     return self.l2 * tf.math.reduce_sum(tf.math.square(x))
...
...   def get_config(self):
...     return {'l2': float(self.l2)}
...
>>> layer = tf.keras.layers.Dense(
...   5, input_dim=5, kernel_initializer='ones',
...   kernel_regularizer=L2Regularizer(l2=0.5))

>>> tensor = tf.ones(shape=(5, 5))
>>> out = layer(tensor)
>>> layer.losses
[<tf.Tensor: shape=(), dtype=float32, numpy=12.5>]

A note on serialization and deserialization:

Registering the regularizers as serializable is optional if you are just training and executing models, exporting to and from SavedModels, or saving and loading weight checkpoints.

Registration is required for Keras model_to_estimator, saving and loading models to HDF5 formats, Keras model cloning, some visualization utilities, and exporting models to and from JSON. If using this functionality, you must make sure any python process running your model has also defined and registered your custom regularizer.

tf.keras.utils.register_keras_serializable is only available in TF 2.1 and beyond. In earlier versions of TensorFlow you must pass your custom regularizer to the custom_objects argument of methods that expect custom regularizers to be registered as serializable.

Expand source code

class Regularizer(object):
  """Regularizer base class.

  Regularizers allow you to apply penalties on layer parameters or layer
  activity during optimization. These penalties are summed into the loss
  function that the network optimizes.

  Regularization penalties are applied on a per-layer basis. The exact API will
  depend on the layer, but many layers (e.g. `Dense`, `Conv1D`, `Conv2D` and
  `Conv3D`) have a unified API.

  These layers expose 3 keyword arguments:

  - `kernel_regularizer`: Regularizer to apply a penalty on the layer's kernel
  - `bias_regularizer`: Regularizer to apply a penalty on the layer's bias
  - `activity_regularizer`: Regularizer to apply a penalty on the layer's output

  All layers (including custom layers) expose `activity_regularizer` as a
  settable property, whether or not it is in the constructor arguments.

  The value returned by the `activity_regularizer` is divided by the input
  batch size so that the relative weighting between the weight regularizers and
  the activity regularizers does not change with the batch size.

  You can access a layer's regularization penalties by calling `layer.losses`
  after calling the layer on inputs.

  ## Example

  >>> layer = tf.keras.layers.Dense(
  ...     5, input_dim=5,
  ...     kernel_initializer='ones',
  ...     kernel_regularizer=tf.keras.regularizers.L1(0.01),
  ...     activity_regularizer=tf.keras.regularizers.L2(0.01))
  >>> tensor = tf.ones(shape=(5, 5)) * 2.0
  >>> out = layer(tensor)

  >>> # The kernel regularization term is 0.25
  >>> # The activity regularization term (after dividing by the batch size) is 5
  >>> tf.math.reduce_sum(layer.losses)
  <tf.Tensor: shape=(), dtype=float32, numpy=5.25>

  ## Available penalties

  ```python
  tf.keras.regularizers.L1(0.3)  # L1 Regularization Penalty
  tf.keras.regularizers.L2(0.1)  # L2 Regularization Penalty
  tf.keras.regularizers.L1L2(l1=0.01, l2=0.01)  # L1 + L2 penalties
  ```

  ## Directly calling a regularizer

  Compute a regularization loss on a tensor by directly calling a regularizer
  as if it is a one-argument function.

  E.g.
  >>> regularizer = tf.keras.regularizers.L2(2.)
  >>> tensor = tf.ones(shape=(5, 5))
  >>> regularizer(tensor)
  <tf.Tensor: shape=(), dtype=float32, numpy=50.0>


  ## Developing new regularizers

  Any function that takes in a weight matrix and returns a scalar
  tensor can be used as a regularizer, e.g.:

  >>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l1')
  ... def l1_reg(weight_matrix):
  ...    return 0.01 * tf.math.reduce_sum(tf.math.abs(weight_matrix))
  ...
  >>> layer = tf.keras.layers.Dense(5, input_dim=5,
  ...     kernel_initializer='ones', kernel_regularizer=l1_reg)
  >>> tensor = tf.ones(shape=(5, 5))
  >>> out = layer(tensor)
  >>> layer.losses
  [<tf.Tensor: shape=(), dtype=float32, numpy=0.25>]

  Alternatively, you can write your custom regularizers in an
  object-oriented way by extending this regularizer base class, e.g.:

  >>> @tf.keras.utils.register_keras_serializable(package='Custom', name='l2')
  ... class L2Regularizer(tf.keras.regularizers.Regularizer):
  ...   def __init__(self, l2=0.):  # pylint: disable=redefined-outer-name
  ...     self.l2 = l2
  ...
  ...   def __call__(self, x):
  ...     return self.l2 * tf.math.reduce_sum(tf.math.square(x))
  ...
  ...   def get_config(self):
  ...     return {'l2': float(self.l2)}
  ...
  >>> layer = tf.keras.layers.Dense(
  ...   5, input_dim=5, kernel_initializer='ones',
  ...   kernel_regularizer=L2Regularizer(l2=0.5))

  >>> tensor = tf.ones(shape=(5, 5))
  >>> out = layer(tensor)
  >>> layer.losses
  [<tf.Tensor: shape=(), dtype=float32, numpy=12.5>]

  ### A note on serialization and deserialization:

  Registering the regularizers as serializable is optional if you are just
  training and executing models, exporting to and from SavedModels, or saving
  and loading weight checkpoints.

  Registration is required for Keras `model_to_estimator`, saving and
  loading models to HDF5 formats, Keras model cloning, some visualization
  utilities, and exporting models to and from JSON. If using this functionality,
  you must make sure any python process running your model has also defined
  and registered your custom regularizer.

  `tf.keras.utils.register_keras_serializable` is only available in TF 2.1 and
  beyond. In earlier versions of TensorFlow you must pass your custom
  regularizer to the `custom_objects` argument of methods that expect custom
  regularizers to be registered as serializable.
  """

  def __call__(self, x):
    """Compute a regularization penalty from an input tensor."""
    return 0.

  @classmethod
  def from_config(cls, config):
    """Creates a regularizer from its config.

    This method is the reverse of `get_config`,
    capable of instantiating the same regularizer from the config
    dictionary.

    This method is used by Keras `model_to_estimator`, saving and
    loading models to HDF5 formats, Keras model cloning, some visualization
    utilities, and exporting models to and from JSON.

    Args:
        config: A Python dictionary, typically the output of get_config.

    Returns:
        A regularizer instance.
    """
    return cls(**config)

  def get_config(self):
    """Returns the config of the regularizer.

    An regularizer config is a Python dictionary (serializable)
    containing all configuration parameters of the regularizer.
    The same regularizer can be reinstantiated later
    (without any saved state) from this configuration.

    This method is optional if you are just training and executing models,
    exporting to and from SavedModels, or using weight checkpoints.

    This method is required for Keras `model_to_estimator`, saving and
    loading models to HDF5 formats, Keras model cloning, some visualization
    utilities, and exporting models to and from JSON.

    Returns:
        Python dictionary.
    """
    raise NotImplementedError(str(self) + ' does not implement get_config()')

Subclasses

• TestRegularizerLoss.IdentityRegularizer
• IdentityRegularizer
• ReduceSumRegularizer
• L1
• L1L2
• L2

Static methods

def from_config(config)

Creates a regularizer from its config.

This method is the reverse of get_config, capable of instantiating the same regularizer from the config dictionary.

This method is used by Keras model_to_estimator, saving and loading models to HDF5 formats, Keras model cloning, some visualization utilities, and exporting models to and from JSON.

Args

config

A Python dictionary, typically the output of get_config.

Returns

A regularizer instance.

Expand source code

@classmethod
def from_config(cls, config):
  """Creates a regularizer from its config.

  This method is the reverse of `get_config`,
  capable of instantiating the same regularizer from the config
  dictionary.

  This method is used by Keras `model_to_estimator`, saving and
  loading models to HDF5 formats, Keras model cloning, some visualization
  utilities, and exporting models to and from JSON.

  Args:
      config: A Python dictionary, typically the output of get_config.

  Returns:
      A regularizer instance.
  """
  return cls(**config)

Methods

def get_config(self)

Returns the config of the regularizer.

An regularizer config is a Python dictionary (serializable) containing all configuration parameters of the regularizer. The same regularizer can be reinstantiated later (without any saved state) from this configuration.

This method is optional if you are just training and executing models, exporting to and from SavedModels, or using weight checkpoints.

This method is required for Keras model_to_estimator, saving and loading models to HDF5 formats, Keras model cloning, some visualization utilities, and exporting models to and from JSON.

Returns

Python dictionary.

Expand source code

def get_config(self):
  """Returns the config of the regularizer.

  An regularizer config is a Python dictionary (serializable)
  containing all configuration parameters of the regularizer.
  The same regularizer can be reinstantiated later
  (without any saved state) from this configuration.

  This method is optional if you are just training and executing models,
  exporting to and from SavedModels, or using weight checkpoints.

  This method is required for Keras `model_to_estimator`, saving and
  loading models to HDF5 formats, Keras model cloning, some visualization
  utilities, and exporting models to and from JSON.

  Returns:
      Python dictionary.
  """
  raise NotImplementedError(str(self) + ' does not implement get_config()')