Module keras.api.keras.layers.experimental

Public API for tf.keras.layers.experimental namespace.

Expand source code
# This file is MACHINE GENERATED! Do not edit.
# Generated by: tensorflow/python/tools/api/generator/create_python_api.py script.
"""Public API for tf.keras.layers.experimental namespace.
"""

from __future__ import print_function as _print_function

import sys as _sys

from keras.api.keras.layers.experimental import preprocessing
from keras.layers.einsum_dense import EinsumDense
from keras.layers.kernelized import RandomFourierFeatures

del _print_function

from tensorflow.python.util import module_wrapper as _module_wrapper

if not isinstance(_sys.modules[__name__], _module_wrapper.TFModuleWrapper):
  _sys.modules[__name__] = _module_wrapper.TFModuleWrapper(
      _sys.modules[__name__], "keras.layers.experimental", public_apis=None, deprecation=True,
      has_lite=False)

Sub-modules

keras.api.keras.layers.experimental.preprocessing

Public API for tf.keras.layers.experimental.preprocessing namespace.

Classes

class EinsumDense (equation, output_shape, activation=None, bias_axes=None, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs)

A layer that uses tf.einsum as the backing computation.

This layer can perform einsum calculations of arbitrary dimensionality.

Args

equation
An equation describing the einsum to perform. This equation must be a valid einsum string of the form ab,bc->ac, ...ab,bc->...ac, or ab...,bc->ac... where 'ab', 'bc', and 'ac' can be any valid einsum axis expression sequence.
output_shape
The expected shape of the output tensor (excluding the batch dimension and any dimensions represented by ellipses). You can specify None for any dimension that is unknown or can be inferred from the input shape.
activation
Activation function to use. If you don't specify anything, no activation is applied (that is, a "linear" activation: a(x) = x).
bias_axes
A string containing the output dimension(s) to apply a bias to. Each character in the bias_axes string should correspond to a character in the output portion of the equation string.
kernel_initializer
Initializer for the kernel weights matrix.
bias_initializer
Initializer for the bias vector.
kernel_regularizer
Regularizer function applied to the kernel weights matrix.
bias_regularizer
Regularizer function applied to the bias vector.
activity_regularizer
Regularizer function applied to the output of the layer (its "activation")..
kernel_constraint
Constraint function applied to the kernel weights matrix.
bias_constraint
Constraint function applied to the bias vector.

Examples:

Biased dense layer with einsums

This example shows how to instantiate a standard Keras dense layer using einsum operations. This example is equivalent to tf.keras.layers.Dense(64, use_bias=True).

>>> layer = EinsumDense("ab,bc->ac", output_shape=64, bias_axes="c")
>>> input_tensor = tf.keras.Input(shape=[32])
>>> output_tensor = layer(input_tensor)
>>> output_tensor
<... shape=(None, 64) dtype=...>

Applying a dense layer to a sequence

This example shows how to instantiate a layer that applies the same dense operation to every element in a sequence. Here, the 'output_shape' has two values (since there are two non-batch dimensions in the output); the first dimension in the output_shape is None, because the sequence dimension b has an unknown shape.

>>> layer = EinsumDense("abc,cd->abd",
...                     output_shape=(None, 64),
...                     bias_axes="d")
>>> input_tensor = tf.keras.Input(shape=[32, 128])
>>> output_tensor = layer(input_tensor)
>>> output_tensor
<... shape=(None, 32, 64) dtype=...>

Applying a dense layer to a sequence using ellipses

This example shows how to instantiate a layer that applies the same dense operation to every element in a sequence, but uses the ellipsis notation instead of specifying the batch and sequence dimensions.

Because we are using ellipsis notation and have specified only one axis, the output_shape arg is a single value. When instantiated in this way, the layer can handle any number of sequence dimensions - including the case where no sequence dimension exists.

>>> layer = EinsumDense("...x,xy->...y", output_shape=64, bias_axes="y")
>>> input_tensor = tf.keras.Input(shape=[32, 128])
>>> output_tensor = layer(input_tensor)
>>> output_tensor
<... shape=(None, 32, 64) dtype=...>
Expand source code
class EinsumDense(Layer):
  """A layer that uses tf.einsum as the backing computation.

  This layer can perform einsum calculations of arbitrary dimensionality.

  Args:
    equation: An equation describing the einsum to perform. This equation must
      be a valid einsum string of the form `ab,bc->ac`, `...ab,bc->...ac`, or
      `ab...,bc->ac...` where 'ab', 'bc', and 'ac' can be any valid einsum axis
      expression sequence.
    output_shape: The expected shape of the output tensor (excluding the batch
      dimension and any dimensions represented by ellipses). You can specify
      None for any dimension that is unknown or can be inferred from the input
      shape.
    activation: Activation function to use. If you don't specify anything, no
      activation is applied (that is, a "linear" activation: `a(x) = x`).
    bias_axes: A string containing the output dimension(s) to apply a bias to.
      Each character in the `bias_axes` string should correspond to a character
      in the output portion of the `equation` string.
    kernel_initializer: Initializer for the `kernel` weights matrix.
    bias_initializer: Initializer for the bias vector.
    kernel_regularizer: Regularizer function applied to the `kernel` weights
      matrix.
    bias_regularizer: Regularizer function applied to the bias vector.
    activity_regularizer: Regularizer function applied to the output of the
      layer (its "activation")..
    kernel_constraint: Constraint function applied to the `kernel` weights
      matrix.
    bias_constraint: Constraint function applied to the bias vector.

  Examples:

  **Biased dense layer with einsums**

  This example shows how to instantiate a standard Keras dense layer using
  einsum operations. This example is equivalent to
  `tf.keras.layers.Dense(64, use_bias=True)`.

  >>> layer = EinsumDense("ab,bc->ac", output_shape=64, bias_axes="c")
  >>> input_tensor = tf.keras.Input(shape=[32])
  >>> output_tensor = layer(input_tensor)
  >>> output_tensor
  <... shape=(None, 64) dtype=...>

  **Applying a dense layer to a sequence**

  This example shows how to instantiate a layer that applies the same dense
  operation to every element in a sequence. Here, the 'output_shape' has two
  values (since there are two non-batch dimensions in the output); the first
  dimension in the output_shape is `None`, because the sequence dimension `b`
  has an unknown shape.

  >>> layer = EinsumDense("abc,cd->abd",
  ...                     output_shape=(None, 64),
  ...                     bias_axes="d")
  >>> input_tensor = tf.keras.Input(shape=[32, 128])
  >>> output_tensor = layer(input_tensor)
  >>> output_tensor
  <... shape=(None, 32, 64) dtype=...>

  **Applying a dense layer to a sequence using ellipses**

  This example shows how to instantiate a layer that applies the same dense
  operation to every element in a sequence, but uses the ellipsis notation
  instead of specifying the batch and sequence dimensions.

  Because we are using ellipsis notation and have specified only one axis, the
  output_shape arg is a single value. When instantiated in this way, the layer
  can handle any number of sequence dimensions - including the case where no
  sequence dimension exists.

  >>> layer = EinsumDense("...x,xy->...y", output_shape=64, bias_axes="y")
  >>> input_tensor = tf.keras.Input(shape=[32, 128])
  >>> output_tensor = layer(input_tensor)
  >>> output_tensor
  <... shape=(None, 32, 64) dtype=...>
  """

  def __init__(self,
               equation,
               output_shape,
               activation=None,
               bias_axes=None,
               kernel_initializer="glorot_uniform",
               bias_initializer="zeros",
               kernel_regularizer=None,
               bias_regularizer=None,
               activity_regularizer=None,
               kernel_constraint=None,
               bias_constraint=None,
               **kwargs):
    super(EinsumDense, self).__init__(**kwargs)
    self.equation = equation
    if isinstance(output_shape, int):
      self.partial_output_shape = [output_shape]
    else:
      self.partial_output_shape = list(output_shape)
    self.bias_axes = bias_axes
    self.activation = activations.get(activation)
    self.kernel_initializer = initializers.get(kernel_initializer)
    self.bias_initializer = initializers.get(bias_initializer)
    self.kernel_regularizer = regularizers.get(kernel_regularizer)
    self.bias_regularizer = regularizers.get(bias_regularizer)
    self.kernel_constraint = constraints.get(kernel_constraint)
    self.bias_constraint = constraints.get(bias_constraint)

  def build(self, input_shape):
    input_shape = tf.TensorShape(input_shape)
    shape_data = _analyze_einsum_string(self.equation,
                                        self.bias_axes,
                                        input_shape,
                                        self.partial_output_shape)
    kernel_shape, bias_shape, self.full_output_shape = shape_data
    self.kernel = self.add_weight(
        "kernel",
        shape=kernel_shape,
        initializer=self.kernel_initializer,
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint,
        dtype=self.dtype,
        trainable=True)

    if bias_shape is not None:
      self.bias = self.add_weight(
          "bias",
          shape=bias_shape,
          initializer=self.bias_initializer,
          regularizer=self.bias_regularizer,
          constraint=self.bias_constraint,
          dtype=self.dtype,
          trainable=True)
    else:
      self.bias = None
    super(EinsumDense, self).build(input_shape)

  def compute_output_shape(self, _):
    return tf.TensorShape(self.full_output_shape)

  def get_config(self):
    config = {
        "output_shape":
            self.partial_output_shape,
        "equation":
            self.equation,
        "activation":
            activations.serialize(self.activation),
        "bias_axes":
            self.bias_axes,
        "kernel_initializer":
            initializers.serialize(self.kernel_initializer),
        "bias_initializer":
            initializers.serialize(self.bias_initializer),
        "kernel_regularizer":
            regularizers.serialize(self.kernel_regularizer),
        "bias_regularizer":
            regularizers.serialize(self.bias_regularizer),
        "activity_regularizer":
            regularizers.serialize(self.activity_regularizer),
        "kernel_constraint":
            constraints.serialize(self.kernel_constraint),
        "bias_constraint":
            constraints.serialize(self.bias_constraint),
    }
    base_config = super(EinsumDense, self).get_config()
    return dict(list(base_config.items()) + list(config.items()))

  def call(self, inputs):
    ret = tf.einsum(self.equation, inputs, self.kernel)
    if self.bias is not None:
      ret += self.bias
    if self.activation is not None:
      ret = self.activation(ret)
    return ret

Ancestors

  • Layer
  • tensorflow.python.module.module.Module
  • tensorflow.python.training.tracking.tracking.AutoTrackable
  • tensorflow.python.training.tracking.base.Trackable
  • LayerVersionSelector

Inherited members

class RandomFourierFeatures (output_dim, kernel_initializer='gaussian', scale=None, trainable=False, name=None, **kwargs)

Layer that projects its inputs into a random feature space.

This layer implements a mapping from input space to a space with output_dim dimensions, which approximates shift-invariant kernels. A kernel function K(x, y) is shift-invariant if K(x, y) == k(x - y) for some function k. Many popular Radial Basis Functions (RBF), including Gaussian and Laplacian kernels, are shift-invariant.

The implementation of this layer is based on the following paper: "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht.

The distribution from which the parameters of the random features map (layer) are sampled determines which shift-invariant kernel the layer approximates (see paper for more details). You can use the distribution of your choice. The layer supports out-of-the-box approximation sof the following two RBF kernels:

  • Gaussian: K(x, y) == exp(- square(x - y) / (2 * square(scale)))
  • Laplacian: K(x, y) = exp(-abs(x - y) / scale))

Note: Unlike what is described in the paper and unlike what is used in the Scikit-Learn implementation, the output of this layer does not apply the sqrt(2 / D) normalization factor.

Usage: Typically, this layer is used to "kernelize" linear models by applying a non-linear transformation (this layer) to the input features and then training a linear model on top of the transformed features. Depending on the loss function of the linear model, the composition of this layer and the linear model results to models that are equivalent (up to approximation) to kernel SVMs (for hinge loss), kernel logistic regression (for logistic loss), kernel linear regression (for squared loss), etc.

Examples:

A kernel multinomial logistic regression model with Gaussian kernel for MNIST:

model = keras.Sequential([
  keras.Input(shape=(784,)),
  RandomFourierFeatures(
      output_dim=4096,
      scale=10.,
      kernel_initializer='gaussian'),
  layers.Dense(units=10, activation='softmax'),
])
model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['categorical_accuracy']
)

A quasi-SVM classifier for MNIST:

model = keras.Sequential([
  keras.Input(shape=(784,)),
  RandomFourierFeatures(
      output_dim=4096,
      scale=10.,
      kernel_initializer='gaussian'),
  layers.Dense(units=10),
])
model.compile(
    optimizer='adam',
    loss='hinge',
    metrics=['categorical_accuracy']
)

To use another kernel, just replace the layer creation line with:

random_features_layer = RandomFourierFeatures(
    output_dim=500,
    kernel_initializer=<my_initializer>,
    scale=...,
    ...)

Args

output_dim
Positive integer, the dimension of the layer's output, i.e., the number of random features used to approximate the kernel.
kernel_initializer
Determines the distribution of the parameters of the random features map (and therefore the kernel approximated by the layer). It can be either a string identifier or a Keras Initializer instance. Currently only 'gaussian' and 'laplacian' are supported string identifiers (case insensitive). Note that the kernel matrix is not trainable.
scale
For Gaussian and Laplacian kernels, this corresponds to a scaling factor of the corresponding kernel approximated by the layer (see concrete definitions above). When provided, it should be a positive float. If None, a default value is used: if the kernel initializer is set to "gaussian", scale defaults to sqrt(input_dim / 2), otherwise, it defaults to 1.0. Both the approximation error of the kernel and the classification quality are sensitive to this parameter. If trainable is set to True, this parameter is learned end-to-end during training and the provided value serves as the initial value. Note: When features from this layer are fed to a linear model, by making scale trainable, the resulting optimization problem is no longer convex (even if the loss function used by the linear model is convex).
trainable
Whether the scaling parameter of the layer should be trainable. Defaults to False.
name
String, name to use for this layer.
Expand source code
class RandomFourierFeatures(base_layer.Layer):
  r"""Layer that projects its inputs into a random feature space.

  This layer implements a mapping from input space to a space with `output_dim`
  dimensions, which approximates shift-invariant kernels. A kernel function
  `K(x, y)` is shift-invariant if `K(x, y) == k(x - y)` for some function `k`.
  Many popular Radial Basis Functions (RBF), including Gaussian and
  Laplacian kernels, are shift-invariant.

  The implementation of this layer is based on the following paper:
  ["Random Features for Large-Scale Kernel Machines"](
    https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf)
  by Ali Rahimi and Ben Recht.

  The distribution from which the parameters of the random features map (layer)
  are sampled determines which shift-invariant kernel the layer approximates
  (see paper for more details). You can use the distribution of your
  choice. The layer supports out-of-the-box
  approximation sof the following two RBF kernels:

  - Gaussian: `K(x, y) == exp(- square(x - y) / (2 * square(scale)))`
  - Laplacian: `K(x, y) = exp(-abs(x - y) / scale))`

  **Note:** Unlike what is described in the paper and unlike what is used in
  the Scikit-Learn implementation, the output of this layer does not apply
  the `sqrt(2 / D)` normalization factor.

  **Usage:** Typically, this layer is used to "kernelize" linear models by
  applying a non-linear transformation (this layer) to the input features and
  then training a linear model on top of the transformed features. Depending on
  the loss function of the linear model, the composition of this layer and the
  linear model results to models that are equivalent (up to approximation) to
  kernel SVMs (for hinge loss), kernel logistic regression (for logistic loss),
  kernel linear regression (for squared loss), etc.

  Examples:

  A kernel multinomial logistic regression model with Gaussian kernel for MNIST:

  ```python
  model = keras.Sequential([
    keras.Input(shape=(784,)),
    RandomFourierFeatures(
        output_dim=4096,
        scale=10.,
        kernel_initializer='gaussian'),
    layers.Dense(units=10, activation='softmax'),
  ])
  model.compile(
      optimizer='adam',
      loss='categorical_crossentropy',
      metrics=['categorical_accuracy']
  )
  ```

  A quasi-SVM classifier for MNIST:

  ```python
  model = keras.Sequential([
    keras.Input(shape=(784,)),
    RandomFourierFeatures(
        output_dim=4096,
        scale=10.,
        kernel_initializer='gaussian'),
    layers.Dense(units=10),
  ])
  model.compile(
      optimizer='adam',
      loss='hinge',
      metrics=['categorical_accuracy']
  )
  ```

  To use another kernel, just replace the layer creation line with:

  ```python
  random_features_layer = RandomFourierFeatures(
      output_dim=500,
      kernel_initializer=<my_initializer>,
      scale=...,
      ...)
  ```

  Args:
    output_dim: Positive integer, the dimension of the layer's output, i.e., the
      number of random features used to approximate the kernel.
    kernel_initializer: Determines the distribution of the parameters of the
      random features map (and therefore the kernel approximated by the layer).
      It can be either a string identifier or a Keras `Initializer` instance.
      Currently only 'gaussian' and 'laplacian' are supported string
      identifiers (case insensitive). Note that the kernel matrix is not
      trainable.
    scale: For Gaussian and Laplacian kernels, this corresponds to a scaling
      factor of the corresponding kernel approximated by the layer (see concrete
      definitions above). When provided, it should be a positive float. If None,
      a default value is used: if the kernel initializer is set to "gaussian",
      `scale` defaults to `sqrt(input_dim / 2)`, otherwise, it defaults to 1.0.
      Both the approximation error of the kernel and the classification quality
      are sensitive to this parameter. If `trainable` is set to `True`, this
      parameter is learned end-to-end during training and the provided value
      serves as the initial value.
      **Note:** When features from this layer are fed to a linear model,
        by making `scale` trainable, the resulting optimization problem is
        no longer convex (even if the loss function used by the linear model
        is convex).
    trainable: Whether the scaling parameter of the layer should be trainable.
      Defaults to `False`.
    name: String, name to use for this layer.
  """

  def __init__(self,
               output_dim,
               kernel_initializer='gaussian',
               scale=None,
               trainable=False,
               name=None,
               **kwargs):
    if output_dim <= 0:
      raise ValueError(
          '`output_dim` should be a positive integer. Given: {}.'.format(
              output_dim))
    if isinstance(kernel_initializer, str):
      if kernel_initializer.lower() not in _SUPPORTED_RBF_KERNEL_TYPES:
        raise ValueError(
            'Unsupported kernel type: \'{}\'. Supported kernel types: {}.'
            .format(kernel_initializer, _SUPPORTED_RBF_KERNEL_TYPES))
    if scale is not None and scale <= 0.0:
      raise ValueError('When provided, `scale` should be a positive float. '
                       'Given: {}.'.format(scale))
    super(RandomFourierFeatures, self).__init__(
        trainable=trainable, name=name, **kwargs)
    self.output_dim = output_dim
    self.kernel_initializer = kernel_initializer
    self.scale = scale

  def build(self, input_shape):
    input_shape = tf.TensorShape(input_shape)
    # TODO(pmol): Allow higher dimension inputs. Currently the input is expected
    # to have shape [batch_size, dimension].
    if input_shape.rank != 2:
      raise ValueError(
          'The rank of the input tensor should be 2. Got {} instead.'.format(
              input_shape.ndims))
    if input_shape.dims[1].value is None:
      raise ValueError(
          'The last dimension of the inputs to `RandomFourierFeatures` '
          'should be defined. Found `None`.')
    self.input_spec = input_spec.InputSpec(
        ndim=2, axes={1: input_shape.dims[1].value})
    input_dim = input_shape.dims[1].value

    kernel_initializer = _get_random_features_initializer(
        self.kernel_initializer, shape=(input_dim, self.output_dim))

    self.unscaled_kernel = self.add_weight(
        name='unscaled_kernel',
        shape=(input_dim, self.output_dim),
        dtype=tf.float32,
        initializer=kernel_initializer,
        trainable=False)

    self.bias = self.add_weight(
        name='bias',
        shape=(self.output_dim,),
        dtype=tf.float32,
        initializer=tf.compat.v1.random_uniform_initializer(
            minval=0.0, maxval=2 * np.pi, dtype=tf.float32),
        trainable=False)

    if self.scale is None:
      self.scale = _get_default_scale(self.kernel_initializer, input_dim)
    self.kernel_scale = self.add_weight(
        name='kernel_scale',
        shape=(1,),
        dtype=tf.float32,
        initializer=tf.compat.v1.constant_initializer(self.scale),
        trainable=True,
        constraint='NonNeg')
    super(RandomFourierFeatures, self).build(input_shape)

  def call(self, inputs):
    inputs = tf.convert_to_tensor(inputs, dtype=self.dtype)
    inputs = tf.cast(inputs, tf.float32)
    kernel = (1.0 / self.kernel_scale) * self.unscaled_kernel
    outputs = tf.raw_ops.MatMul(a=inputs, b=kernel)
    outputs = tf.nn.bias_add(outputs, self.bias)
    return tf.cos(outputs)

  def compute_output_shape(self, input_shape):
    input_shape = tf.TensorShape(input_shape)
    input_shape = input_shape.with_rank(2)
    if input_shape.dims[-1].value is None:
      raise ValueError(
          'The innermost dimension of input shape must be defined. Given: %s' %
          input_shape)
    return input_shape[:-1].concatenate(self.output_dim)

  def get_config(self):
    kernel_initializer = self.kernel_initializer
    if not isinstance(kernel_initializer, str):
      kernel_initializer = initializers.serialize(kernel_initializer)
    config = {
        'output_dim': self.output_dim,
        'kernel_initializer': kernel_initializer,
        'scale': self.scale,
    }
    base_config = super(RandomFourierFeatures, self).get_config()
    return dict(list(base_config.items()) + list(config.items()))

Ancestors

  • Layer
  • tensorflow.python.module.module.Module
  • tensorflow.python.training.tracking.tracking.AutoTrackable
  • tensorflow.python.training.tracking.base.Trackable
  • LayerVersionSelector

Inherited members