Add new feature: add LSTMCell and test

Author: Wanglongzhi2001

src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs

@@ -1,7 +1,35 @@
-namespace Tensorflow.Keras.ArgsDefinition.Rnn
+using Newtonsoft.Json;
+using static Tensorflow.Binding; 
+
+namespace Tensorflow.Keras.ArgsDefinition.Rnn
 {
     // TODO: complete the implementation
-    public class LSTMCellArgs : LayerArgs
+    public class LSTMCellArgs : AutoSerializeLayerArgs
     {
+        [JsonProperty("units")]
+        public int Units { get; set; }
+        // TODO(Rinne): lack of initialized value of Activation. Merging keras
+        // into tf.net could resolve it.
+        [JsonProperty("activation")]
+        public Activation Activation { get; set; }
+        [JsonProperty("recurrent_activation")]
+        public Activation RecurrentActivation { get; set; }
+        [JsonProperty("use_bias")]
+        public bool UseBias { get; set; } = true;
+        [JsonProperty("dropout")]
+        public float Dropout { get; set; } = .0f;
+        [JsonProperty("recurrent_dropout")]
+        public float RecurrentDropout { get; set; } = .0f;
+        [JsonProperty("kernel_initializer")]
+        public IInitializer KernelInitializer { get; set; }
+        [JsonProperty("recurrent_initializer")]
+        public IInitializer RecurrentInitializer { get; set; }
+        [JsonProperty("bias_initializer")]
+        public IInitializer BiasInitializer { get; set; }
+        [JsonProperty("unit_forget_bias")]
+        public bool UnitForgetBias { get; set; } = true;
+        [JsonProperty("implementation")]
+        public int Implementation { get; set; } = 2;
+
     }
 }

src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs

@@ -1,7 +1,4 @@
 using Newtonsoft.Json;
-using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace Tensorflow.Keras.ArgsDefinition.Rnn
 {
@@ -25,5 +22,6 @@ public class SimpleRNNCellArgs: AutoSerializeLayerArgs
         public IInitializer RecurrentInitializer { get; set; }
         [JsonProperty("bias_initializer")]
         public IInitializer BiasInitializer { get; set; }
+
     }
 }

src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs

@@ -160,6 +160,18 @@ public ILayer LayerNormalization(Axis? axis,
         public ILayer Normalization(Shape? input_shape = null, int? axis = -1, float? mean = null, float? variance = null, bool invert = false);
         public ILayer LeakyReLU(float alpha = 0.3f);
 
+        public IRnnCell LSTMCell(int uints,
+            string activation = "tanh",
+            string recurrent_activation = "sigmoid",
+            bool use_bias = true,
+            string kernel_initializer = "glorot_uniform",
+            string recurrent_initializer = "orthogonal",
+            string bias_initializer = "zeros",
+            bool unit_forget_bias = true,
+            float dropout = 0f,
+            float recurrent_dropout = 0f,
+            int implementation = 2);
+
         public ILayer LSTM(int units,
             Activation activation = null,
             Activation recurrent_activation = null,

src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs

@@ -58,8 +58,7 @@ private Tensor _generate_init_val(Shape shape, TF_DataType dtype)
 
         if (num_rows < num_cols)
         {
-            // q = tf.linalg.matrix_transpose(q);
-            throw new NotImplementedException("");
+            q = array_ops.matrix_transpose(q);
         }
 
         return _gain * tf.reshape(q, shape);

src/TensorFlowNET.Core/Operations/array_ops.cs

@@ -971,6 +971,49 @@ public static Tensor transpose(Tensor a, Tensor perm, string name = "transpose",
             });
         }
 
+        /// <summary>
+        /// Transposes last two dimensions of tensor `a`.
+        /// For example:
+        /// <code> python
+        ///   x = tf.constant([[1, 2, 3], [4, 5, 6]])
+        ///   tf.matrix_transpose(x) # [[1, 4],
+        ///                         #  [2, 5],
+        ///                         #  [3, 6]]
+        /// </code>
+        /// Matrix with two batch dimensions.
+        /// x.shape is [1, 2, 3, 4]
+        /// tf.linalg.matrix_transpose(x) is shape [1, 2, 4, 3]
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="name"></param>
+        /// <param name="conjugate"></param>
+        /// <returns></returns>
+        /// <exception cref="ValueError"></exception>
+        public static Tensor matrix_transpose(Tensor a, string name = "matrix_transpose", bool conjugate = false)
+        {
+            return tf_with(ops.name_scope(name, "transpose", new { a }), scope =>
+            {
+                var a_shape = a.shape;
+                var ndims = a.shape.ndim;
+                Axis perm;
+                if(ndims != 0)
+                {
+                    if (ndims < 2)
+                    {
+                        throw new ValueError("Argument `a` should be a (batch) matrix with rank " +
+                            $">= 2.  Received `a` = {a} with shape: {a_shape}");
+                    }
+                    perm = new Axis(Enumerable.Range(0, ndims - 2).Concat(new int[] { ndims - 1, ndims - 2 }).ToArray());
+                }
+                else
+                {
+                    var a_rank = a.rank;
+                    perm = new Axis(Enumerable.Range(0, a_rank - 2).Concat(new int[] { a_rank - 1, a_rank - 2 }).ToArray());
+                }
+                return transpose(a, perm:perm, conjugate:conjugate);
+            });
+        }
+
         public static Tensor[] split(Tensor value, Tensor size_splits, int axis, int num = -1,
             string name = "split")
         {

src/TensorFlowNET.Keras/Layers/LayersApi.cs

@@ -702,6 +702,7 @@ public IRnnCell SimpleRNNCell(
                 UseBias = use_bias,
                 KernelInitializer = GetInitializerByName(kernel_initializer),
                 RecurrentInitializer = GetInitializerByName(recurrent_initializer),
+                BiasInitializer = GetInitializerByName(bias_initializer),
                 Dropout = dropout,
                 RecurrentDropout = recurrent_dropout
             });
@@ -786,6 +787,33 @@ public ILayer RNN(
                 TimeMajor = time_major
             });
 
+
+        public IRnnCell LSTMCell(int uints,
+            string activation = "tanh",
+            string recurrent_activation = "sigmoid",
+            bool use_bias = true,
+            string kernel_initializer = "glorot_uniform",
+            string recurrent_initializer = "orthogonal", // TODO(Wanglongzhi2001),glorot_uniform has not been developed.
+            string bias_initializer = "zeros",
+            bool unit_forget_bias = true,
+            float dropout = 0f,
+            float recurrent_dropout = 0f,
+            int implementation = 2)
+            => new LSTMCell(new LSTMCellArgs
+            {
+                Units = uints,
+                Activation = keras.activations.GetActivationFromName(activation),
+                RecurrentActivation = keras.activations.GetActivationFromName(recurrent_activation),
+                UseBias = use_bias,
+                KernelInitializer = GetInitializerByName(kernel_initializer),
+                RecurrentInitializer = GetInitializerByName(recurrent_initializer),
+                BiasInitializer = GetInitializerByName(bias_initializer),
+                UnitForgetBias = unit_forget_bias,
+                Dropout = dropout,
+                RecurrentDropout = recurrent_dropout,
+                Implementation = implementation
+            });
+
         /// <summary>
         /// Long Short-Term Memory layer - Hochreiter 1997.
         /// </summary>

src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs

@@ -41,7 +41,7 @@ public void reset_recurrent_dropout_mask()
 
         }
 
-        public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1)
+        public Tensors? get_dropout_mask_for_cell(Tensors input, bool training, int count = 1)
         {
             if (dropout == 0f)
                 return null;
@@ -53,7 +53,7 @@ public void reset_recurrent_dropout_mask()
         }
 
         // Get the recurrent dropout mask for RNN cell.
-        public Tensors? get_recurrent_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1)
+        public Tensors? get_recurrent_dropout_mask_for_cell(Tensors input, bool training, int count = 1)
         {
             if (dropout == 0f)
                 return null;