Update super-activation layers

2021-05-12 13:54:06 +08:00
parent 0dbbc286c9
commit 4da19d6efe
7 changed files with 349 additions and 127 deletions
--- a/lib/xlayers/super_core.py
+++ b/lib/xlayers/super_core.py
@@ -28,9 +28,13 @@ from .super_transformer import SuperTransformerEncoderLayer
 from .super_activations import SuperReLU
 from .super_activations import SuperLeakyReLU
 from .super_activations import SuperTanh
+from .super_activations import SuperGELU
+from .super_activations import SuperSigmoid

 super_name2activation = {
    "relu": SuperReLU,
+    "sigmoid": SuperSigmoid,
+    "gelu": SuperGELU,
    "leaky_relu": SuperLeakyReLU,
    "tanh": SuperTanh,
 }