layers -> xlayers

lib/layers/__init__.py

@@ -1,5 +0,0 @@
-from .drop import DropBlock2d, DropPath
-from .mlp import MLP
-from .weight_init import trunc_normal_
-
-from .positional_embedding import PositionalEncoder

lib/spaces/__init__.py

@@ -7,6 +7,8 @@
 from .basic_space import Categorical
 from .basic_space import Continuous
 from .basic_space import Integer
+from .basic_space import Space
+from .basic_space import VirtualNode
 from .basic_op import has_categorical
 from .basic_op import has_continuous
 from .basic_op import get_min

lib/spaces/basic_space.py

@@ -7,6 +7,7 @@ import math
 import copy
 import random
 import numpy as np
+from collections import OrderedDict
 
 from typing import Optional
 
@@ -44,6 +45,32 @@ class Space(metaclass=abc.ABCMeta):
         return copy.deepcopy(self)
 
 
+class VirtualNode(Space):
+    """For a nested search space, we represent it as a tree structure.
+
+    For example,
+    """
+
+    def __init__(self, id=None, value=None):
+        self._id = id
+        self._value = value
+        self._attributes = OrderedDict()
+
+    def has(self, x):
+        for key, value in self._attributes.items():
+            if isinstance(value, Space) and value.has(x):
+                return True
+        return False
+
+    def __repr__(self):
+        strs = [self.__class__.__name__ + "("]
+        indent = " " * 4
+        for key, value in self._attributes.items():
+            strs.append(indent + strs(value))
+        strs.append(")")
+        return "\n".join(strs)
+
+
 class Categorical(Space):
     """A space contains the categorical values.
     It can be a nested space, which means that the candidate in this space can also be a search space.

lib/trade_models/transformers.py

@@ -12,7 +12,7 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
-import layers as xlayers
+import xlayers
 
 
 DEFAULT_NET_CONFIG = dict(

lib/xlayers/__init__.py

@@ -0,0 +1,11 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+# This file is expected to be self-contained, expect
+# for importing from spaces to include search space.
+#####################################################
+from .drop import DropBlock2d, DropPath
+from .mlp import MLP
+from .weight_init import trunc_normal_
+
+from .positional_embedding import PositionalEncoder

lib/xlayers/super_mlp.py

@@ -1,16 +1,15 @@
 #####################################################
 # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.03 #
 #####################################################
+import torch
 import torch.nn as nn
-from torch.nn.parameter import Parameter
-from torch import Tensor
 
 import math
 from typing import Optional, Union
 
 import spaces
-from layers.super_module import SuperModule
-from layers.super_module import SuperRunType
+from .super_module import SuperModule
+from .super_module import SuperRunMode
 
 IntSpaceType = Union[int, spaces.Integer, spaces.Categorical]
 BoolSpaceType = Union[bool, spaces.Categorical]
@@ -32,11 +31,11 @@ class SuperLinear(SuperModule):
         self._out_features = out_features
         self._bias = bias
 
-        self._super_weight = Parameter(
+        self._super_weight = torch.nn.Parameter(
             torch.Tensor(self.out_features, self.in_features)
         )
-        if bias:
-            self._super_bias = Parameter(torch.Tensor(self.out_features))
+        if self.bias:
+            self._super_bias = torch.nn.Parameter(torch.Tensor(self.out_features))
         else:
             self.register_parameter("_super_bias", None)
         self.reset_parameters()
@@ -53,6 +52,9 @@ class SuperLinear(SuperModule):
     def bias(self):
         return spaces.has_categorical(self._bias, True)
 
+    def abstract_search_space(self):
+        print('-')
+
     def reset_parameters(self) -> None:
         nn.init.kaiming_uniform_(self._super_weight, a=math.sqrt(5))
         if self.bias:
@@ -60,7 +62,7 @@ class SuperLinear(SuperModule):
             bound = 1 / math.sqrt(fan_in)
             nn.init.uniform_(self._super_bias, -bound, bound)
 
-    def forward_raw(self, input: Tensor) -> Tensor:
+    def forward_raw(self, input: torch.Tensor) -> torch.Tensor:
         return F.linear(input, self._super_weight, self._super_bias)
 
     def extra_repr(self) -> str:

lib/xlayers/super_module.py

@@ -14,12 +14,12 @@ class SuperRunMode(Enum):
     Default = "fullmodel"
 
 
-class SuperModule(abc.ABCMeta, nn.Module):
+class SuperModule(abc.ABC, nn.Module):
     """This class equips the nn.Module class with the ability to apply AutoDL."""
 
     def __init__(self):
         super(SuperModule, self).__init__()
-        self._super_run_type = SuperRunMode.default
+        self._super_run_type = SuperRunMode.Default
 
     @abc.abstractmethod
     def abstract_search_space(self):