ihatebaselines commited on
Commit
53126e3
Β·
verified Β·
1 Parent(s): 04adc54

Upload config.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.py +47 -0
config.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ sx.Config β€” Model architecture configuration for SocrateX.
3
+ Contains only Transformer + SocratePool parameters.
4
+ """
5
+
6
+ class Config:
7
+ """
8
+ Unified architecture config for SOCRATE.
9
+ Covers the Transformer layers and SocratePool.
10
+
11
+ Pass to sx.init() to build a fully custom model:
12
+ config = sx.Config(d_model=256, nhead=4, num_layers=3)
13
+ model = sx.init(config=config, tokenizer=tokenizer)
14
+ """
15
+
16
+ def __init__(
17
+ self,
18
+ # ─── Transformer ────────────────────────────────────────────
19
+ d_model: int = 640,
20
+ max_len: int = 512,
21
+ nhead: int = 10,
22
+ dim_feedforward: int = 2560,
23
+ activation: str = "gelu",
24
+ norm_first: bool = True,
25
+ num_layers: int = 12,
26
+
27
+ # ─── SocratePool ────────────────────────────────────────────
28
+ pool_height: int = 4, # target_height in nn.AdaptiveMaxPool2d((pool_height, None))
29
+ ):
30
+ self.d_model = d_model
31
+ self.max_len = max_len
32
+ self.nhead = nhead
33
+ self.dim_feedforward = dim_feedforward
34
+ self.activation = activation
35
+ self.norm_first = norm_first
36
+ self.num_layers = num_layers
37
+ self.pool_height = pool_height
38
+
39
+ def __repr__(self):
40
+ return (
41
+ f"sx.Config(\n"
42
+ f" d_model={self.d_model}, nhead={self.nhead}, num_layers={self.num_layers},\n"
43
+ f" dim_feedforward={self.dim_feedforward}, activation='{self.activation}',\n"
44
+ f" norm_first={self.norm_first}, max_len={self.max_len},\n"
45
+ f" pool_height={self.pool_height} # AdaptiveMaxPool2d((pool_height, None))\n"
46
+ f")"
47
+ )