[Scipy-svn] r2171 - trunk/Lib/sandbox/ann

scipy-svn at scipy.org scipy-svn at scipy.org
Sun Aug 20 20:02:49 CDT 2006


Author: fred.mailhot
Date: 2006-08-20 20:02:46 -0500 (Sun, 20 Aug 2006)
New Revision: 2171

Modified:
   trunk/Lib/sandbox/ann/mlp.py
   trunk/Lib/sandbox/ann/rbf.py
   trunk/Lib/sandbox/ann/srn.py
Log:
Updates/refactorings for mlp and srn, addition of rbf.


Modified: trunk/Lib/sandbox/ann/mlp.py
===================================================================
--- trunk/Lib/sandbox/ann/mlp.py	2006-08-20 06:37:03 UTC (rev 2170)
+++ trunk/Lib/sandbox/ann/mlp.py	2006-08-21 01:02:46 UTC (rev 2171)
@@ -17,10 +17,10 @@
         zero-mean Gaussian w/ variance is scaled by fan-in.
         Input:
             ni  - <int> # of inputs
-            nh  - <int> # of hidden & context units
+            nh  - <int> # of hidden units
             no  - <int> # of outputs
             f   - <str> output activation fxn
-            w   - <array dtype=Float> weight vector
+            w   - <array of float> vector of initial weights
         """
         if f not in self._outfxns:
             print "Undefined activation fxn. Using linear"

Modified: trunk/Lib/sandbox/ann/rbf.py
===================================================================
--- trunk/Lib/sandbox/ann/rbf.py	2006-08-20 06:37:03 UTC (rev 2170)
+++ trunk/Lib/sandbox/ann/rbf.py	2006-08-21 01:02:46 UTC (rev 2171)
@@ -1,39 +1,137 @@
 # rbf2.py
 # tilde
-# 2006/08/18
-# - new attempt at RBF net to get my ideas straight...deadline is fast approaching!
+# 2006/08/20
 
-from numpy import *
+import numpy as N
+from scipy.optimize import leastsq
 
 class rbf:
+    """Class to define/train/test a radial basis function network
+    """
 
     _type = 'rbf'
-    
-    def __init__(nin,nhid,nout,trndata):
-        # set easy params
-        self.nin = nin
-        self.nhid = nhid
-        self.nout = nout
-        # choose subset (1/5?) of training data for basis fxn centers and
-        self.centers = []
-        for i in trndata:
-            if random.random < 0.2:
-                self.centers.append(i)
-        # set common variance proportional to max dist between centers
+    _outfxns = ('linear','logistic','softmax')
+
+
+    def __init__(self,ni,no,f='linear'):
+        """ Set up instance of RBF net. N.B. RBF centers and variance are selected at training time 
+        Input:
+            ni  - <int> # of inputs
+            no  - <int> # of outputs
+            f   - <str> output activation fxn
+        """
+        
+        self.ni = ni
+        self.no = no
+        self.outfxn = f
+
+    def unpack(self):
+        """ Decompose 1-d vector of weights w into appropriate weight
+        matrices (self.{w/b}) and reinsert them into net
+        """
+        self.w = N.array(self.wp)[:self.centers.shape[0]*self.no].reshape(self.centers.shape[0],self.no)
+        self.b = N.array(self.wp)[(self.centers.shape[0]*self.no):].reshape(1,self.no)
+
+    def pack(self):
+        """ Compile weight matrices w,b from net into a
+        single vector, suitable for optimization routines.
+        """
+        self.wp = N.hstack([self.w.reshape(N.size(self.w)),
+                            self.b.reshape(N.size(self.b))])
+
+    def fwd_all(self,X,w=None):
+        """ Propagate values forward through the net.
+        Inputs:
+                inputs      - vector of input values
+                w           - packed array of weights
+        Returns:
+                array of outputs for all input patterns
+        """
+        if w is not None:
+            self.wp = w
+        self.unpack()
+        # compute hidden unit values
+        z = N.zeros((len(X),self.centers.shape[0]))
+        for i in range(len(X)):
+             z[i] = N.exp((-1.0/(2*self.variance))*(N.sum((X[i]-self.centers)**2,axis=1)))
+        # compute net outputs
+        o = N.dot(z,self.w) + N.dot(N.ones((len(z),1)),self.b)
+        # compute final output activations
+        if self.outfxn == 'linear':
+            y = o
+        elif self.outfxn == 'logistic':     # TODO: check for overflow here...
+            y = 1/(1+N.exp(-o))
+        elif self.outfxn == 'softmax':      # TODO: and here...
+            tmp = N.exp(o)
+            y = tmp/(N.sum(temp,1)*N.ones((1,self.no)))
+
+        return N.array(y)
+
+
+    def err_fxn(self,w,X,Y):
+        """ Return vector of squared-errors for the leastsq optimizer
+        """
+        O = self.fwd_all(X,w)
+        return N.sum(N.array(O-Y)**2,axis=1)
+
+    def train(self,X,Y):
+        """ Train RBF network:
+            (i) select fixed centers randomly from input data (10%)
+            (ii) set fixed variance from max dist between centers
+            (iii) learn output weights using scipy's leastsq optimizer
+        """
+        # set centers
+        self.centers = N.zeros((len(X)/10,X.shape[1]))
+        for i in range(len(X)):
+            if i%10 == 0:
+                self.centers[i/10] = X[i]
+        # set variance
         d_max = 0.0
         for i in self.centers:
             for j in self.centers:
-                tmp = sqrt((i-j)**2)
+                tmp = N.sum(N.sqrt((i-j)**2))
                 if tmp > d_max:
                     d_max = tmp
-        self.variance = d_max/2.0*size(trndata)
-        
-    
-    def fwd(self,inputs):
-        """ Propagate values forward through the net.
-        Inputs:
-                inputs      - vector of input values
+        self.variance = d_max/(2.0*len(X))
+        # train weights
+        self.nw = self.centers.shape[0]*self.no
+        self.w = N.random.randn(self.centers.shape[0],self.no)/N.sqrt(self.centers.shape[0]+1)
+        self.b = N.random.randn(1,self.no)/N.sqrt(self.centers.shape[0]+1)
+        self.pack()
+        self.wp = leastsq(self.err_fxn,self.wp,args=(X,Y))[0]
+
+    def test_all(self,X,Y):
+        """ Test network on an array (size>1) of patterns
+        Input:
+            x   - array of input data
+            t   - array of targets
+        Returns:
+            sum-squared-error over all data
         """
-        z = exp((-1.0/(2*self.variance))*
-        o = dot(z,self.w) + dot(ones((len(z),1)),self.b)
+        return N.sum(self.err_fxn(self.wp,X,Y))
 
+def main():
+    """ Build/train/test RBF net
+    """
+    from scipy.io import read_array
+    print "\nCreating RBF net"
+    net = rbf(12,2)
+    print "\nLoading training and test sets...",
+    X_trn = read_array('data/oil-trn.dat',columns=(0,(1,12)),lines=(3,-1))
+    Y_trn = read_array('data/oil-trn.dat',columns=(12,-1),lines=(3,-1))
+    X_tst = read_array('data/oil-tst.dat',columns=(0,(1,12)),lines=(3,-1))
+    Y_tst = read_array('data/oil-tst.dat',columns=(12,-1),lines=(3,-1))
+    print "done."
+    #print "\nInitial SSE:\n"
+    #print "\ttraining set: ",net.test_all(X_trn,Y_trn)
+    #print "\ttesting set: ",net.test_all(X_tst,Y_tst),"\n"
+    print "Training...",
+    net.train(X_trn,Y_trn)
+    print "done."
+    print "\nFinal SSE:\n"
+    print "\ttraining set: ",net.test_all(X_trn,Y_trn)
+    print "\ttesting set: ",net.test_all(X_tst,Y_tst),"\n"
+
+
+if __name__ == '__main__':
+    main()

Modified: trunk/Lib/sandbox/ann/srn.py
===================================================================
--- trunk/Lib/sandbox/ann/srn.py	2006-08-20 06:37:03 UTC (rev 2170)
+++ trunk/Lib/sandbox/ann/srn.py	2006-08-21 01:02:46 UTC (rev 2171)
@@ -85,13 +85,22 @@
         if w is not None:
             self.wp = w
         self.unpack()
+        
+        ### NEW ATTEMPT ###
+        z = N.array(N.ones(self.nh)*0.5)    # init to 0.5, it will be updated on-the-fly
+        o = N.zeros((x.shape[0],self.no))   # this will hold the non-squashed outputs
+        for i in range(len(x)):
+            z = N.tanh(N.dot(x[i],self.w1) + N.dot(z,self.wc) + self.b1)
+            o[i] = (N.dot(z,self.w2) + self.b2)[0]
+            
         # compute vector of context values for current weight matrix
-        c = N.tanh(N.dot(x,self.w1) + N.dot(N.ones((len(x),1)),self.b1))
-        c = N.vstack([c[1:],c[0]])
+        #c = N.tanh(N.dot(x,self.w1) + N.dot(N.ones((len(x),1)),self.b1))
+        #c = N.vstack([c[1:],c[0]])
         # compute vector of hidden unit values
-        z = N.tanh(N.dot(x,self.w1) + N.dot(c,self.wc) + N.dot(N.ones((len(x),1)),self.b1))
+        #z = N.tanh(N.dot(x,self.w1) + N.dot(c,self.wc) + N.dot(N.ones((len(x),1)),self.b1))
         # compute vector of net outputs
-        o = N.dot(z,self.w2) + N.dot(N.ones((len(z),1)),self.b2)
+        #o = N.dot(z,self.w2) + N.dot(N.ones((len(z),1)),self.b2)
+        
         # compute final output activations
         if self.outfxn == 'linear':
             y = o
@@ -101,7 +110,7 @@
             tmp = N.exp(o)
             y = tmp/(N.sum(temp,1)*N.ones((1,self.no)))
             
-        return N.array(y)
+        return y
         
     def errfxn(self,w,x,t):
         """ Return vector of squared-errors for the leastsq optimizer



More information about the Scipy-svn mailing list