Creating directory /dataZaGE6ut7Es35haM/meta/goplayer failed
Writing /dataZaGE6ut7Es35haM/meta/goplayer/mcffnngmmdist.meta failed
Unable to save metadata file. Hint: disk full; file permissions; safe_mode setting.

# Differences

This shows you the differences between two versions of the page.

goplayer:mcffnngmmdist [2011/12/16 12:29] (current)
Line 1: Line 1:
+====== Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution ======

+The Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution (MCMPGMMDist).
+
+A Bayesian network node conditional probability distribution is represented by a series of Gaussian mixture models.  A Gaussian mixture model is defined for every combination of states the parent nodes of the given node can take on.  The parameters for each Gaussian mixture model is determined from a multilayer perceptron.  As the mixture models for each set of parent states are not independent of each other, the neural network serves to also define and maintain the relationship of the mixture models of the conditional probability distribution.
+
+Ultimately, the states of the parents of the given node are fed into the neural network.  There is an input node for each parent node.  The output nodes of the neural network then provide the parameters of the Gaussian mixture model for the given state set of the parent nodes.  These parameters are the weight, mean, and standard deviation of each normal distribution in the Gaussian mixture model.  This set of parameters define the Gaussian mixture model for the particular state set of the parent nodes of the given node.
+
+The advantage of this methodology is that any continuous probability distribution can be approximated by the combination of the multilayer perceptron and the Gaussian mixture model.  Furthermore, the relationship between the distributions for different parent state sets is maintained by the neural network.  By this methodology recognizes this relationship, we are able to train this distribution more easily and represent and represent the nodes conditional probability distribution more compactly.
+
+x <m>right~P(X=1)</m>
+
+x[1, {'y':2, 'z':3}] <m>right~P(X=1 | Y=2, Z=3)</m>
+
+
+<code python MCMPGMMDistTest.py>
+'''
+Created on Nov 19, 2009
+
+@author: Stephen
+'''
+import unittest
+from MCMPGMMDist import MCMPGMMDist
+from numpy import sqrt;
+
+
+class Test(unittest.TestCase):
+ input1 = [2, 3, 7];
+ input2 = [2, 3, 7, 0];
+ dist = MCMPGMMDist(['a', 'b'], 0, 10);
+ output1 = dist.net(input1);
+ output2 = dist.net(input2);
+ self.assertEqual(output1, output2);
+ self.assertEquals(dist.condVars, ['a', 'b', 'c']);
+
+ def testRemoveConditional(self):
+ input1 = [2, 3, 0, 7];
+ input2 = [2, 3, 7];
+ dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
+ output1 = dist.net(input1);
+ dist.removeConditional('b');
+ output2 = dist.net(input2);
+ self.assertEqual(output1, output2);
+ self.assertEquals(dist.condVars, ['a', 'c']);
+
+ def testNumWeights(self):
+ dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
+ self.assertEquals(dist.numWeights(), 5 * MCMPGMMDist.NUM_HIDDEN_VARS + (MCMPGMMDist.NUM_HIDDEN_VARS + 1) * MCMPGMMDist.NUM_NORMALS * 3);
+
+ def testModifyWeightWithinBounds(self):
+ # Setup test
+ dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
+ wStartValue = dist.net.weights;
+ bound = 2.38 / sqrt(5);
+
+ # Modify weights a little and test
+ dist.modifyWeightWithinBounds(1, 0.1);
+ self.assertAlmostEquals(dist.net.weights - wStartValue, (bound - wStartValue) * .1);
+
+ # Test extremes of modifier
+ dist.modifyWeightWithinBounds(1, 1);
+ self.assertAlmostEquals(dist.net.weights, bound);
+ dist.modifyWeightWithinBounds(1, -1);
+ self.assertAlmostEquals(dist.net.weights, -bound);
+ dist.modifyWeightWithinBounds(1, 0.5);
+ self.assertAlmostEquals(dist.net.weights, 0);
+
+ # Test exception thrown when going out of bounds
+ self.assertRaises(ValueError, dist.modifyWeightWithinBounds, 1, 1.1);
+
+ def testModifyWeight(self):
+ dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
+ wStartValue = dist.net.weights;
+ dist.modifyWeight(1, 4.4);
+ self.assertAlmostEquals(dist.net.weights - wStartValue, wStartValue * 4.4);
+
+
+if __name__ == "__main__":
+ #import sys;sys.argv = ['', 'Test.testAddConditional', 'Test.testRemoveConditional', 'Test.testNumWeights', 'Test.testModifyWeightWithinBounds', 'Test.testModifyWeight']
+ unittest.main()
+</code>
+
+
+<code python MCMPGMMDist.py>
+'''
+Created on Aug 1, 2009
+
+@author: Stephen
+'''
+
+from ffnet import ffnet, mlgraph, savenet, loadnet, exportnet;
+from numpy.random import normal, random_sample;
+from numpy import array, digitize, zeros, where, insert, delete, sqrt;
+from copy import deepcopy
+
+
+class MCMPGMMDist(object):
+ '''
+ Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution
+ '''
+
+ NUM_HIDDEN_VARS = 10;
+ NUM_NORMALS = 10;
+
+ def __init__(self, condVars, normMin, normMax):
+ '''
+ Constructor
+ condVars - List of identifiers for conditional variables for this conditional distribution
+ normMin - Minimum possible value any of the conditional variables can be
+ normMax - Maximum possible value any of the conditional variables can be
+ '''
+ if(condVars.__class__ != [].__class__):
+ raise TypeError("concVars must be a list");
+ self.condVars = condVars;
+ self.numCondVars = len(self.condVars);
+ self.normMin = normMin;
+ self.normMax = normMax;
+ conec = mlgraph((self.numCondVars + 1, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
+ self.net = ffnet(conec);
+
+ def sample(self, *args, **keywords):
+ '''
+ Produces a sample from this distribution
+ *args - Value of each of the variables this distribution is conditioned
+ upon and x.  Must have a value for each conditional variable and x.
+ numSamples - Number of samples to generate.  Default is 1000.
+ '''
+ # Ensure each conditional variable is provided
+ assert(len(args) == self.numCondVars + 1);
+
+ # Get number of samples if provided
+ numSamples = 1000;
+ if 'numSamples' in keywords:
+ numSamples = keywords['numSamples'];
+
+ # Normalize values of variables this distribution is conditioned upon
+ inputs = [args];
+ if len(args) > 1:
+ for condVals in args.keys():
+ if condVals not in self.condVars:
+ raise Exception("Conditional value not a parent of node.");
+ for cond in self.condVars:
+ inputs.append(args[cond]);
+ values = (array(inputs) - self.normMin) / (self.normMax - self.normMin);
+
+ # Get parameters for each normal distribution
+ normParams = self.net(values); # Need exact number of inputs in values
+ normParams = [normParams[3 * i : 3 * i + 3] for i in range(len(normParams) / 3)];
+ normParams = array(normParams).T;
+ weights = normParams;
+ weights = weights / sum(weights);
+ normParams = weights;
+ normParams = normParams.T;
+
+ # Generate sample
+ retVal = [];
+ cWeights = [sum(weights[0 : i + 1]) for i in range(len(weights))];
+ bins = digitize(random_sample(numSamples), cWeights);
+ for i in range(numSamples):
+ for i in range(MCMPGMMDist.NUM_NORMALS):
+ mean = normParams[i] * (self.normMax - self.normMin) + self.normMin;
+ sd = normParams[i] * (self.normMax - self.normMin) + self.normMin;
+ retVal += [normal(mean, sd, numSamples).T];
+ retVal = retVal * mask;
+
+ return retVal[where(retVal != 0)];
+
+ '''
+ Adds a conditional variable to this distribution
+ newCondVar - Identifier for new conditional variable
+ '''
+ # net1 = ffnet(mlgraph((2, 3, 3), False))
+ # net2 = ffnet(mlgraph((3, 3, 3), False))
+ # net1.weights = array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
+ # net2.weights = array([1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
+ # net1([1, 1])
+ # -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]
+ # net2([1, 1, 1])
+ # -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]
+
+ # Create new neural network
+ conec = mlgraph((self.numCondVars + 2, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
+ newNet = ffnet(conec);
+
+ # Insert weights for new variable
+ newNet.weights = self.net.weights;
+ insertPoints = range(1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2) + 1, self.numCondVars + 2);
+ bound = 2.38 / sqrt(self.numCondVars + 3);
+ newNet.weights = insert(newNet.weights, insertPoints, random_sample(len(insertPoints)) * bound * 2 - bound);
+ self.net = newNet;
+
+ # Update remaining variables
+ self.condVars.append(newCondVar);
+ self.numCondVars = len(self.condVars);
+
+ def removeConditional(self, condVar):
+ '''
+ Removes a conditional variable from this distribution
+ condVar - Identifier for the conditional variable to remove
+ '''
+ # Create new neural network
+ conec = mlgraph((self.numCondVars, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
+ newNet = ffnet(conec);
+
+ # Insert weights for new variable
+ newNet.weights = self.net.weights;
+ removalPoints = range((self.numCondVars - self.condVars.index(condVar) - 1) + 1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2), self.numCondVars + 2);
+ newNet.weights = delete(newNet.weights, removalPoints);
+ self.net = newNet;
+
+ # Update remaining variables
+ self.condVars.remove(condVar);
+ self.numCondVars = len(self.condVars);
+
+ def numWeights(self):
+ '''
+ Returns the number of weights in the neural network backing this distribution
+ '''
+ return len(self.net.weights);
+
+ def modifyWeightWithinBounds(self, wIndex, modifier):
+ '''
+ Modifies one of the weights of the neural network backing this distribution within
+ certain bounds for those weights.  Due to the behavior of this method, the amount in which
+ a unit modifier modifies the weight is a function of how far that weight is from its
+ bounds.  Thus, a modifier of -0.1 will modify a positive weight more than a modifier of 0.1
+ since a positive weight is farther from its lower bound.  In essence, a modifier of -0.1
+ will move a weight 10% towards its lower bound while a modifier of 0.1 will move a weight
+ 10% towards its upper bound.
+ wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
+ the neural network backing this distribution.
+ modifier - A value between -1 and 1 representing the amount to modify the weight towards
+ its bounds.  A positive value will increase the weight while a negative value will decrease
+ the weight.  A value of 1 will put the weight at its maximum bound while a value of -1 will
+ put the weight at its minimum bound.  A value of 0 will not modify the weight and any value
+ in between 0 and 1 or 0 and -1 will modify the value linearly towards its bound.  Values
+ greater than 1 or -1 are not allowed.
+ '''
+ # Check bounding conditions of modifier
+ if modifier > 1:
+ raise ValueError, 'Modifier value greater than 1:  modifier = ' + str(modifier);
+ if modifier < -1:
+ raise ValueError, 'Modifier value less than -1:  modifier = ' + str(modifier);
+
+ # Modify weight
+ inputs = self.numCondVars + 2;
+ bound = 0;
+ if wIndex < MCMPGMMDist.NUM_HIDDEN_VARS * inputs:
+ bound = 2.38 / sqrt(inputs);
+ else:
+ bound = 2.38 / sqrt(MCMPGMMDist.NUM_HIDDEN_VARS + 1);
+ if modifier > 0:
+ range = bound - self.net.weights[wIndex];
+ self.net.weights[wIndex] += range * modifier;
+ else:
+ range = bound + self.net.weights[wIndex];
+ self.net.weights[wIndex] -= range * -1 * modifier;
+
+ def modifyWeight(self, wIndex, modifier):
+ '''
+ Modifies one of the weights of the neural network backing this distribution within
+ certain bounds for those weights.
+ wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
+ the neural network backing this distribution.
+ modifier - The modifier for the weight.  This value is the percent change to apply to the
+ weight where 1 is 100% and -1 is -100%
+ '''
+ self.net.weights[wIndex] += self.net.weights[wIndex] * modifier;
+
+ def __deepcopy__(self, memo):
+ cpy = MCMPGMMDist(self.condVars[:], self.normMin, self.normMax);
+ #cpy.condVars = self.condVars[:];
+ #cpy.numCondVars = self.numCondVars;
+ #cpy.normMin = self.normMin;
+ #cpy.normMax = self.normMax;
+ cpy.net = deepcopy(self.net, memo);
+ return cpy;
+</code>
+
+<code C MCMPGMMDistTest.py>
+float sigmoid(__private float x)
+{
+ return 1/(1 + exp(-1*x))
+}
+</code>
+
+~~ODT~~
+\\
+\\
+\\
+\\
+\\
+\\
+\\
+\\
+\\
+\\
+~~DISCUSSION~~ 