# Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution

The Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution (MCMPGMMDist).

A Bayesian network node conditional probability distribution is represented by a series of Gaussian mixture models. A Gaussian mixture model is defined for every combination of states the parent nodes of the given node can take on. The parameters for each Gaussian mixture model is determined from a multilayer perceptron. As the mixture models for each set of parent states are not independent of each other, the neural network serves to also define and maintain the relationship of the mixture models of the conditional probability distribution.

Ultimately, the states of the parents of the given node are fed into the neural network. There is an input node for each parent node. The output nodes of the neural network then provide the parameters of the Gaussian mixture model for the given state set of the parent nodes. These parameters are the weight, mean, and standard deviation of each normal distribution in the Gaussian mixture model. This set of parameters define the Gaussian mixture model for the particular state set of the parent nodes of the given node.

The advantage of this methodology is that any continuous probability distribution can be approximated by the combination of the multilayer perceptron and the Gaussian mixture model. Furthermore, the relationship between the distributions for different parent state sets is maintained by the neural network. By this methodology recognizes this relationship, we are able to train this distribution more easily and represent and represent the nodes conditional probability distribution more compactly.

x x[1, {'y':2, 'z':3}] MCMPGMMDistTest.py
```'''
Created on Nov 19, 2009

@author: Stephen
'''
import unittest
from MCMPGMMDist import MCMPGMMDist
from numpy import sqrt;

class Test(unittest.TestCase):
def testAddConditional(self):
input1 = [2, 3, 7];
input2 = [2, 3, 7, 0];
dist = MCMPGMMDist(['a', 'b'], 0, 10);
output1 = dist.net(input1);
dist.addConditional('c');
output2 = dist.net(input2);
self.assertEqual(output1, output2);
self.assertEquals(dist.condVars, ['a', 'b', 'c']);

def testRemoveConditional(self):
input1 = [2, 3, 0, 7];
input2 = [2, 3, 7];
dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
output1 = dist.net(input1);
dist.removeConditional('b');
output2 = dist.net(input2);
self.assertEqual(output1, output2);
self.assertEquals(dist.condVars, ['a', 'c']);

def testNumWeights(self):
dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
self.assertEquals(dist.numWeights(), 5 * MCMPGMMDist.NUM_HIDDEN_VARS + (MCMPGMMDist.NUM_HIDDEN_VARS + 1) * MCMPGMMDist.NUM_NORMALS * 3);

def testModifyWeightWithinBounds(self):
# Setup test
dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
wStartValue = dist.net.weights;
bound = 2.38 / sqrt(5);

# Modify weights a little and test
dist.modifyWeightWithinBounds(1, 0.1);
self.assertAlmostEquals(dist.net.weights - wStartValue, (bound - wStartValue) * .1);

# Test extremes of modifier
dist.modifyWeightWithinBounds(1, 1);
self.assertAlmostEquals(dist.net.weights, bound);
dist.modifyWeightWithinBounds(1, -1);
self.assertAlmostEquals(dist.net.weights, -bound);
dist.modifyWeightWithinBounds(1, 0.5);
self.assertAlmostEquals(dist.net.weights, 0);

# Test exception thrown when going out of bounds
self.assertRaises(ValueError, dist.modifyWeightWithinBounds, 1, 1.1);

def testModifyWeight(self):
dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
wStartValue = dist.net.weights;
dist.modifyWeight(1, 4.4);
self.assertAlmostEquals(dist.net.weights - wStartValue, wStartValue * 4.4);

if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testAddConditional', 'Test.testRemoveConditional', 'Test.testNumWeights', 'Test.testModifyWeightWithinBounds', 'Test.testModifyWeight']
unittest.main()```
MCMPGMMDist.py
```'''
Created on Aug 1, 2009

@author: Stephen
'''

from ffnet import ffnet, mlgraph, savenet, loadnet, exportnet;
from numpy.random import normal, random_sample;
from numpy import array, digitize, zeros, where, insert, delete, sqrt;
from copy import deepcopy

class MCMPGMMDist(object):
'''
Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution
'''

NUM_HIDDEN_VARS = 10;
NUM_NORMALS = 10;

def __init__(self, condVars, normMin, normMax):
'''
Constructor
condVars - List of identifiers for conditional variables for this conditional distribution
normMin - Minimum possible value any of the conditional variables can be
normMax - Maximum possible value any of the conditional variables can be
'''
if(condVars.__class__ != [].__class__):
raise TypeError("concVars must be a list");
self.condVars = condVars;
self.numCondVars = len(self.condVars);
self.normMin = normMin;
self.normMax = normMax;
conec = mlgraph((self.numCondVars + 1, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
self.net = ffnet(conec);

def sample(self, *args, **keywords):
'''
Produces a sample from this distribution
*args - Value of each of the variables this distribution is conditioned
upon and x.  Must have a value for each conditional variable and x.
numSamples - Number of samples to generate.  Default is 1000.
'''
# Ensure each conditional variable is provided
assert(len(args) == self.numCondVars + 1);

# Get number of samples if provided
numSamples = 1000;
if 'numSamples' in keywords:
numSamples = keywords['numSamples'];

# Normalize values of variables this distribution is conditioned upon
inputs = [args];
if len(args) > 1:
for condVals in args.keys():
if condVals not in self.condVars:
raise Exception("Conditional value not a parent of node.");
for cond in self.condVars:
inputs.append(args[cond]);
values = (array(inputs) - self.normMin) / (self.normMax - self.normMin);

# Get parameters for each normal distribution
normParams = self.net(values);		# Need exact number of inputs in values
normParams = [normParams[3 * i : 3 * i + 3] for i in range(len(normParams) / 3)];
normParams = array(normParams).T;
weights = normParams;
weights = weights / sum(weights);
normParams = weights;
normParams = normParams.T;

# Generate sample
retVal = [];
cWeights = [sum(weights[0 : i + 1]) for i in range(len(weights))];
bins = digitize(random_sample(numSamples), cWeights);
mask = zeros([MCMPGMMDist.NUM_NORMALS, numSamples]);
for i in range(numSamples):
mask[bins[i]][i] = 1;
for i in range(MCMPGMMDist.NUM_NORMALS):
mean = normParams[i] * (self.normMax - self.normMin) + self.normMin;
sd = normParams[i] * (self.normMax - self.normMin) + self.normMin;
retVal += [normal(mean, sd, numSamples).T];
retVal = retVal * mask;

return retVal[where(retVal != 0)];

def addConditional(self, newCondVar):
'''
Adds a conditional variable to this distribution
newCondVar - Identifier for new conditional variable
'''
# net1 = ffnet(mlgraph((2, 3, 3), False))
# net2 = ffnet(mlgraph((3, 3, 3), False))
# net1.weights = array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
# net2.weights = array([1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
# net1([1, 1])
# -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]
# net2([1, 1, 1])
# -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]

# Create new neural network
conec = mlgraph((self.numCondVars + 2, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
newNet = ffnet(conec);

# Insert weights for new variable
newNet.weights = self.net.weights;
insertPoints = range(1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2) + 1, self.numCondVars + 2);
bound = 2.38 / sqrt(self.numCondVars + 3);
newNet.weights = insert(newNet.weights, insertPoints, random_sample(len(insertPoints)) * bound * 2 - bound);
self.net = newNet;

# Update remaining variables
self.condVars.append(newCondVar);
self.numCondVars = len(self.condVars);

def removeConditional(self, condVar):
'''
Removes a conditional variable from this distribution
condVar - Identifier for the conditional variable to remove
'''
# Create new neural network
conec = mlgraph((self.numCondVars, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
newNet = ffnet(conec);

# Insert weights for new variable
newNet.weights = self.net.weights;
removalPoints = range((self.numCondVars - self.condVars.index(condVar) - 1) + 1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2), self.numCondVars + 2);
newNet.weights = delete(newNet.weights, removalPoints);
self.net = newNet;

# Update remaining variables
self.condVars.remove(condVar);
self.numCondVars = len(self.condVars);

def numWeights(self):
'''
Returns the number of weights in the neural network backing this distribution
'''
return len(self.net.weights);

def modifyWeightWithinBounds(self, wIndex, modifier):
'''
Modifies one of the weights of the neural network backing this distribution within
certain bounds for those weights.  Due to the behavior of this method, the amount in which
a unit modifier modifies the weight is a function of how far that weight is from its
bounds.  Thus, a modifier of -0.1 will modify a positive weight more than a modifier of 0.1
since a positive weight is farther from its lower bound.  In essence, a modifier of -0.1
will move a weight 10% towards its lower bound while a modifier of 0.1 will move a weight
10% towards its upper bound.
wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
the neural network backing this distribution.
modifier - A value between -1 and 1 representing the amount to modify the weight towards
its bounds.  A positive value will increase the weight while a negative value will decrease
the weight.  A value of 1 will put the weight at its maximum bound while a value of -1 will
put the weight at its minimum bound.  A value of 0 will not modify the weight and any value
in between 0 and 1 or 0 and -1 will modify the value linearly towards its bound.  Values
greater than 1 or -1 are not allowed.
'''
# Check bounding conditions of modifier
if modifier > 1:
raise ValueError, 'Modifier value greater than 1:  modifier = ' + str(modifier);
if modifier < -1:
raise ValueError, 'Modifier value less than -1:  modifier = ' + str(modifier);

# Modify weight
inputs = self.numCondVars + 2;
bound = 0;
if wIndex < MCMPGMMDist.NUM_HIDDEN_VARS * inputs:
bound = 2.38 / sqrt(inputs);
else:
bound = 2.38 / sqrt(MCMPGMMDist.NUM_HIDDEN_VARS + 1);
if modifier > 0:
range = bound - self.net.weights[wIndex];
self.net.weights[wIndex] += range * modifier;
else:
range = bound + self.net.weights[wIndex];
self.net.weights[wIndex] -= range * -1 * modifier;

def modifyWeight(self, wIndex, modifier):
'''
Modifies one of the weights of the neural network backing this distribution within
certain bounds for those weights.
wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
the neural network backing this distribution.
modifier - The modifier for the weight.  This value is the percent change to apply to the
weight where 1 is 100% and -1 is -100%
'''
self.net.weights[wIndex] += self.net.weights[wIndex] * modifier;

def __deepcopy__(self, memo):
cpy = MCMPGMMDist(self.condVars[:], self.normMin, self.normMax);
#cpy.condVars = self.condVars[:];
#cpy.numCondVars = self.numCondVars;
#cpy.normMin = self.normMin;
#cpy.normMax = self.normMax;
cpy.net = deepcopy(self.net, memo);
return cpy;```
MCMPGMMDistTest.py
```float sigmoid(__private float x)
{
return 1/(1 + exp(-1*x))
}```
You could leave a comment if you were logged in.

goplayer/mcffnngmmdist.txt · Last modified: 2011/12/16 12:29 by aiartificer 