This is an old revision of the document!


Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution

The Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution (MCMPGMMDist).

A Bayesian network node conditional probability distribution is represented by a series of Gaussian mixture models. A Gaussian mixture model is defined for every combination of states the parent nodes of the given node can take on. The parameters for each Gaussian mixture model is determined from a multilayer perceptron. As the mixture models for each set of parent states are not independent of each other, the neural network serves to also define and maintain the relationship of the mixture models of the conditional probability distribution.

Ultimately, the states of the parents of the given node are fed into the neural network. There is an input node for each parent node. The output nodes of the neural network then provide the parameters of the Gaussian mixture model for the given state set of the parent nodes. These parameters are the weight, mean, and standard deviation of each normal distribution in the Gaussian mixture model. This set of parameters define the Gaussian mixture model for the particular state set of the parent nodes of the given node.

The advantage of this methodology is that any continuous probability distribution can be approximated by the combination of the multilayer perceptron and the Gaussian mixture model. Furthermore, the relationship between the distributions for different parent state sets is maintained by the neural network. By this methodology recognizes this relationship, we are able to train this distribution more easily and represent and represent the nodes conditional probability distribution more compactly.

x[1] right~P(X=1)

x[1, {'y':2, 'z':3}] right~P(X=1 | Y=2, Z=3)

MCMPGMMDistTest.py
'''
Created on Nov 19, 2009
 
@author: Stephen
'''
import unittest
from MCMPGMMDist import MCMPGMMDist
from numpy import sqrt;
 
 
class Test(unittest.TestCase):
	def testAddConditional(self):
		input1 = [2, 3, 7];
		input2 = [2, 3, 7, 0];
		dist = MCMPGMMDist(['a', 'b'], 0, 10);
		output1 = dist.net(input1);
		dist.addConditional('c');
		output2 = dist.net(input2);
		self.assertEqual(output1, output2);
		self.assertEquals(dist.condVars, ['a', 'b', 'c']);
 
	def testRemoveConditional(self):
		input1 = [2, 3, 0, 7];
		input2 = [2, 3, 7];
		dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
		output1 = dist.net(input1);
		dist.removeConditional('b');
		output2 = dist.net(input2);
		self.assertEqual(output1, output2);
		self.assertEquals(dist.condVars, ['a', 'c']);
 
	def testNumWeights(self):
		dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
		self.assertEquals(dist.numWeights(), 5 * MCMPGMMDist.NUM_HIDDEN_VARS + (MCMPGMMDist.NUM_HIDDEN_VARS + 1) * MCMPGMMDist.NUM_NORMALS * 3);
 
	def testModifyWeightWithinBounds(self):
		# Setup test
		dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
		wStartValue = dist.net.weights[1];
		bound = 2.38 / sqrt(5);
 
		# Modify weights a little and test
		dist.modifyWeightWithinBounds(1, 0.1);
		self.assertAlmostEquals(dist.net.weights[1] - wStartValue, (bound - wStartValue) * .1);
 
		# Test extremes of modifier
		dist.modifyWeightWithinBounds(1, 1);
		self.assertAlmostEquals(dist.net.weights[1], bound);
		dist.modifyWeightWithinBounds(1, -1);
		self.assertAlmostEquals(dist.net.weights[1], -bound);
		dist.modifyWeightWithinBounds(1, 0.5);
		self.assertAlmostEquals(dist.net.weights[1], 0);
 
		# Test exception thrown when going out of bounds
		self.assertRaises(ValueError, dist.modifyWeightWithinBounds, 1, 1.1);
 
	def testModifyWeight(self):
		dist = MCMPGMMDist(['a', 'b', 'c'], 0, 10);
		wStartValue = dist.net.weights[1];
		dist.modifyWeight(1, 4.4);
		self.assertAlmostEquals(dist.net.weights[1] - wStartValue, wStartValue * 4.4);
 
 
if __name__ == "__main__":
	#import sys;sys.argv = ['', 'Test.testAddConditional', 'Test.testRemoveConditional', 'Test.testNumWeights', 'Test.testModifyWeightWithinBounds', 'Test.testModifyWeight']
	unittest.main()
MCMPGMMDist.py
'''
Created on Aug 1, 2009
 
@author: Stephen
'''
 
from ffnet import ffnet, mlgraph, savenet, loadnet, exportnet;
from numpy.random import normal, random_sample;
from numpy import array, digitize, zeros, where, insert, delete, sqrt;
from copy import deepcopy
 
 
class MCMPGMMDist(object):
	'''
	Monte Carlo Multilayer Perceptron Gaussian Mixture Model Distribution
	'''
 
	NUM_HIDDEN_VARS = 10;
	NUM_NORMALS = 10;
 
	def __init__(self, condVars, normMin, normMax):
		'''
		Constructor
		condVars - List of identifiers for conditional variables for this conditional distribution
		normMin - Minimum possible value any of the conditional variables can be
		normMax - Maximum possible value any of the conditional variables can be
		'''
		if(condVars.__class__ != [].__class__):
			raise TypeError("concVars must be a list");
		self.condVars = condVars;
		self.numCondVars = len(self.condVars);
		self.normMin = normMin;
		self.normMax = normMax;
		conec = mlgraph((self.numCondVars + 1, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
		self.net = ffnet(conec);
 
	def sample(self, *args, **keywords):
		'''
		Produces a sample from this distribution
		*args - Value of each of the variables this distribution is conditioned
		upon and x.  Must have a value for each conditional variable and x.
		numSamples - Number of samples to generate.  Default is 1000.
		'''
		# Ensure each conditional variable is provided
		assert(len(args) == self.numCondVars + 1);
 
		# Get number of samples if provided
		numSamples = 1000;
		if 'numSamples' in keywords:
			numSamples = keywords['numSamples'];
 
		# Normalize values of variables this distribution is conditioned upon
		inputs = [args[0]];
		if len(args) > 1:
			for condVals in args[1].keys():
				if condVals not in self.condVars:
					raise Exception("Conditional value not a parent of node.");
		for cond in self.condVars:
			inputs.append(args[1][cond]);
		values = (array(inputs) - self.normMin) / (self.normMax - self.normMin);
 
		# Get parameters for each normal distribution
		normParams = self.net(values);		# Need exact number of inputs in values
		normParams = [normParams[3 * i : 3 * i + 3] for i in range(len(normParams) / 3)];
		normParams = array(normParams).T;
		weights = normParams[0];
		weights = weights / sum(weights);
		normParams[0] = weights;
		normParams = normParams.T;
 
		# Generate sample
		retVal = [];
		cWeights = [sum(weights[0 : i + 1]) for i in range(len(weights))];
		bins = digitize(random_sample(numSamples), cWeights);
		mask = zeros([MCMPGMMDist.NUM_NORMALS, numSamples]);
		for i in range(numSamples):
			mask[bins[i]][i] = 1;
		for i in range(MCMPGMMDist.NUM_NORMALS):
			mean = normParams[i][1] * (self.normMax - self.normMin) + self.normMin;
			sd = normParams[i][2] * (self.normMax - self.normMin) + self.normMin;
			retVal += [normal(mean, sd, numSamples).T];
		retVal = retVal * mask;
 
		return retVal[where(retVal != 0)];
 
	def addConditional(self, newCondVar):
		'''
		Adds a conditional variable to this distribution
		newCondVar - Identifier for new conditional variable
		'''
		# net1 = ffnet(mlgraph((2, 3, 3), False))
		# net2 = ffnet(mlgraph((3, 3, 3), False))
		# net1.weights = array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
		# net2.weights = array([1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
		# net1([1, 1])
		# -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]
		# net2([1, 1, 1])
		# -> [0.97932062734407388, 0.97932062734407388, 0.97932062734407388]
 
		# Create new neural network
		conec = mlgraph((self.numCondVars + 2, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
		newNet = ffnet(conec);
 
		# Insert weights for new variable
		newNet.weights = self.net.weights;
		insertPoints = range(1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2) + 1, self.numCondVars + 2);
		bound = 2.38 / sqrt(self.numCondVars + 3);
		newNet.weights = insert(newNet.weights, insertPoints, random_sample(len(insertPoints)) * bound * 2 - bound);
		self.net = newNet;
 
		# Update remaining variables
		self.condVars.append(newCondVar);
		self.numCondVars = len(self.condVars);
 
	def removeConditional(self, condVar):
		'''
		Removes a conditional variable from this distribution
		condVar - Identifier for the conditional variable to remove
		'''
		# Create new neural network
		conec = mlgraph((self.numCondVars, MCMPGMMDist.NUM_HIDDEN_VARS, MCMPGMMDist.NUM_NORMALS * 3));
		newNet = ffnet(conec);
 
		# Insert weights for new variable
		newNet.weights = self.net.weights;
		removalPoints = range((self.numCondVars - self.condVars.index(condVar) - 1) + 1, MCMPGMMDist.NUM_HIDDEN_VARS * (self.numCondVars + 2), self.numCondVars + 2);
		newNet.weights = delete(newNet.weights, removalPoints);
		self.net = newNet;
 
		# Update remaining variables
		self.condVars.remove(condVar);
		self.numCondVars = len(self.condVars);
 
	def numWeights(self):
		'''
		Returns the number of weights in the neural network backing this distribution
		'''
		return len(self.net.weights);
 
	def modifyWeightWithinBounds(self, wIndex, modifier):
		'''
		Modifies one of the weights of the neural network backing this distribution within
		certain bounds for those weights.  Due to the behavior of this method, the amount in which
		a unit modifier modifies the weight is a function of how far that weight is from its
		bounds.  Thus, a modifier of -0.1 will modify a positive weight more than a modifier of 0.1
		since a positive weight is farther from its lower bound.  In essence, a modifier of -0.1
		will move a weight 10% towards its lower bound while a modifier of 0.1 will move a weight
		10% towards its upper bound.
		wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
		the neural network backing this distribution.
		modifier - A value between -1 and 1 representing the amount to modify the weight towards
		its bounds.  A positive value will increase the weight while a negative value will decrease
		the weight.  A value of 1 will put the weight at its maximum bound while a value of -1 will
		put the weight at its minimum bound.  A value of 0 will not modify the weight and any value
		in between 0 and 1 or 0 and -1 will modify the value linearly towards its bound.  Values
		greater than 1 or -1 are not allowed.
		'''
		# Check bounding conditions of modifier
		if modifier > 1:
			raise ValueError, 'Modifier value greater than 1:  modifier = ' + str(modifier);
		if modifier < -1:
			raise ValueError, 'Modifier value less than -1:  modifier = ' + str(modifier);
 
		# Modify weight
		inputs = self.numCondVars + 2;
		bound = 0;
		if wIndex < MCMPGMMDist.NUM_HIDDEN_VARS * inputs:
			bound = 2.38 / sqrt(inputs);
		else:
			bound = 2.38 / sqrt(MCMPGMMDist.NUM_HIDDEN_VARS + 1);
		if modifier > 0:
			range = bound - self.net.weights[wIndex];
			self.net.weights[wIndex] += range * modifier;
		else:
			range = bound + self.net.weights[wIndex];
			self.net.weights[wIndex] -= range * -1 * modifier;
 
	def modifyWeight(self, wIndex, modifier):
		'''
		Modifies one of the weights of the neural network backing this distribution within
		certain bounds for those weights.
		wIndex - The index of the weight to modify.  Must be between 0 and the number of weights in
		the neural network backing this distribution.
		modifier - The modifier for the weight.  This value is the percent change to apply to the
		weight where 1 is 100% and -1 is -100%
		'''
		self.net.weights[wIndex] += self.net.weights[wIndex] * modifier;
 
	def __deepcopy__(self, memo):
		cpy = MCMPGMMDist(self.condVars[:], self.normMin, self.normMax);
		#cpy.condVars = self.condVars[:];
		#cpy.numCondVars = self.numCondVars;
		#cpy.normMin = self.normMin;
		#cpy.normMax = self.normMax;
		cpy.net = deepcopy(self.net, memo);
		return cpy;

Export page to Open Document format









You could leave a comment if you were logged in.

goplayer/mcffnngmmdist.1324054677.txt.gz · Last modified: 2011/12/16 08:57 by aiartificer