protected double Run(Array[] observations)
{
// Baum-Welch algorithm.
// The Baum–Welch algorithm is a particular case of a generalized expectation-maximization
// (GEM) algorithm. It can compute maximum likelihood estimates and posterior mode estimates
// for the parameters (transition and emission probabilities) of an HMM, when given only
// emissions as training data.
// The algorithm has two steps:
// - Calculating the forward probability and the backward probability for each HMM state;
// - On the basis of this, determining the frequency of the transition-emission pair values
// and dividing it by the probability of the entire string. This amounts to calculating
// the expected count of the particular transition-emission pair. Each time a particular
// transition is found, the value of the quotient of the transition divided by the probability
// of the entire string goes up, and this value can then be made the new value of the transition.
// Grab model information
int states = model.States;
double[,] A = model.Transitions;
double[] pi = model.Probabilities;
// Initialize the algorithm
int N = observations.Length;
Ksi = new double[N][][,]; // also referred as ksi, psi or xi
Gamma = new double[N][,];
for (int i = 0; i < observations.Length; i++)
{
int T = observations[i].Length;
Ksi[i] = new double[T][,];
Gamma[i] = new double[T,states];
for (int t = 0; t < T; t++)
Ksi[i][t] = new double[states,states];
}
int iteration = 1;
bool stop = false;
// Initialize the model log-likelihoods
double newLikelihood = 0;
double oldLikelihood = Double.MinValue;
do // Until convergence or max iterations is reached
{
// For each sequence in the observations input
for (int i = 0; i < observations.Length; i++)
{
int T = observations[i].Length;
double[,] gamma = Gamma[i];
double[,] fwd, bwd;
double[] scaling;
// 1st step - Calculating the forward probability and the
// backward probability for each HMM state.
ComputeForwardBackward(i, out fwd, out bwd, out scaling);
// 2nd step - Determining the frequency of the transition-emission pair values
// and dividing it by the probability of the entire string.
// Calculate gamma values for next computations
for (int t = 0; t < T; t++)
{
double s = 0;
for (int k = 0; k < states; k++)
s += gamma[t, k] = fwd[t, k]*bwd[t, k];
if (s != 0) // Scaling
{
for (int k = 0; k < states; k++)
gamma[t, k] /= s;
}
}
// Calculate ksi values for next computations
ComputeKsi(i, fwd, bwd, scaling);
// Compute log-likelihood for the given sequence
for (int t = 0; t < scaling.Length; t++)
newLikelihood += System.Math.Log(scaling[t]);
}
// Average the likelihood for all sequences
newLikelihood /= observations.Length;
// Check if the model has converged or if we should stop
if (!HasConverged(oldLikelihood, newLikelihood, iteration))
{
// We haven't converged yet
// 3. Continue with parameter re-estimation
iteration++;
oldLikelihood = newLikelihood;
newLikelihood = 0.0;
// 3.1 Re-estimation of initial state probabilities
for (int k = 0; k < states; k++)
{
double sum = 0;
for (int i = 0; i < Gamma.Length; i++)
sum += Gamma[i][0, k];
pi[k] = sum/N;
}
// 3.2 Re-estimation of transition probabilities
for (int i = 0; i < states; i++)
{
for (int j = 0; j < states; j++)
{
double den = 0, num = 0;
for (int k = 0; k < Gamma.Length; k++)
{
int T = observations[k].Length;
double[,] gammak = Gamma[k];
double[][,] ksik = Ksi[k];
for (int l = 0; l < T - 1; l++)
{
num += ksik[l][i, j];
den += gammak[l, i];
}
}
A[i, j] = (den != 0) ? num/den : 0.0;
}
}
// 3.3 Re-estimation of emission probabilities
UpdateEmissions();
}
else
{
stop = true; // The model has converged.
}
} while (!stop);
// Returns the model average log-likelihood
return newLikelihood;
}