HiddenQuasiNewtonLearningT Class

public class HiddenQuasiNewtonLearning<T> : BaseHiddenGradientOptimizationLearning<T, BoundedBroydenFletcherGoldfarbShanno>, 
	ISupervisedLearning<HiddenConditionalRandomField<T>, T[], int>, IParallel, ISupportsCancellation, 
	IHiddenConditionalRandomFieldLearning<T>, IConvergenceLearning, IDisposable

Public Class HiddenQuasiNewtonLearning(Of T)
	Inherits BaseHiddenGradientOptimizationLearning(Of T, BoundedBroydenFletcherGoldfarbShanno)
	Implements ISupervisedLearning(Of HiddenConditionalRandomField(Of T), T(), Integer), 
	IParallel, ISupportsCancellation, IHiddenConditionalRandomFieldLearning(Of T), IConvergenceLearning, IDisposable

// Let's say we would like to do a very simple mechanism for gesture recognition. 
// In this example, we will be trying to create a classifier that can distinguish 
// between the words "hello", "car", and "wardrobe". 

// Let's say we decided to acquire some data, and we asked some people to perform 
// those words in front of a Kinect camera, and, using Microsoft's SDK, we were able 
// to captured the x and y coordinates of each hand while the word was being performed.

// Let's say we decided to represent our frames as:
// 
//    double[] frame = { leftHandX, leftHandY, rightHandX, rightHandY }; // 4 dimensions
// 
// Since we captured words, this means we captured sequences of frames as we described 
// above. Let's write some of those as rough examples to explain how gesture recognition 
// can be done:

double[][] hello =
{
    new double[] { 1.0, 0.1, 0.0, 0.0 }, // let's say the word
    new double[] { 0.0, 1.0, 0.1, 0.1 }, // hello took 6 frames
    new double[] { 0.0, 1.0, 0.1, 0.1 }, // to be recorded.
    new double[] { 0.0, 0.0, 1.0, 0.0 },
    new double[] { 0.0, 0.0, 1.0, 0.0 },
    new double[] { 0.0, 0.0, 0.1, 1.1 },
};

double[][] car =
{
    new double[] { 0.0, 0.0, 0.0, 1.0 }, // the car word
    new double[] { 0.1, 0.0, 1.0, 0.1 }, // took only 4.
    new double[] { 0.0, 0.0, 0.1, 0.0 },
    new double[] { 1.0, 0.0, 0.0, 0.0 },
};

double[][] wardrobe =
{
    new double[] { 0.0, 0.0, 1.0, 0.0 }, // same for the
    new double[] { 0.1, 0.0, 1.0, 0.1 }, // wardrobe word.
    new double[] { 0.0, 0.1, 1.0, 0.0 },
    new double[] { 0.1, 0.0, 1.0, 0.1 },
};

// Please note that a real-world example would involve *lots* of samples for each word. 
// Here, we are considering just one from each class which is clearly sub-optimal and 
// should _never_ be done on practice. Please keep in mind that we are doing like this
// only to simplify this example on how to create and use HCRFs.

// These are the words we have in our vocabulary:
double[][][] words = { hello, car, wardrobe };

// Now, let's associate integer labels with them. This is needed
// for the case where there are multiple samples for each word.
int[] labels = { 0, 1, 2 };

// Create a new learning algorithm to train the hidden Markov model sequence classifier
var teacher = new HiddenMarkovClassifierLearning<Independent<NormalDistribution>, double[]>()
{
    // Train each model until the log-likelihood changes less than 0.001
    Learner = (i) => new BaumWelchLearning<Independent<NormalDistribution>, double[]>()
    {
        Topology = new Forward(5), // this value can be found by trial-and-error

        // We will create our classifiers assuming an independent Gaussian distribution 
        // for each component in our feature vectors (assuming a Naive Bayes assumption).
        Emissions = (s) => new Independent<NormalDistribution>(dimensions: 4), // 4 dimensions

        Tolerance = 0.001,
        Iterations = 100,

        // This is necessary so the code doesn't blow up when it realizes there is only one 
        // sample per word class. But this could also be needed in normal situations as well:
        FittingOptions = new IndependentOptions()
        {
            InnerOption = new NormalOptions() { Regularization = 1e-5 }
        }
    }
};

// PS: In case you find exceptions trying to configure your model, you might want 
//     to try disabling parallel processing to get more descriptive error messages:
// teacher.ParallelOptions.MaxDegreeOfParallelism = 1;

// Finally, we can run the learning algorithm!
var hmm = teacher.Learn(words, labels);
double logLikelihood = teacher.LogLikelihood;

// At this point, the classifier should be successfully 
// able to distinguish between our three word classes:
// 
int tc1 = hmm.Decide(hello);    // should be 0
int tc2 = hmm.Decide(car);      // should be 1
int tc3 = hmm.Decide(wardrobe); // should be 2

// Now, we can use the Markov classifier to initialize a HCRF
var baseline = HiddenConditionalRandomField.FromHiddenMarkov(hmm);

// We can check that both are equivalent, although they have
// formulations that can be learned with different methods:
int[] predictedLabels = baseline.Decide(words);

// Now we can learn the HCRF using one of the best learning
// algorithms available, Resilient Backpropagation learning:

// Create the Resilient Backpropagation learning algorithm
var rprop = new HiddenResilientGradientLearning<double[]>()
{
    Function = baseline.Function, // use the same HMM function

    Iterations = 50,
    Tolerance = 1e-5
};

// Run the algorithm and learn the models
var hcrf = rprop.Learn(words, labels);

// At this point, the HCRF should be successfully 
// able to distinguish between our three word classes:
// 
int hc1 = hcrf.Decide(hello);    // should be 0
int hc2 = hcrf.Decide(car);      // should be 1
int hc3 = hcrf.Decide(wardrobe); // should be 2

// Ensure we get reproducible results
Accord.Math.Random.Generator.Seed = 0;

// Download the PENDIGITS dataset from UCI ML repository
var pendigits = new Pendigits(path: localDownloadPath);

// Get and pre-process the training set
double[][][] trainInputs = pendigits.Training.Item1;
int[] trainOutputs = pendigits.Training.Item2;

// Pre-process the digits so each of them is centered and scaled
trainInputs = trainInputs.Apply(Accord.Statistics.Tools.ZScores);
trainInputs = trainInputs.Apply((x) => x.Subtract(x.Min())); // make them positive

// Create some prior distributions to help initialize our parameters
var priorC = new WishartDistribution(dimension: 2, degreesOfFreedom: 5);
var priorM = new MultivariateNormalDistribution(dimension: 2);

// Create a new learning algorithm for creating continuous hidden Markov model classifiers
var teacher1 = new HiddenMarkovClassifierLearning<MultivariateNormalDistribution, double[]>()
{
    // This tells the generative algorithm how to train each of the component models. Note: The learning
    // algorithm is more efficient if all generic parameters are specified, including the fitting options
    Learner = (i) => new BaumWelchLearning<MultivariateNormalDistribution, double[], NormalOptions>()
    {
        Topology = new Forward(5), // Each model will have a forward topology with 5 states

        // Their emissions will be multivariate Normal distributions initialized using the prior distributions
        Emissions = (j) => new MultivariateNormalDistribution(mean: priorM.Generate(), covariance: priorC.Generate()),

        // We will train until the relative change in the average log-likelihood is less than 1e-6 between iterations
        Tolerance = 1e-6,
        MaxIterations = 1000, // or until we perform 1000 iterations (which is unlikely for this dataset)

        // We will prevent our covariance matrices from becoming degenerate by adding a small 
        // regularization value to their diagonal until they become positive-definite again:
        FittingOptions = new NormalOptions()
        {
            Regularization = 1e-6
        }
    }
};

// The following line is only needed to ensure reproducible results. Please remove it to enable full parallelization
teacher1.ParallelOptions.MaxDegreeOfParallelism = 1; // (Remove, comment, or change this line to enable full parallelism)

// Use the learning algorithm to create a classifier
var hmmc = teacher1.Learn(trainInputs, trainOutputs);


// Create a new learning algorithm for creating HCRFs
var teacher2 = new HiddenQuasiNewtonLearning<double[]>()
{
    Function = new MarkovMultivariateFunction(hmmc),

    MaxIterations = 10
};

// The following line is only needed to ensure reproducible results. Please remove it to enable full parallelization
teacher2.ParallelOptions.MaxDegreeOfParallelism = 1; // (Remove, comment, or change this line to enable full parallelism)

// Use the learning algorithm to create a classifier
var hcrf = teacher2.Learn(trainInputs, trainOutputs);

// Compute predictions for the training set
int[] trainPredicted = hcrf.Decide(trainInputs);

// Check the performance of the classifier by comparing with the ground-truth:
var m1 = new GeneralConfusionMatrix(predicted: trainPredicted, expected: trainOutputs);
double trainAcc = m1.Accuracy; // should be 0.66523727844482561


// Prepare the testing set
double[][][] testInputs = pendigits.Testing.Item1;
int[] testOutputs = pendigits.Testing.Item2;

// Apply the same normalizations
testInputs = testInputs.Apply(Accord.Statistics.Tools.ZScores);
testInputs = testInputs.Apply((x) => x.Subtract(x.Min())); // make them positive

// Compute predictions for the test set
int[] testPredicted = hcrf.Decide(testInputs);

// Check the performance of the classifier by comparing with the ground-truth:
var m2 = new GeneralConfusionMatrix(predicted: testPredicted, expected: testOutputs);
double testAcc = m2.Accuracy; // should be 0.66506538564184681