GeneralConfusionMatrix Class

[SerializableAttribute]
public class GeneralConfusionMatrix

<SerializableAttribute>
Public Class GeneralConfusionMatrix

// Let's say we have a decision problem involving 3 classes. In a typical
// machine learning problem, have a set of expected, ground truth values:
// 
int[] expected = { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2 };

// And we have a set of values that have been predicted by a machine model:
// 
int[] predicted = { 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };

// We can get different performance measures to assess how good our model was at 
// predicting the true, expected, ground-truth labels for the decision problem:
var cm = new GeneralConfusionMatrix(classes: 3, expected: expected, predicted: predicted);

// We can obtain the proper confusion matrix using:
int[,] matrix = cm.Matrix;

// The values of this matrix should be the same as:
int[,] expectedMatrix =
{
    //              expected
    /*predicted*/ { 4, 0, 0 },
                  { 0, 4, 4 },
                  { 0, 0, 0 },
};


// We can get more information about our problem as well:
int classes = cm.NumberOfClasses; // should be 3
int samples = cm.NumberOfSamples; // should be 12

// And multiple performance measures:
double accuracy = cm.Accuracy;                      // should be 0.66666666666666663
double error = cm.Error;                            // should be 0.33333333333333337
double chanceAgreement = cm.ChanceAgreement;        // should be 0.33333333333333331
double geommetricAgreement = cm.GeometricAgreement; // should be 0 (the classifier completely missed one class)
double pearson = cm.Pearson;                        // should be 0.70710678118654757
double kappa = cm.Kappa;                            // should be 0.49999999999999994
double tau = cm.Tau;                                // should be 0.49999999999999994
double chiSquare = cm.ChiSquare;                    // should be 12

// and some of their standard errors:
double kappaStdErr = cm.StandardError;              // should be 0.15590239111558091
double kappaStdErr0 = cm.StandardErrorUnderNull;    // should be 0.16666666666666663

// Let's say we have the following matrix
int[,] matrix =
{
    { 29,  6,  5 },
    {  8, 20,  7 },
    {  1,  2, 22 },
};

// Create a new multi-class Confusion Matrix
var cm = new GeneralConfusionMatrix(matrix);

// Now we can use it to obtain info such as
int classes = cm.NumberOfClasses; // should be 3
int samples = cm.NumberOfSamples; // should be 100

double acc = cm.Accuracy;          // should be 0.71
double err = cm.Error;             // should be 0.29
double ca = cm.ChanceAgreement;    // should be 0.335
double ga = cm.GeometricAgreement; // should be 23.37
double kappa = cm.Kappa;           // should be 0.563

// Generate always same random numbers
Accord.Math.Random.Generator.Seed = 0;

// Let's say we would like to learn a classifier for the famous Iris
// dataset, and measure its performance using a GeneralConfusionMatrix

// Download and load the Iris dataset
var iris = new Iris();
double[][] inputs = iris.Instances;
int[] outputs = iris.ClassLabels;

// Create the multi-class learning algorithm for the machine
var teacher = new MulticlassSupportVectorLearning<Linear>()
{
    // Configure the learning algorithm to use SMO to train the
    //  underlying SVMs in each of the binary class subproblems.
    Learner = (param) => new SequentialMinimalOptimization<Linear>()
    {
        // If you would like to use other kernels, simply replace
        // the generic parameter to the desired kernel class, such
        // as for example, Polynomial or Gaussian:

        Kernel = new Linear() // use the Linear kernel
    }
};

// Estimate the multi-class support vector machine using one-vs-one method
MulticlassSupportVectorMachine<Linear> ovo = teacher.Learn(inputs, outputs);

// Compute classification error
GeneralConfusionMatrix cm = GeneralConfusionMatrix.Estimate(ovo, inputs, outputs);

double error = cm.Error;         // should be 0.066666666666666652
double accuracy = cm.Accuracy;   // should be 0.93333333333333335
double kappa = cm.Kappa;         // should be 0.9
double chiSquare = cm.ChiSquare; // should be 248.52216748768473

// Example from R. Congalton's book, "Assesssing 
// the accuracy of remotely sensed data", 1999.

int[,] table = // Analyst #1 (page 108)
{
    { 65,  4, 22, 24 },
    {  6, 81,  5,  8 },
    {  0, 11, 85, 19 },
    {  4,  7,  3, 90 },
};

// Create a new multi-class confusion matrix
var cm = new GeneralConfusionMatrix(table);

int[] rowTotals = cm.RowTotals;     // should be { 115, 100, 115, 104 }
int[] colTotals = cm.ColumnTotals;  // should be { 115, 100, 115, 104 }

double kappa = cm.Kappa;            // should be 0.65
double var = cm.Variance;           // should be 0.00076995084473426684
double var0 = cm.VarianceUnderNull; // should be 0.00074886435981842887
double varD = Accord.Statistics.Testing.KappaTest.DeltaMethodKappaVariance(cm); // should be 0.0007778

// Example from Ientilucci, Emmett (2006). "On Using and Computing the Kappa Statistic".
// Available on: http://www.cis.rit.edu/~ejipci/Reports/On_Using_and_Computing_the_Kappa_Statistic.pdf 

// Note: Congalton's method uses the Delta Method for approximating the Kappa variance,
// but the framework uses the corrected methods by Cohen and Fleiss. For more information, see:
// https://stats.stackexchange.com/questions/30604/computing-cohens-kappa-variance-and-standard-errors

int[,] matrix = // Matrix A (page 1)
{
    { 317,  23,  0,  0 },
    {  61, 120,  0,  0 },
    {   2,   4, 60,  0 },
    {  35,  29,  0,  8 },
};

// Create the multi-class confusion matrix
var a = new GeneralConfusionMatrix(matrix);

// Method A row totals (page 2)
int[] rowTotals = a.RowTotals;       // should be { 340, 181, 66, 72 }

// Method A col totals (page 2)
int[] colTotals = a.ColumnTotals;    // should be { 415, 176, 60, 8 }

// Number of samples for A (page 2)
int samples = a.NumberOfSamples;     // should be 659
int classes = a.NumberOfClasses;     // should be 4

// Po for A (page 2)
double po = a.OverallAgreement;      // should be 0.7663

// Pc for A (page 2)
double ca = a.ChanceAgreement;       // should be 0.4087

// Kappa value k_hat for A (page 3)
double kappa = a.Kappa;              // should be 0.605

// Variance value var_k for A (page 4)
double varD = a.VarianceDeltaMethod; // should be 0.00073735

// Other variance values according to Fleiss and Cohen
double var = a.Variance;             // should be 0.00071760415564207924
double var0 = a.VarianceUnderNull;   // should be 0.00070251065008366978

// Example from J. L. Fleiss, J. Cohen, B. S. Everitt, "Large sample
//  standard errors of kappa and weighted kappa" Psychological Bulletin (1969)
//  Volume: 72, Issue: 5, American Psychological Association, Pages: 323-327

// This was the paper which presented the finally correct
// large sample variance for Kappa after so many attempts.

double[,] matrix =
{
    { 0.53,  0.05,  0.02 },
    { 0.11,  0.14,  0.05 },
    { 0.01,  0.06,  0.03 },
};

// Create a new multi-class confusion matrix:
var a = new GeneralConfusionMatrix(matrix, 200);

double[] rowProportions = a.RowProportions;    // should be { 0.6, 0.3, 0.1 }
double[] colProportions = a.ColumnProportions; // should be { 0.65, 0.25, 0.10 }

double kappa = a.Kappa;                        // should be 0.429
double var = a.Variance;                       // should be 0.002885
double var0 = a.VarianceUnderNull;             // should be 0.003082