BagOfWords Class

[SerializableAttribute]
public class BagOfWords : ParallelLearningBase, IBagOfWords<string[]>, 
	ITransform<string[], double[]>, ICovariantTransform<string[], double[]>, 
	ITransform, ITransform<string[], int[]>, ICovariantTransform<string[], int[]>, 
	ITransform<string[], Sparse<double>>, ICovariantTransform<string[], Sparse<double>>, 
	IUnsupervisedLearning<BagOfWords, string[], int[]>

<SerializableAttribute>
Public Class BagOfWords
	Inherits ParallelLearningBase
	Implements IBagOfWords(Of String()), ITransform(Of String(), Double()), 
	ICovariantTransform(Of String(), Double()), ITransform, ITransform(Of String(), Integer()), 
	ICovariantTransform(Of String(), Integer()), ITransform(Of String(), Sparse(Of Double)), 
	ICovariantTransform(Of String(), Sparse(Of Double)), 
	IUnsupervisedLearning(Of BagOfWords, String(), Integer())

// The Bag-Of-Words model can be used to extract finite-length feature 
// vectors from sequences of arbitrary length, like for example, texts:


string[] texts =
{
    @"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas molestie malesuada 
      nisi et placerat. Curabitur blandit porttitor suscipit. Nunc facilisis ultrices felis,
      vitae luctus arcu semper in. Fusce ut felis ipsum. Sed faucibus tortor ut felis placerat
      euismod. Vestibulum pharetra velit et dolor ornare quis malesuada leo aliquam. Aenean 
      lobortis, tortor iaculis vestibulum dictum, tellus nisi vestibulum libero, ultricies 
      pretium nisi ante in neque. Integer et massa lectus. Aenean ut sem quam. Mauris at nisl 
      augue, volutpat tempus nisl. Suspendisse luctus convallis metus, vitae pretium risus 
      pretium vitae. Duis tristique euismod aliquam",

    @"Sed consectetur nisl et diam mattis varius. Aliquam ornare tincidunt arcu eget adipiscing. 
      Etiam quis augue lectus, vel sollicitudin lorem. Fusce lacinia, leo non porttitor adipiscing, 
      mauris purus lobortis ipsum, id scelerisque erat neque eget nunc. Suspendisse potenti. Etiam 
      non urna non libero pulvinar consequat ac vitae turpis. Nam urna eros, laoreet id sagittis eu,
      posuere in sapien. Phasellus semper convallis faucibus. Nulla fermentum faucibus tellus in 
      rutrum. Maecenas quis risus augue, eu gravida massa."
};

string[][] words = texts.Tokenize();

// Create a new BoW with options:
var codebook = new BagOfWords()
{
    MaximumOccurance = 1 // the resulting vector will have only 0's and 1's
};

// Compute the codebook (note: this would have to be done only for the training set)
codebook.Learn(words);


// Now, we can use the learned codebook to extract fixed-length
// representations of the different texts (paragraphs) above:

// Extract a feature vector from the text 1:
double[] bow1 = codebook.Transform(words[0]);

// Extract a feature vector from the text 2:
double[] bow2 = codebook.Transform(words[1]);

// we could also have transformed everything at once, i.e.
// double[][] bow = codebook.Transform(words);


// Now, since we have finite length representations (both bow1 and bow2 should
// have the same size), we can pass them to any classifier or machine learning
// method. For example, we can pass them to a Logistic Regression Classifier to
// discern between the first and second paragraphs

// Lets create a Logistic classifier to separate the two paragraphs:
var learner = new IterativeReweightedLeastSquares<LogisticRegression>()
{
    Tolerance = 1e-4,  // Let's set some convergence parameters
    Iterations = 100,  // maximum number of iterations to perform
    Regularization = 0
};

// Now, we use the learning algorithm to learn the distinction between the two:
LogisticRegression reg = learner.Learn(new[] { bow1, bow2 }, new[] { false, true });

// Finally, we can predict using the classifier:
bool c1 = reg.Decide(bow1); // Should be false
bool c2 = reg.Decide(bow2); // Should be true

// The Bag-Of-Words model can be used to extract finite-length feature 
// vectors from sequences of arbitrary length, like handwritten digits

// Ensure we get reproducible results
Accord.Math.Random.Generator.Seed = 0;

// Download the PENDIGITS dataset from UCI ML repository
var pendigits = new Pendigits(path: localDownloadPath);

// Get and pre-process the training set
double[][][] trainInputs = pendigits.Training.Item1;
int[] trainOutputs = pendigits.Training.Item2;

// Pre-process the digits so each of them is centered and scaled
trainInputs = trainInputs.Apply(Accord.Statistics.Tools.ZScores);

// Create a Bag-of-Words learning algorithm
var bow = new BagOfWords<double[], KMeans>()
{
    Clustering = new KMeans(5),
};

// Use the BoW to create a quantizer
var quantizer = bow.Learn(trainInputs);

// Extract vector representations from the pen sequences
double[][] trainVectors = quantizer.Transform(trainInputs);

// Create a new learning algorithm for support vector machines
var teacher = new MulticlassSupportVectorLearning<ChiSquare, double[]>
{
    Learner = (p) => new SequentialMinimalOptimization<ChiSquare, double[]>()
    {
        Complexity = 1
    }
};

// Use the learning algorithm to create a classifier
var svm = teacher.Learn(trainVectors, trainOutputs);

// Compute predictions for the training set
int[] trainPredicted = svm.Decide(trainVectors);

// Check the performance of the classifier by comparing with the ground-truth:
var m1 = new GeneralConfusionMatrix(predicted: trainPredicted, expected: trainOutputs);
double trainAcc = m1.Accuracy; // should be 0.690


// Prepare the testing set
double[][][] testInputs = pendigits.Testing.Item1;
int[] testOutputs = pendigits.Testing.Item2;

// Apply the same normalizations
testInputs = testInputs.Apply(Accord.Statistics.Tools.ZScores);

double[][] testVectors = quantizer.Transform(testInputs);

// Compute predictions for the test set
int[] testPredicted = svm.Decide(testVectors);

// Check the performance of the classifier by comparing with the ground-truth:
var m2 = new GeneralConfusionMatrix(predicted: testPredicted, expected: testOutputs);
double testAcc = m2.Accuracy; // should be 0.600