From 6e6b4e697d4c762190a0677399df376389e9dda2 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Fri, 5 Apr 2019 02:03:10 -0700 Subject: [PATCH 1/7] Add time series samples for stateful prediction engine. --- .../TimeSeries/DetectChangePointBySsa.cs | 99 +++++++++++------ .../DetectChangePointBySsaBatchPrediction.cs | 90 ++++++++++++++++ .../TimeSeries/DetectIidChangePoint.cs | 78 +++++++++++--- .../DetectIidChangePointBatchPrediction.cs | 86 +++++++++++++++ .../Transforms/TimeSeries/DetectIidSpike.cs | 66 ++++++++---- .../DetectIidSpikeBatchPrediction.cs | 77 +++++++++++++ .../Transforms/TimeSeries/DetectSpikeBySsa.cs | 101 +++++++++++------- .../DetectSpikeBySsaBatchPrediction.cs | 92 ++++++++++++++++ 8 files changed, 579 insertions(+), 110 deletions(-) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs index 0e14dd6f6c..6ce526e715 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.ML; using Microsoft.ML.Data; +using Microsoft.ML.Transforms.TimeSeries; namespace Samples.Dynamic { @@ -24,6 +26,7 @@ public SsaChangePointData(float value) } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // It demostrates stateful prediction engine that updates the state of the model and allows for saving/reloading. // The estimator is applied then to identify points where data distribution changed. // This estimator can account for temporal seasonality in the data. public static void Example() @@ -32,7 +35,7 @@ public static void Example() // as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern and then a change in trend + // Generate sample series data with a recurring pattern const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; @@ -40,52 +43,78 @@ public static void Example() for (int i = 0; i < TrainingSeasons; i++) for (int j = 0; j < SeasonalitySize; j++) data.Add(new SsaChangePointData(j)); - // This is a change point - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new SsaChangePointData(i * 100)); // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); - // Setup estimator arguments + // Setup SsaChangePointDetector arguments var inputColumnName = nameof(SsaChangePointData.Value); var outputColumnName = nameof(ChangePointPrediction.Prediction); - // The transformed data. - var transformedData = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + // Train the change point detector. + ITransformer model = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView); - // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Create a prediction engine from the model for feeding new data. + var engine = model.CreateTimeSeriesPredictionFunction(ml); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + // Start streaming new data points with no change point to the prediction engine. + Console.WriteLine($"Output from ChangePoint predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); - int k = 0; - foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - Console.WriteLine(""); + ChangePointPrediction prediction = null; + for (int i = 0; i < 5; i++) + { + var value = i; + prediction = engine.Predict(new SsaChangePointData(value)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Now stream data points that reflect a change in trend. + for (int i = 0; i < 5; i++) + { + var value = (i + 1) * 100; + prediction = engine.Predict(new SsaChangePointData(value)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Now we demonstrate saving and loading the model. + + // Save the model that exists within the prediction engine. + // The engine has been updating this model with every new data point. + var modelPath = "model.zip"; + engine.CheckPoint(ml, modelPath); + + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = ml.Model.Load(file, out DataViewSchema schema); + + // We must create a new prediction engine from the persisted model. + engine = model.CreateTimeSeriesPredictionFunction(ml); + + // Run predictions on the loaded model. + for (int i = 0; i < 5; i++) + { + var value = (i + 1) * 100; + prediction = engine.Predict(new SsaChangePointData(value)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } - // Prediction column obtained post-transformation. + // Output from ChangePoint predictions on new data: // Data Alert Score P-Value Martingale value - // 0 0 - 2.53 0.50 0.00 - // 1 0 - 0.01 0.01 0.00 - // 2 0 0.76 0.14 0.00 - // 3 0 0.69 0.28 0.00 - // 4 0 1.44 0.18 0.00 - // 0 0 - 1.84 0.17 0.00 - // 1 0 0.22 0.44 0.00 - // 2 0 0.20 0.45 0.00 - // 3 0 0.16 0.47 0.00 - // 4 0 1.33 0.18 0.00 - // 0 0 - 1.79 0.07 0.00 - // 1 0 0.16 0.50 0.00 - // 2 0 0.09 0.50 0.00 - // 3 0 0.08 0.45 0.00 - // 4 0 1.31 0.12 0.00 - // 0 0 - 1.79 0.07 0.00 - // 100 1 99.16 0.00 4031.94 <-- alert is on, predicted changepoint - // 200 0 185.23 0.00 731260.87 - // 300 0 270.40 0.01 3578470.47 - // 400 0 357.11 0.03 45298370.86 + // 0 0 - 1.01 0.50 0.00 + // 1 0 - 0.24 0.22 0.00 + // 2 0 - 0.31 0.30 0.00 + // 3 0 0.44 0.01 0.00 + // 4 0 2.16 0.00 0.24 + // 100 0 86.23 0.00 2076098.24 + // 200 0 171.38 0.00 809668524.21 + // 300 1 256.83 0.01 22130423541.93 <-- alert is on, note that delay is expected + // 400 0 326.55 0.04 241162710263.29 + // 500 0 364.82 0.08 597660527041.45 <-- saved to disk + // 100 0 - 58.58 0.15 1096021098844.34 <-- loaded from disk and running new predictions + // 200 0 - 41.24 0.20 97579154688.98 + // 300 0 - 30.61 0.24 95319753.87 + // 400 0 58.87 0.38 14.24 + // 500 0 219.28 0.36 0.05 } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs new file mode 100644 index 0000000000..350e8731ac --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.Samples.Dynamic +{ + public static class DetectChangePointBySsaBatchPrediction + { + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class SsaChangePointData + { + public float Value; + + public SsaChangePointData(float value) + { + Value = value; + } + } + + // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // The estimator is applied then to identify points where data distribution changed. + // This estimator can account for temporal seasonality in the data. + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a recurring pattern and then a change in trend + const int SeasonalitySize = 5; + const int TrainingSeasons = 3; + const int TrainingSize = SeasonalitySize * TrainingSeasons; + var data = new List(); + for (int i = 0; i < TrainingSeasons; i++) + for (int j = 0; j < SeasonalitySize; j++) + data.Add(new SsaChangePointData(j)); + // This is a change point + for (int i = 0; i < SeasonalitySize; i++) + data.Add(new SsaChangePointData(i * 100)); + + // Convert data to IDataView. + var dataView = ml.Data.LoadFromEnumerable(data); + + // Setup estimator arguments + var inputColumnName = nameof(SsaChangePointData.Value); + var outputColumnName = nameof(ChangePointPrediction.Prediction); + + // The transformed data. + var transformedData = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + + // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. + var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); + int k = 0; + foreach (var prediction in predictionColumn) + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + Console.WriteLine(""); + + // Prediction column obtained post-transformation. + // Data Alert Score P-Value Martingale value + // 0 0 - 2.53 0.50 0.00 + // 1 0 - 0.01 0.01 0.00 + // 2 0 0.76 0.14 0.00 + // 3 0 0.69 0.28 0.00 + // 4 0 1.44 0.18 0.00 + // 0 0 - 1.84 0.17 0.00 + // 1 0 0.22 0.44 0.00 + // 2 0 0.20 0.45 0.00 + // 3 0 0.16 0.47 0.00 + // 4 0 1.33 0.18 0.00 + // 0 0 - 1.79 0.07 0.00 + // 1 0 0.16 0.50 0.00 + // 2 0 0.09 0.50 0.00 + // 3 0 0.08 0.45 0.00 + // 4 0 1.31 0.12 0.00 + // 0 0 - 1.79 0.07 0.00 + // 100 1 99.16 0.00 4031.94 <-- alert is on, predicted changepoint + // 200 0 185.23 0.00 731260.87 + // 300 0 270.40 0.01 3578470.47 + // 400 0 357.11 0.03 45298370.86 + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs index 268c28238a..8d174b8abb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs @@ -4,8 +4,10 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.ML; using Microsoft.ML.Data; +using Microsoft.ML.Transforms.TimeSeries; namespace Samples.Dynamic { @@ -47,25 +49,63 @@ public static void Example() // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); - // Setup estimator arguments + // Setup IidSpikeDetector arguments string outputColumnName = nameof(ChangePointPrediction.Prediction); string inputColumnName = nameof(IidChangePointData.Value); - // The transformed data. - var transformedData = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + // Time Series model. + ITransformer model = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView); - // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Create a time series prediction engine from the model. + var engine = model.CreateTimeSeriesPredictionFunction(ml); + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Change point + var changePointPrediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, changePointPrediction.Prediction[0], + changePointPrediction.Prediction[1], changePointPrediction.Prediction[2], changePointPrediction.Prediction[3]); + + // Checkpoint the model. + var modelPath = "temp.zip"; + engine.CheckPoint(ml, modelPath); + + // Reference to current time series engine because in the next step "engine" will point to the + // checkpointed model being loaded from disk. + var timeseries1 = engine; - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); - Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); - int k = 0; - foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - Console.WriteLine(""); + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = ml.Model.Load(file, out DataViewSchema schema); + + // Create a time series prediction engine from the checkpointed model. + engine = model.CreateTimeSeriesPredictionFunction(ml); + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Prediction from the original time series engine should match the prediction from + // check pointed model. + engine = timeseries1; + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } - // Prediction column obtained post-transformation. // Data Alert Score P-Value Martingale value + // 5 0 5.00 0.50 0.00 <-- Time Series 1. // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 @@ -73,9 +113,17 @@ public static void Example() // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint - // 7 0 7.00 0.13 33950.16 + // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed). + + // 7 0 7.00 0.13 33950.16 <-- Time Series 2 : Model loaded back from disk and prediction is made. + // 7 0 7.00 0.26 60866.34 + // 7 0 7.00 0.38 78362.04 + // 7 0 7.00 0.50 0.01 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + + // 7 0 7.00 0.13 33950.16 <-- Time Series 1 and prediction is made. // 7 0 7.00 0.26 60866.34 // 7 0 7.00 0.38 78362.04 // 7 0 7.00 0.50 0.01 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs new file mode 100644 index 0000000000..88893190be --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.Samples.Dynamic +{ + public static class DetectIidChangePointBatchPrediction + { + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class IidChangePointData + { + public float Value; + + public IidChangePointData(float value) + { + Value = value; + } + } + + // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // The estimator is applied then to identify points where data distribution changed. + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a change + const int Size = 16; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) + data.Add(new IidChangePointData(5)); + // This is a change point + for (int i = 0; i < Size / 2; i++) + data.Add(new IidChangePointData(7)); + + // Convert data to IDataView. + var dataView = ml.Data.LoadFromEnumerable(data); + + // Setup estimator arguments + string outputColumnName = nameof(ChangePointPrediction.Prediction); + string inputColumnName = nameof(IidChangePointData.Value); + + // The transformed data. + var transformedData = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + + // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. + var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); + int k = 0; + foreach (var prediction in predictionColumn) + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + Console.WriteLine(""); + + // Prediction column obtained post-transformation. + // Data Alert Score P-Value Martingale value + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint + // 7 0 7.00 0.13 33950.16 + // 7 0 7.00 0.26 60866.34 + // 7 0 7.00 0.38 78362.04 + // 7 0 7.00 0.50 0.01 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs index 862c390ac7..e84ac3ad1b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs @@ -46,35 +46,55 @@ public static void Example() // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); - // Setup the estimator arguments + // Setup IidSpikeDetector arguments string outputColumnName = nameof(IidSpikePrediction.Prediction); string inputColumnName = nameof(IidSpikeData.Value); + // The transformed model. + ITransformer model = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size).Fit(dataView); - // The transformed data. - var transformedData = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + // Create a time series prediction engine from the model. + var engine = model.CreateTimeSeriesPredictionFunction(ml); + for (int index = 0; index < 5; index++) + { + // Anomaly spike detection. + var prediction = engine.Predict(new IidSpikeData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + } + + // Spike. + var spikePrediction = engine.Predict(new IidSpikeData(10)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 10, spikePrediction.Prediction[0], + spikePrediction.Prediction[1], spikePrediction.Prediction[2]); - // Getting the data of the newly created column as an IEnumerable of IidSpikePrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Checkpoint the model. + var modelPath = "temp.zip"; + engine.CheckPoint(ml, modelPath); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); - Console.WriteLine("Alert\tScore\tP-Value"); - foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); - Console.WriteLine(""); + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = ml.Model.Load(file, out DataViewSchema schema); + + for (int index = 0; index < 5; index++) + { + // Anomaly spike detection. + var prediction = engine.Predict(new IidSpikeData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + } - // Prediction column obtained post-transformation. - // Alert Score P-Value - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 1 10.00 0.00 <-- alert is on, predicted spike - // 0 5.00 0.26 - // 0 5.00 0.26 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 + // Data Alert Score P-Value + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model) + // 5 0 5.00 0.26 <-- load model from disk. + // 5 0 5.00 0.26 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs new file mode 100644 index 0000000000..971e5c741d --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.Samples.Dynamic +{ + public static class DetectIidSpikeBatchPrediction + { + class IidSpikeData + { + public float Value; + + public IidSpikeData(float value) + { + Value = value; + } + } + + class IidSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } + } + + // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // The estimator is applied then to identify spiking points in the series. + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a spike + const int Size = 10; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) + data.Add(new IidSpikeData(5)); + // This is a spike + data.Add(new IidSpikeData(10)); + for (int i = 0; i < Size / 2; i++) + data.Add(new IidSpikeData(5)); + + // Convert data to IDataView. + var dataView = ml.Data.LoadFromEnumerable(data); + + // Setup the estimator arguments + string outputColumnName = nameof(IidSpikePrediction.Prediction); + string inputColumnName = nameof(IidSpikeData.Value); + + // The transformed data. + var transformedData = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + + // Getting the data of the newly created column as an IEnumerable of IidSpikePrediction. + var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Alert\tScore\tP-Value"); + foreach (var prediction in predictionColumn) + Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + Console.WriteLine(""); + + // Prediction column obtained post-transformation. + // Alert Score P-Value + // 0 5.00 0.50 + // 0 5.00 0.50 + // 0 5.00 0.50 + // 0 5.00 0.50 + // 0 5.00 0.50 + // 1 10.00 0.00 <-- alert is on, predicted spike + // 0 5.00 0.26 + // 0 5.00 0.26 + // 0 5.00 0.50 + // 0 5.00 0.50 + // 0 5.00 0.50 + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs index c7293052f2..0381eeaffd 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.ML; using Microsoft.ML.Data; +using Microsoft.ML.Transforms.TimeSeries; namespace Samples.Dynamic { @@ -32,7 +34,7 @@ public static void Example() // as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern and a spike within the pattern + // Generate sample series data with a recurring pattern const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; @@ -40,54 +42,79 @@ public static void Example() for (int i = 0; i < TrainingSeasons; i++) for (int j = 0; j < SeasonalitySize; j++) data.Add(new SsaSpikeData(j)); - // This is a spike - data.Add(new SsaSpikeData(100)); - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new SsaSpikeData(i)); // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); - // Setup estimator arguments + // Setup IidSpikeDetector arguments var inputColumnName = nameof(SsaSpikeData.Value); var outputColumnName = nameof(SsaSpikePrediction.Prediction); - // The transformed data. - var transformedData = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + // Train the change point detector. + ITransformer model = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView); - // Getting the data of the newly created column as an IEnumerable of SsaSpikePrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Create a prediction engine from the model for feeding new data. + var engine = model.CreateTimeSeriesPredictionFunction(ml); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + // Start streaming new data points with no change point to the prediction engine. + Console.WriteLine($"Output from spike predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value"); - int k = 0; - foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); - Console.WriteLine(""); + SsaSpikePrediction prediction = null; + for (int j = 0; j < 2; j++) + { + for (int i = 0; i < 5; i++) + { + var value = i; + prediction = engine.Predict(new SsaSpikeData(value)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + } + } + + // Now send a data point that reflects a spike. + var newValue = 100; + prediction = engine.Predict(new SsaSpikeData(newValue)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", newValue, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + + // Now we demonstrate saving and loading the model. + + // Save the model that exists within the prediction engine. + // The engine has been updating this model with every new data point. + var modelPath = "model.zip"; + engine.CheckPoint(ml, modelPath); + + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = ml.Model.Load(file, out DataViewSchema schema); + + // We must create a new prediction engine from the persisted model. + engine = model.CreateTimeSeriesPredictionFunction(ml); + + // Run predictions on the loaded model. + for (int i = 0; i < 5; i++) + { + var value = i; + prediction = engine.Predict(new SsaSpikeData(value)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + } - // Prediction column obtained post-transformation. + // Output from spike predictions on new data: // Data Alert Score P-Value - // 0 0 - 2.53 0.50 - // 1 0 - 0.01 0.01 - // 2 0 0.76 0.14 - // 3 0 0.69 0.28 - // 4 0 1.44 0.18 - // 0 0 - 1.84 0.17 - // 1 0 0.22 0.44 - // 2 0 0.20 0.45 - // 3 0 0.16 0.47 - // 4 0 1.33 0.18 - // 0 0 - 1.79 0.07 - // 1 0 0.16 0.50 - // 2 0 0.09 0.50 - // 3 0 0.08 0.45 - // 4 0 1.31 0.12 - // 100 1 98.21 0.00 <-- alert is on, predicted spike - // 0 0 - 13.83 0.29 - // 1 0 - 1.74 0.44 - // 2 0 - 0.47 0.46 - // 3 0 - 16.50 0.29 - // 4 0 - 29.82 0.21 + // 0 0 - 1.01 0.50 + // 1 0 - 0.24 0.22 + // 2 0 - 0.31 0.30 + // 3 0 0.44 0.01 + // 4 0 2.16 0.00 + // 0 0 - 0.78 0.27 + // 1 0 - 0.80 0.30 + // 2 0 - 0.84 0.31 + // 3 0 0.33 0.31 + // 4 0 2.21 0.07 + // 100 1 86.17 0.00 <-- alert is on, predicted spike + // 0 0 - 2.74 0.40 <-- saved to disk, re-loaded, and running new predictions + // 1 0 - 1.47 0.42 + // 2 0 - 17.50 0.24 + // 3 0 - 30.82 0.16 + // 4 0 - 23.24 0.28 } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs new file mode 100644 index 0000000000..c0be801c86 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.Samples.Dynamic +{ + public static class DetectSpikeBySsaBatchPrediction + { + class SsaSpikeData + { + public float Value; + + public SsaSpikeData(float value) + { + Value = value; + } + } + + class SsaSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } + } + + // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // The estimator is applied then to identify spiking points in the series. + // This estimator can account for temporal seasonality in the data. + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a recurring pattern and a spike within the pattern + const int SeasonalitySize = 5; + const int TrainingSeasons = 3; + const int TrainingSize = SeasonalitySize * TrainingSeasons; + var data = new List(); + for (int i = 0; i < TrainingSeasons; i++) + for (int j = 0; j < SeasonalitySize; j++) + data.Add(new SsaSpikeData(j)); + // This is a spike + data.Add(new SsaSpikeData(100)); + for (int i = 0; i < SeasonalitySize; i++) + data.Add(new SsaSpikeData(i)); + + // Convert data to IDataView. + var dataView = ml.Data.LoadFromEnumerable(data); + + // Setup estimator arguments + var inputColumnName = nameof(SsaSpikeData.Value); + var outputColumnName = nameof(SsaSpikePrediction.Prediction); + + // The transformed data. + var transformedData = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + + // Getting the data of the newly created column as an IEnumerable of SsaSpikePrediction. + var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Data\tAlert\tScore\tP-Value"); + int k = 0; + foreach (var prediction in predictionColumn) + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + Console.WriteLine(""); + + // Prediction column obtained post-transformation. + // Data Alert Score P-Value + // 0 0 - 2.53 0.50 + // 1 0 - 0.01 0.01 + // 2 0 0.76 0.14 + // 3 0 0.69 0.28 + // 4 0 1.44 0.18 + // 0 0 - 1.84 0.17 + // 1 0 0.22 0.44 + // 2 0 0.20 0.45 + // 3 0 0.16 0.47 + // 4 0 1.33 0.18 + // 0 0 - 1.79 0.07 + // 1 0 0.16 0.50 + // 2 0 0.09 0.50 + // 3 0 0.08 0.45 + // 4 0 1.31 0.12 + // 100 1 98.21 0.00 <-- alert is on, predicted spike + // 0 0 - 13.83 0.29 + // 1 0 - 1.74 0.44 + // 2 0 - 0.47 0.46 + // 3 0 - 16.50 0.29 + // 4 0 - 29.82 0.21 + } + } +} From 28de467062b6818d6df4e533106c48d052b2bb9f Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Fri, 5 Apr 2019 09:58:07 -0700 Subject: [PATCH 2/7] PR feedback. --- src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 8 ++++---- src/Microsoft.ML.TimeSeries/PredictionFunction.cs | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index aa64de5495..681d53fe40 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -25,7 +25,7 @@ public static class TimeSeriesCatalog /// /// /// /// /// @@ -47,7 +47,7 @@ public static IidChangePointEstimator DetectIidChangePoint(this TransformsCatalo /// /// /// /// /// @@ -73,7 +73,7 @@ public static IidSpikeEstimator DetectIidSpike(this TransformsCatalog catalog, s /// /// /// /// /// @@ -110,7 +110,7 @@ public static SsaChangePointEstimator DetectChangePointBySsa(this TransformsCata /// /// /// /// /// diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 070e0c3d10..186d96da7b 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -65,6 +65,13 @@ public sealed class TimeSeriesPredictionFunction : PredictionEngineB /// /// Usually . /// Path to file on disk where the updated model needs to be saved. + /// + /// + /// + /// + /// public void CheckPoint(IHostEnvironment env, string modelPath) { using (var file = File.Create(modelPath)) From c668e075b2e43186f227922ecbd4f4103e6306d7 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Tue, 9 Apr 2019 14:39:25 -0700 Subject: [PATCH 3/7] PR feedback. --- .../TimeSeries/DetectChangePointBySsaBatchPrediction.cs | 2 +- .../Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs | 2 ++ .../Dynamic/Transforms/TimeSeries/DetectIidSpike.cs | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs index 350e8731ac..a08d4f6493 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using Microsoft.ML.Data; -namespace Microsoft.ML.Samples.Dynamic +namespace Microsoft.Samples.Dynamic { public static class DetectChangePointBySsaBatchPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs index 8d174b8abb..284c7fdff9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs @@ -58,6 +58,8 @@ public static void Example() // Create a time series prediction engine from the model. var engine = model.CreateTimeSeriesPredictionFunction(ml); + + // Create non-anomalous data and check for change point. for (int index = 0; index < 8; index++) { // Anomaly change point detection. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs index e84ac3ad1b..da09eeefe4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs @@ -54,6 +54,8 @@ public static void Example() // Create a time series prediction engine from the model. var engine = model.CreateTimeSeriesPredictionFunction(ml); + + // Create non-anomalous data and check for anomaly. for (int index = 0; index < 5; index++) { // Anomaly spike detection. From f81d115b58c9d76396fde8876a669442fd7fb218 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Wed, 10 Apr 2019 13:45:04 -0700 Subject: [PATCH 4/7] PR feedback. --- .../TimeSeries/DetectChangePointBySsa.cs | 116 +++++++++------- .../DetectChangePointBySsaBatchPrediction.cs | 88 +++++++----- .../TimeSeries/DetectIidChangePoint.cs | 127 ++++++++++-------- .../DetectIidChangePointBatchPrediction.cs | 70 ++++++---- .../Transforms/TimeSeries/DetectIidSpike.cs | 98 ++++++++------ .../DetectIidSpikeBatchPrediction.cs | 95 +++++++------ .../Transforms/TimeSeries/DetectSpikeBySsa.cs | 121 +++++++++-------- .../DetectSpikeBySsaBatchPrediction.cs | 99 +++++++++----- .../PredictionFunction.cs | 4 +- 9 files changed, 484 insertions(+), 334 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs index 6ce526e715..66da84080a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs @@ -9,21 +9,6 @@ namespace Samples.Dynamic { public static class DetectChangePointBySsa { - class ChangePointPrediction - { - [VectorType(4)] - public double[] Prediction { get; set; } - } - - class SsaChangePointData - { - public float Value; - - public SsaChangePointData(float value) - { - Value = value; - } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // It demostrates stateful prediction engine that updates the state of the model and allows for saving/reloading. @@ -39,42 +24,67 @@ public static void Example() const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; - var data = new List(); - for (int i = 0; i < TrainingSeasons; i++) - for (int j = 0; j < SeasonalitySize; j++) - data.Add(new SsaChangePointData(j)); + var data = new List() + { + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup SsaChangePointDetector arguments - var inputColumnName = nameof(SsaChangePointData.Value); + var inputColumnName = nameof(TimeSeriesData.Value); var outputColumnName = nameof(ChangePointPrediction.Prediction); // Train the change point detector. ITransformer model = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView); // Create a prediction engine from the model for feeding new data. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesPredictionFunction(ml); // Start streaming new data points with no change point to the prediction engine. Console.WriteLine($"Output from ChangePoint predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); - ChangePointPrediction prediction = null; + + // Output from ChangePoint predictions on new data: + // Data Alert Score P-Value Martingale value + for (int i = 0; i < 5; i++) - { - var value = i; - prediction = engine.Predict(new SsaChangePointData(value)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - } + PrintPrediction(i, engine.Predict(new TimeSeriesData(i))); + + // 0 0 -1.01 0.50 0.00 + // 1 0 -0.24 0.22 0.00 + // 2 0 -0.31 0.30 0.00 + // 3 0 0.44 0.01 0.00 + // 4 0 2.16 0.00 0.24 // Now stream data points that reflect a change in trend. for (int i = 0; i < 5; i++) { - var value = (i + 1) * 100; - prediction = engine.Predict(new SsaChangePointData(value)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + int value = (i + 1) * 100; + PrintPrediction(value, engine.Predict(new TimeSeriesData(value))); } + // 100 0 86.23 0.00 2076098.24 + // 200 0 171.38 0.00 809668524.21 + // 300 1 256.83 0.01 22130423541.93 <-- alert is on, note that delay is expected + // 400 0 326.55 0.04 241162710263.29 + // 500 0 364.82 0.08 597660527041.45 <-- saved to disk // Now we demonstrate saving and loading the model. @@ -88,33 +98,41 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // We must create a new prediction engine from the persisted model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesPredictionFunction(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) { - var value = (i + 1) * 100; - prediction = engine.Predict(new SsaChangePointData(value)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + int value = (i + 1) * 100; + PrintPrediction(value, engine.Predict(new TimeSeriesData(value))); } - // Output from ChangePoint predictions on new data: - // Data Alert Score P-Value Martingale value - // 0 0 - 1.01 0.50 0.00 - // 1 0 - 0.24 0.22 0.00 - // 2 0 - 0.31 0.30 0.00 - // 3 0 0.44 0.01 0.00 - // 4 0 2.16 0.00 0.24 - // 100 0 86.23 0.00 2076098.24 - // 200 0 171.38 0.00 809668524.21 - // 300 1 256.83 0.01 22130423541.93 <-- alert is on, note that delay is expected - // 400 0 326.55 0.04 241162710263.29 - // 500 0 364.82 0.08 597660527041.45 <-- saved to disk - // 100 0 - 58.58 0.15 1096021098844.34 <-- loaded from disk and running new predictions - // 200 0 - 41.24 0.20 97579154688.98 - // 300 0 - 30.61 0.24 95319753.87 + // 100 0 -58.58 0.15 1096021098844.34 <-- loaded from disk and running new predictions + // 200 0 -41.24 0.20 97579154688.98 + // 300 0 -30.61 0.24 95319753.87 // 400 0 58.87 0.38 14.24 // 500 0 219.28 0.36 0.05 + + } + + private static void PrintPrediction(float value, ChangePointPrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs index a08d4f6493..2e0df19007 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs @@ -1,27 +1,12 @@ using System; using System.Collections.Generic; +using Microsoft.ML; using Microsoft.ML.Data; -namespace Microsoft.Samples.Dynamic +namespace Samples.Dynamic { public static class DetectChangePointBySsaBatchPrediction { - class ChangePointPrediction - { - [VectorType(4)] - public double[] Prediction { get; set; } - } - - class SsaChangePointData - { - public float Value; - - public SsaChangePointData(float value) - { - Value = value; - } - } - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify points where data distribution changed. // This estimator can account for temporal seasonality in the data. @@ -35,19 +20,39 @@ public static void Example() const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; - var data = new List(); - for (int i = 0; i < TrainingSeasons; i++) - for (int j = 0; j < SeasonalitySize; j++) - data.Add(new SsaChangePointData(j)); - // This is a change point - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new SsaChangePointData(i * 100)); + var data = new List() + { + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + //This is a change point + new TimeSeriesData(0), + new TimeSeriesData(100), + new TimeSeriesData(200), + new TimeSeriesData(300), + new TimeSeriesData(400), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup estimator arguments - var inputColumnName = nameof(SsaChangePointData.Value); + var inputColumnName = nameof(TimeSeriesData.Value); var outputColumnName = nameof(ChangePointPrediction.Prediction); // The transformed data. @@ -60,31 +65,50 @@ public static void Example() Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); int k = 0; foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - Console.WriteLine(""); + PrintPrediction(data[k++].Value, prediction); // Prediction column obtained post-transformation. // Data Alert Score P-Value Martingale value - // 0 0 - 2.53 0.50 0.00 - // 1 0 - 0.01 0.01 0.00 + // 0 0 -2.53 0.50 0.00 + // 1 0 -0.01 0.01 0.00 // 2 0 0.76 0.14 0.00 // 3 0 0.69 0.28 0.00 // 4 0 1.44 0.18 0.00 - // 0 0 - 1.84 0.17 0.00 + // 0 0 -1.84 0.17 0.00 // 1 0 0.22 0.44 0.00 // 2 0 0.20 0.45 0.00 // 3 0 0.16 0.47 0.00 // 4 0 1.33 0.18 0.00 - // 0 0 - 1.79 0.07 0.00 + // 0 0 -1.79 0.07 0.00 // 1 0 0.16 0.50 0.00 // 2 0 0.09 0.50 0.00 // 3 0 0.08 0.45 0.00 // 4 0 1.31 0.12 0.00 - // 0 0 - 1.79 0.07 0.00 + // 0 0 -1.79 0.07 0.00 // 100 1 99.16 0.00 4031.94 <-- alert is on, predicted changepoint // 200 0 185.23 0.00 731260.87 // 300 0 270.40 0.01 3578470.47 // 400 0 357.11 0.03 45298370.86 } + + private static void PrintPrediction(float value, ChangePointPrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs index 284c7fdff9..e7292dbb3c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs @@ -13,21 +13,6 @@ namespace Samples.Dynamic { public static class DetectIidChangePoint { - class ChangePointPrediction - { - [VectorType(4)] - public double[] Prediction { get; set; } - } - - class IidChangePointData - { - public float Value; - - public IidChangePointData(float value) - { - Value = value; - } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify points where data distribution changed. @@ -39,39 +24,66 @@ public static void Example() // Generate sample series data with a change const int Size = 16; - var data = new List(Size); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidChangePointData(5)); - // This is a change point - for (int i = 0; i < Size / 2; i++) - data.Add(new IidChangePointData(7)); + var data = new List(Size) + { + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + + //Change point data. + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup IidSpikeDetector arguments string outputColumnName = nameof(ChangePointPrediction.Prediction); - string inputColumnName = nameof(IidChangePointData.Value); + string inputColumnName = nameof(TimeSeriesData.Value); // Time Series model. ITransformer model = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView); // Create a time series prediction engine from the model. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesPredictionFunction(ml); + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); + + // Data Alert Score P-Value Martingale value + // Create non-anomalous data and check for change point. for (int index = 0; index < 8; index++) { // Anomaly change point detection. - var prediction = engine.Predict(new IidChangePointData(5)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 5, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + PrintPrediction(5, engine.Predict(new TimeSeriesData(5))); } + // 5 0 5.00 0.50 0.00 <-- Time Series 1. + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // Change point - var changePointPrediction = engine.Predict(new IidChangePointData(7)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, changePointPrediction.Prediction[0], - changePointPrediction.Prediction[1], changePointPrediction.Prediction[2], changePointPrediction.Prediction[3]); + PrintPrediction(7, engine.Predict(new TimeSeriesData(7))); + + // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed). // Checkpoint the model. var modelPath = "temp.zip"; @@ -86,37 +98,13 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // Create a time series prediction engine from the checkpointed model. - engine = model.CreateTimeSeriesPredictionFunction(ml); - for (int index = 0; index < 8; index++) - { - // Anomaly change point detection. - var prediction = engine.Predict(new IidChangePointData(7)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - } - - // Prediction from the original time series engine should match the prediction from - // check pointed model. - engine = timeseries1; + engine = model.CreateTimeSeriesPredictionFunction(ml); for (int index = 0; index < 8; index++) { // Anomaly change point detection. - var prediction = engine.Predict(new IidChangePointData(7)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + PrintPrediction(7, engine.Predict(new TimeSeriesData(7))); } - // Data Alert Score P-Value Martingale value - // 5 0 5.00 0.50 0.00 <-- Time Series 1. - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 5 0 5.00 0.50 0.00 - // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed). - // 7 0 7.00 0.13 33950.16 <-- Time Series 2 : Model loaded back from disk and prediction is made. // 7 0 7.00 0.26 60866.34 // 7 0 7.00 0.38 78362.04 @@ -125,6 +113,15 @@ public static void Example() // 7 0 7.00 0.50 0.00 // 7 0 7.00 0.50 0.00 + // Prediction from the original time series engine should match the prediction from + // check pointed model. + engine = timeseries1; + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + PrintPrediction(7, engine.Predict(new TimeSeriesData(7))); + } + // 7 0 7.00 0.13 33950.16 <-- Time Series 1 and prediction is made. // 7 0 7.00 0.26 60866.34 // 7 0 7.00 0.38 78362.04 @@ -133,5 +130,25 @@ public static void Example() // 7 0 7.00 0.50 0.00 // 7 0 7.00 0.50 0.00 } + + private static void PrintPrediction(float value, ChangePointPrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs index 88893190be..477662d06a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs @@ -4,27 +4,13 @@ using System; using System.Collections.Generic; +using Microsoft.ML; using Microsoft.ML.Data; -namespace Microsoft.ML.Samples.Dynamic +namespace Samples.Dynamic { public static class DetectIidChangePointBatchPrediction { - class ChangePointPrediction - { - [VectorType(4)] - public double[] Prediction { get; set; } - } - - class IidChangePointData - { - public float Value; - - public IidChangePointData(float value) - { - Value = value; - } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify points where data distribution changed. @@ -36,19 +22,34 @@ public static void Example() // Generate sample series data with a change const int Size = 16; - var data = new List(Size); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidChangePointData(5)); - // This is a change point - for (int i = 0; i < Size / 2; i++) - data.Add(new IidChangePointData(7)); + var data = new List(Size) + { + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + + //Change point data. + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + new TimeSeriesData(7), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup estimator arguments string outputColumnName = nameof(ChangePointPrediction.Prediction); - string inputColumnName = nameof(IidChangePointData.Value); + string inputColumnName = nameof(TimeSeriesData.Value); // The transformed data. var transformedData = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); @@ -60,8 +61,7 @@ public static void Example() Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); int k = 0; foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); - Console.WriteLine(""); + PrintPrediction(data[k++].Value, prediction); // Prediction column obtained post-transformation. // Data Alert Score P-Value Martingale value @@ -82,5 +82,25 @@ public static void Example() // 7 0 7.00 0.50 0.00 // 7 0 7.00 0.50 0.00 } + + private static void PrintPrediction(float value, ChangePointPrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + + class ChangePointPrediction + { + [VectorType(4)] + public double[] Prediction { get; set; } + } + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs index da09eeefe4..ba1a18b999 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs @@ -9,21 +9,6 @@ namespace Samples.Dynamic { public static class DetectIidSpike { - class IidSpikeData - { - public float Value; - - public IidSpikeData(float value) - { - Value = value; - } - } - - class IidSpikePrediction - { - [VectorType(3)] - public double[] Prediction { get; set; } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. @@ -35,39 +20,60 @@ public static void Example() // Generate sample series data with a spike const int Size = 10; - var data = new List(Size); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidSpikeData(5)); - // This is a spike - data.Add(new IidSpikeData(10)); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidSpikeData(5)); + var data = new List(Size + 1) + { + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + + // This is a spike. + new TimeSeriesData(10), + + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup IidSpikeDetector arguments string outputColumnName = nameof(IidSpikePrediction.Prediction); - string inputColumnName = nameof(IidSpikeData.Value); + string inputColumnName = nameof(TimeSeriesData.Value); + // The transformed model. ITransformer model = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size).Fit(dataView); // Create a time series prediction engine from the model. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesPredictionFunction(ml); + + Console.WriteLine($"{outputColumnName} column obtained post-transformation."); + Console.WriteLine("Data\tAlert\tScore\tP-Value"); + + // Prediction column obtained post-transformation. + // Data Alert Score P-Value // Create non-anomalous data and check for anomaly. for (int index = 0; index < 5; index++) { // Anomaly spike detection. - var prediction = engine.Predict(new IidSpikeData(5)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + PrintPrediction(5, engine.Predict(new TimeSeriesData(5))); } + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // Spike. - var spikePrediction = engine.Predict(new IidSpikeData(10)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 10, spikePrediction.Prediction[0], - spikePrediction.Prediction[1], spikePrediction.Prediction[2]); + PrintPrediction(10, engine.Predict(new TimeSeriesData(10))); + + // 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model) // Checkpoint the model. var modelPath = "temp.zip"; @@ -80,23 +86,35 @@ public static void Example() for (int index = 0; index < 5; index++) { // Anomaly spike detection. - var prediction = engine.Predict(new IidSpikeData(5)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + PrintPrediction(5, engine.Predict(new TimeSeriesData(5))); } - // Data Alert Score P-Value - // 5 0 5.00 0.50 - // 5 0 5.00 0.50 - // 5 0 5.00 0.50 - // 5 0 5.00 0.50 - // 5 0 5.00 0.50 - // 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model) // 5 0 5.00 0.26 <-- load model from disk. // 5 0 5.00 0.26 // 5 0 5.00 0.50 // 5 0 5.00 0.50 // 5 0 5.00 0.50 + + } + + private static void PrintPrediction(float value, IidSpikePrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } + + class IidSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs index 971e5c741d..725c701c49 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs @@ -1,26 +1,12 @@ using System; using System.Collections.Generic; +using Microsoft.ML; using Microsoft.ML.Data; -namespace Microsoft.ML.Samples.Dynamic +namespace Samples.Dynamic { public static class DetectIidSpikeBatchPrediction { - class IidSpikeData - { - public float Value; - - public IidSpikeData(float value) - { - Value = value; - } - } - - class IidSpikePrediction - { - [VectorType(3)] - public double[] Prediction { get; set; } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. @@ -32,20 +18,30 @@ public static void Example() // Generate sample series data with a spike const int Size = 10; - var data = new List(Size); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidSpikeData(5)); - // This is a spike - data.Add(new IidSpikeData(10)); - for (int i = 0; i < Size / 2; i++) - data.Add(new IidSpikeData(5)); + var data = new List(Size + 1) + { + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + + // This is a spike. + new TimeSeriesData(10), + + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + new TimeSeriesData(5), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup the estimator arguments string outputColumnName = nameof(IidSpikePrediction.Prediction); - string inputColumnName = nameof(IidSpikeData.Value); + string inputColumnName = nameof(TimeSeriesData.Value); // The transformed data. var transformedData = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); @@ -54,24 +50,45 @@ public static void Example() var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); Console.WriteLine($"{outputColumnName} column obtained post-transformation."); - Console.WriteLine("Alert\tScore\tP-Value"); + Console.WriteLine("Data\tAlert\tScore\tP-Value"); + + int k = 0; foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); - Console.WriteLine(""); + PrintPrediction(data[k++].Value, prediction); // Prediction column obtained post-transformation. - // Alert Score P-Value - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 1 10.00 0.00 <-- alert is on, predicted spike - // 0 5.00 0.26 - // 0 5.00 0.26 - // 0 5.00 0.50 - // 0 5.00 0.50 - // 0 5.00 0.50 + // Data Alert Score P-Value + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 10 1 10.00 0.00 <-- alert is on, predicted spike + // 5 0 5.00 0.26 + // 5 0 5.00 0.26 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + } + + private static void PrintPrediction(float value, IidSpikePrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } + + class IidSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs index 0381eeaffd..d1dc929528 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs @@ -9,21 +9,6 @@ namespace Samples.Dynamic { public static class DetectSpikeBySsa { - class SsaSpikeData - { - public float Value; - - public SsaSpikeData(float value) - { - Value = value; - } - } - - class SsaSpikePrediction - { - [VectorType(3)] - public double[] Prediction { get; set; } - } // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. @@ -38,45 +23,68 @@ public static void Example() const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; - var data = new List(); - for (int i = 0; i < TrainingSeasons; i++) - for (int j = 0; j < SeasonalitySize; j++) - data.Add(new SsaSpikeData(j)); + var data = new List() + { + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup IidSpikeDetector arguments - var inputColumnName = nameof(SsaSpikeData.Value); + var inputColumnName = nameof(TimeSeriesData.Value); var outputColumnName = nameof(SsaSpikePrediction.Prediction); // Train the change point detector. ITransformer model = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView); // Create a prediction engine from the model for feeding new data. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesPredictionFunction(ml); // Start streaming new data points with no change point to the prediction engine. Console.WriteLine($"Output from spike predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value"); - SsaSpikePrediction prediction = null; + + // Output from spike predictions on new data: + // Data Alert Score P-Value + for (int j = 0; j < 2; j++) - { for (int i = 0; i < 5; i++) - { - var value = i; - prediction = engine.Predict(new SsaSpikeData(value)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); - } - } + PrintPrediction(i, engine.Predict(new TimeSeriesData(i))); + + // 0 0 -1.01 0.50 + // 1 0 -0.24 0.22 + // 2 0 -0.31 0.30 + // 3 0 0.44 0.01 + // 4 0 2.16 0.00 + // 0 0 -0.78 0.27 + // 1 0 -0.80 0.30 + // 2 0 -0.84 0.31 + // 3 0 0.33 0.31 + // 4 0 2.21 0.07 // Now send a data point that reflects a spike. - var newValue = 100; - prediction = engine.Predict(new SsaSpikeData(newValue)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", newValue, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + PrintPrediction(100, engine.Predict(new TimeSeriesData(100))); - // Now we demonstrate saving and loading the model. + // 100 1 86.17 0.00 <-- alert is on, predicted spike + // Now we demonstrate saving and loading the model. // Save the model that exists within the prediction engine. // The engine has been updating this model with every new data point. var modelPath = "model.zip"; @@ -87,34 +95,37 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // We must create a new prediction engine from the persisted model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesPredictionFunction(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) + PrintPrediction(i, engine.Predict(new TimeSeriesData(i))); + + // 0 0 -2.74 0.40 <-- saved to disk, re-loaded, and running new predictions + // 1 0 -1.47 0.42 + // 2 0 -17.50 0.24 + // 3 0 -30.82 0.16 + // 4 0 -23.24 0.28 + } + + private static void PrintPrediction(float value, SsaSpikePrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) { - var value = i; - prediction = engine.Predict(new SsaSpikeData(value)); - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); + Value = value; } + } - // Output from spike predictions on new data: - // Data Alert Score P-Value - // 0 0 - 1.01 0.50 - // 1 0 - 0.24 0.22 - // 2 0 - 0.31 0.30 - // 3 0 0.44 0.01 - // 4 0 2.16 0.00 - // 0 0 - 0.78 0.27 - // 1 0 - 0.80 0.30 - // 2 0 - 0.84 0.31 - // 3 0 0.33 0.31 - // 4 0 2.21 0.07 - // 100 1 86.17 0.00 <-- alert is on, predicted spike - // 0 0 - 2.74 0.40 <-- saved to disk, re-loaded, and running new predictions - // 1 0 - 1.47 0.42 - // 2 0 - 17.50 0.24 - // 3 0 - 30.82 0.16 - // 4 0 - 23.24 0.28 + class SsaSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs index c0be801c86..cc6a798dcf 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs @@ -1,27 +1,12 @@ using System; using System.Collections.Generic; +using Microsoft.ML; using Microsoft.ML.Data; -namespace Microsoft.ML.Samples.Dynamic +namespace Samples.Dynamic { public static class DetectSpikeBySsaBatchPrediction { - class SsaSpikeData - { - public float Value; - - public SsaSpikeData(float value) - { - Value = value; - } - } - - class SsaSpikePrediction - { - [VectorType(3)] - public double[] Prediction { get; set; } - } - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. // This estimator can account for temporal seasonality in the data. @@ -35,20 +20,41 @@ public static void Example() const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; - var data = new List(); - for (int i = 0; i < TrainingSeasons; i++) - for (int j = 0; j < SeasonalitySize; j++) - data.Add(new SsaSpikeData(j)); - // This is a spike - data.Add(new SsaSpikeData(100)); - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new SsaSpikeData(i)); + var data = new List() + { + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + + //This is a spike. + new TimeSeriesData(100), + + new TimeSeriesData(0), + new TimeSeriesData(1), + new TimeSeriesData(2), + new TimeSeriesData(3), + new TimeSeriesData(4), + }; // Convert data to IDataView. var dataView = ml.Data.LoadFromEnumerable(data); // Setup estimator arguments - var inputColumnName = nameof(SsaSpikeData.Value); + var inputColumnName = nameof(TimeSeriesData.Value); var outputColumnName = nameof(SsaSpikePrediction.Prediction); // The transformed data. @@ -61,32 +67,51 @@ public static void Example() Console.WriteLine("Data\tAlert\tScore\tP-Value"); int k = 0; foreach (var prediction in predictionColumn) - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); - Console.WriteLine(""); + PrintPrediction(data[k++].Value, prediction); // Prediction column obtained post-transformation. // Data Alert Score P-Value - // 0 0 - 2.53 0.50 - // 1 0 - 0.01 0.01 + // 0 0 -2.53 0.50 + // 1 0 -0.01 0.01 // 2 0 0.76 0.14 // 3 0 0.69 0.28 // 4 0 1.44 0.18 - // 0 0 - 1.84 0.17 + // 0 0 -1.84 0.17 // 1 0 0.22 0.44 // 2 0 0.20 0.45 // 3 0 0.16 0.47 // 4 0 1.33 0.18 - // 0 0 - 1.79 0.07 + // 0 0 -1.79 0.07 // 1 0 0.16 0.50 // 2 0 0.09 0.50 // 3 0 0.08 0.45 // 4 0 1.31 0.12 // 100 1 98.21 0.00 <-- alert is on, predicted spike - // 0 0 - 13.83 0.29 - // 1 0 - 1.74 0.44 - // 2 0 - 0.47 0.46 - // 3 0 - 16.50 0.29 - // 4 0 - 29.82 0.21 + // 0 0 -13.83 0.29 + // 1 0 -1.74 0.44 + // 2 0 -0.47 0.46 + // 3 0 -16.50 0.29 + // 4 0 -29.82 0.21 + } + + private static void PrintPrediction(float value, SsaSpikePrediction prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + + class TimeSeriesData + { + public float Value; + + public TimeSeriesData(float value) + { + Value = value; + } + } + + class SsaSpikePrediction + { + [VectorType(3)] + public double[] Prediction { get; set; } } } } diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 186d96da7b..1ec34f25bc 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -268,8 +268,8 @@ public static class PredictionFunctionExtensions /// /// /// /// /// From cab1b81e2c1e16a493564c3304359d98335a21a8 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Wed, 10 Apr 2019 13:48:51 -0700 Subject: [PATCH 5/7] cleanup. --- .../Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs | 1 - .../Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs | 1 - .../Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs | 1 - .../Dynamic/Transforms/TimeSeries/DetectIidSpike.cs | 1 - .../Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs | 1 - .../Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs | 1 - 6 files changed, 6 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs index 66da84080a..833fdceda7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs @@ -9,7 +9,6 @@ namespace Samples.Dynamic { public static class DetectChangePointBySsa { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // It demostrates stateful prediction engine that updates the state of the model and allows for saving/reloading. // The estimator is applied then to identify points where data distribution changed. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs index e7292dbb3c..2890c4ec57 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs @@ -13,7 +13,6 @@ namespace Samples.Dynamic { public static class DetectIidChangePoint { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify points where data distribution changed. public static void Example() diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs index 477662d06a..760305df33 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs @@ -11,7 +11,6 @@ namespace Samples.Dynamic { public static class DetectIidChangePointBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify points where data distribution changed. public static void Example() diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs index ba1a18b999..439006a7ec 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs @@ -9,7 +9,6 @@ namespace Samples.Dynamic { public static class DetectIidSpike { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. public static void Example() diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs index 725c701c49..4145214918 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs @@ -7,7 +7,6 @@ namespace Samples.Dynamic { public static class DetectIidSpikeBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. public static void Example() diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs index d1dc929528..38094bb672 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs @@ -9,7 +9,6 @@ namespace Samples.Dynamic { public static class DetectSpikeBySsa { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // The estimator is applied then to identify spiking points in the series. // This estimator can account for temporal seasonality in the data. From a98751acd0770d4ee972997e900b00fb770abd59 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Wed, 10 Apr 2019 14:48:44 -0700 Subject: [PATCH 6/7] PR feedback. --- .../PredictionFunction.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 1ec34f25bc..aeb4bc99d1 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -68,7 +68,14 @@ public sealed class TimeSeriesPredictionFunction : PredictionEngineB /// /// /// /// /// @@ -268,8 +275,14 @@ public static class PredictionFunctionExtensions /// /// /// /// /// From 45acb3147a89917e812d415b9fd81018d98b88f1 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Wed, 10 Apr 2019 16:43:37 -0700 Subject: [PATCH 7/7] cleanup. --- src/Microsoft.ML.TimeSeries/PredictionFunction.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index aeb4bc99d1..7978b7142f 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -70,12 +70,6 @@ public sealed class TimeSeriesPredictionFunction : PredictionEngineB /// /// /// @@ -277,12 +271,6 @@ public static class PredictionFunctionExtensions /// /// ///