enhancement - add save / restore to neural net

[johndpope]

Been watching the flappy bird clone on coding train on youtube
https://www.youtube.com/watch?v=188B6k_F9jU

CodingTrain/Toy-Neural-Network-JS#50

being able to save out the best neurons would be neat and help persist training across sessions.
https://github.com/CodingTrain/Toy-Neural-Network-JS/blob/master/lib/nn.js#L149

[johndpope]

I took a stab at this - but solution is not so simple.

1. Unfortunately - this guff can't be simply tucked away in protocol extension.

2. these can't be saved I'm afraid...
   activationFunction = { _ in return 0.0 }// try container.decode(func.self, forKey: .activationFunction)
   derivativeActivationFunction = { _ in return 0.0 }//try container.decode(Double.self, forKey: .derivativeActivationFunction)

class Neuron:Codable{
  // CODABLE BEGIN
    //  guff for saving / restoring
    private enum CodingKeys: CodingKey {
        case weights
        case activationFunction
        case derivativeActivationFunction
        case inputCache
        case delta
        case learningRate
    }
    
    required convenience init(from decoder: Decoder) throws
    {
        self.init(weights: [0], activationFunction: { _ in return 0.0 }, derivativeActivationFunction: { _ in return 0.0 })
        let container = try decoder.container(keyedBy: CodingKeys.self);
        weights = try container.decode([Double].self, forKey: .weights);
        activationFunction       = { _ in return 0.0 }//  try container.decode(func.self, forKey: .activationFunction)
        derivativeActivationFunction       =  { _ in return 0.0 }//try container.decode(Double.self, forKey: .derivativeActivationFunction)
        inputCache  = try container.decode(Double.self, forKey: .inputCache)
        delta  = try container.decode(Double.self, forKey: .delta)
        learningRate  = try container.decode(Double.self, forKey: .learningRate)
    }
    
    func encode(to encoder: Encoder) throws
    {
        var container = encoder.container(keyedBy: CodingKeys.self)
        try container.encode(weights, forKey: .weights)
        try container.encode(inputCache, forKey: .inputCache)
        try container.encode(delta, forKey: .delta)
         try container.encode(learningRate, forKey: .learningRate)
        // encoding escape functions ?? activationFunction /derivativeActivationFunction
    }
}

[johndpope]

currently blowing up - https://github.com/johndpope/Bresenham-Line/tree/master/Bresenham-Line/NN

[johndpope]

N.B. - this library loads what seems as official weights / biases from tensorflow

https://github.com/qoncept/TensorSwift/tree/master/Sources/MNIST/Models

public struct Classifier {
    public let W_conv1: Tensor
    public let b_conv1: Tensor
    public let W_conv2: Tensor
    public let b_conv2: Tensor
    public let W_fc1: Tensor
    public let b_fc1: Tensor
    public let W_fc2: Tensor
    public let b_fc2: Tensor
    
    public func classify(_ x_image: Tensor) -> Int {
        let h_conv1 = (x_image.conv2d(filter: W_conv1, strides: [1, 1, 1]) + b_conv1).relu()
        let h_pool1 = h_conv1.maxPool(kernelSize: [2, 2, 1], strides: [2, 2, 1])
        
        let h_conv2 = (h_pool1.conv2d(filter: W_conv2, strides: [1, 1, 1]) + b_conv2).relu()
        let h_pool2 = h_conv2.maxPool(kernelSize: [2, 2, 1], strides: [2, 2, 1])
        
        let h_pool2_flat = h_pool2.reshaped([1, Dimension(7 * 7 * 64)])
        let h_fc1 = (h_pool2_flat.matmul(W_fc1) + b_fc1).relu()
        
        let y_conv = (h_fc1.matmul(W_fc2) + b_fc2).softmax()

        return y_conv.elements.enumerated().max { $0.1 < $1.1 }!.0
    }
}

extension Classifier {
    public init(path: String) {
        W_conv1 = Tensor(shape: [5, 5, 1, 32], elements: loadFloatArray(path, file: "W_conv1"))
        b_conv1 = Tensor(shape: [32], elements: loadFloatArray(path, file: "b_conv1"))
        W_conv2 = Tensor(shape: [5, 5, 32, 64], elements: loadFloatArray(path, file: "W_conv2"))
        b_conv2 = Tensor(shape: [64], elements: loadFloatArray(path, file: "b_conv2"))
        W_fc1 = Tensor(shape: [Dimension(7 * 7 * 64), 1024], elements: loadFloatArray(path, file: "W_fc1"))
        b_fc1 = Tensor(shape: [1024], elements: loadFloatArray(path, file: "b_fc1"))
        W_fc2 = Tensor(shape: [1024, 10], elements: loadFloatArray(path, file: "W_fc2"))
        b_fc2 = Tensor(shape: [10], elements: loadFloatArray(path, file: "b_fc2"))
    }
}

private func loadFloatArray(_ directory: String, file: String) -> [Float] {
    let data = try! Data(contentsOf: URL(fileURLWithPath: directory.stringByAppendingPathComponent(file)))
    return Array(UnsafeBufferPointer(start: UnsafeMutablePointer<Float>(mutating: (data as NSData).bytes.bindMemory(to: Float.self, capacity: data.count)), count: data.count / 4))
}

https://github.com/imane0897/simple_OCR/blob/dc54d9275149ee3768536caf534091d029e3596c/mnist_deep.py
saver = tf.train.Saver({"W_conv1": W_conv1, "b_conv1": b_conv1,
"W_conv2": W_conv2, "b_conv2": b_conv2,
"W_fc1": W_fc1, "b_fc1": b_fc1,
"W_fc2": W_fc2, "b_fc2": b_fc2})

[johndpope]

https://github.com/imane0897/simple_OCR/blob/dc54d9275149ee3768536caf534091d029e3596c/mnist_deep.py
saver = tf.train.Saver({"W_conv1": W_conv1, "b_conv1": b_conv1,
"W_conv2": W_conv2, "b_conv2": b_conv2,
"W_fc1": W_fc1, "b_fc1": b_fc1,
"W_fc2": W_fc2, "b_fc2": b_fc2})

[johndpope]

also relevant / using swift for tensorflow / python /sklearn to import weights / biases

https://github.com/johndpope/swift-digits/blob/master/Models/main.swift

import Python

let load_digits = Python.import("sklearn.datasets").load_digits
let LinearSVC = Python.import("sklearn.svm").LinearSVC
let train_test_split = Python.import("sklearn.model_selection").train_test_split

let classifier = LinearSVC()
let dataset = load_digits()
let (X_train, X_test, y_train, y_test)
    = train_test_split(dataset["data"], dataset["target"]).tuple4
classifier.fit(X_train, y_train)

print("train: \(classifier.score(X_train, y_train))")
print("test:  \(classifier.score(X_test, y_test))")

let coremltools = Python.import("coremltools")
let coreml_model = coremltools.converters.sklearn.convert(classifier)
coreml_model.save("Digits.mlmodel")