diff --git a/opencodeblocks/graphics/window.py b/opencodeblocks/graphics/window.py index 3e2a6af4..e0017f7c 100644 --- a/opencodeblocks/graphics/window.py +++ b/opencodeblocks/graphics/window.py @@ -302,7 +302,8 @@ def createNewMdiChild(self, filename: str = None): ocb_widget = OCBWidget() if filename is not None: ocb_widget.scene.load(filename) - ocb_widget.savepath = filename + if filename.split(".")[-1] == "ipyg": + ocb_widget.savepath = filename return self.mdiArea.addSubWindow(ocb_widget) def onFileNew(self): @@ -374,7 +375,6 @@ def oneFileSaveAsJupyter(self) -> bool: ) if filename == "": return False - current_window.savepath = filename current_window.saveAsJupyter() self.statusbar.showMessage( f"Successfully saved ipygraph as jupter notebook at {current_window.savepath}", diff --git a/opencodeblocks/scene/from_ipynb_conversion.py b/opencodeblocks/scene/from_ipynb_conversion.py index ab9b41c7..7c2ba094 100644 --- a/opencodeblocks/scene/from_ipynb_conversion.py +++ b/opencodeblocks/scene/from_ipynb_conversion.py @@ -41,6 +41,7 @@ def get_blocks_data(data: OrderedDict) -> List[OrderedDict]: next_block_x_pos: float = 0 next_block_y_pos: float = 0 + next_block_id = 0 for cell in data["cells"]: if "cell_type" not in cell or cell["cell_type"] not in ["code", "markdown"]: @@ -62,7 +63,7 @@ def get_blocks_data(data: OrderedDict) -> List[OrderedDict]: block_height: float = text_height + MARGIN_Y block_data = { - "id": len(blocks_data), + "id": next_block_id, "block_type": BLOCK_TYPE_TO_NAME[block_type], "width": block_width, "height": block_height, @@ -93,6 +94,7 @@ def get_blocks_data(data: OrderedDict) -> List[OrderedDict]: next_block_y_pos += block_height + MARGIN_BETWEEN_BLOCKS_Y blocks_data.append(block_data) + next_block_id += 1 adujst_markdown_blocks_width(blocks_data) @@ -144,9 +146,13 @@ def get_edges_data(blocks_data: OrderedDict) -> OrderedDict: ] edges_data: List[OrderedDict] = [] + greatest_block_id: int = 0 + if len(blocks_data) > 0: + greatest_block_id = blocks_data[-1]["id"] + for i in range(1, len(code_blocks)): - socket_id_out = len(blocks_data) + 2 * i - socket_id_in = len(blocks_data) + 2 * i + 1 + socket_id_out = greatest_block_id + 2 * i + 2 + socket_id_in = greatest_block_id + 2 * i + 1 code_blocks[i - 1]["sockets"].append( get_output_socket_data(socket_id_out, code_blocks[i - 1]["width"]) ) diff --git a/tests/assets/complex.ipynb b/tests/assets/complex.ipynb new file mode 100644 index 00000000..0fd86aa3 --- /dev/null +++ b/tests/assets/complex.ipynb @@ -0,0 +1,914 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deep Neural Networks\n", + "\n", + "---\n", + "\n", + "\n", + "\n", + "\n", + "> Version: **1.0**\n", + "\n", + "\n", + "\n", + "\n", + "# 1 . Implementation of a Neural Network\n", + "\n", + "In this exercise you will learn how to implement from scratch a deep neural network.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set-up\n", + "\n", + "Firstly you will import all the packages used through the notebook. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\_distributor_init.py:32: UserWarning: loaded more than 1 DLL from .libs:\n", + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.QVLO2T66WEPI7JZ63PS3HMOHFEY472BC.gfortran-win_amd64.dll\n", + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll\n", + " stacklevel=1)\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import h5py\n", + "\n", + "%matplotlib inline\n", + "\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "np.random.seed(3)\n", + "\n", + "from utils import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialization\n", + "\n", + "Start by defining a function that allows to initialize the parameters of a deep neural network where the dimensions. The number of units in the different layers are passed as argument with `layer_dims`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "def initialization(layer_dims):\n", + "\n", + " np.random.seed(5)\n", + " parameters = {}\n", + " L = len(layer_dims) \n", + " \n", + " for l in range(1, L):\n", + " parameters['W' + str(l)] = np.random.randn(layer_dims[l],layer_dims[l-1])*0.01\n", + " parameters['b' + str(l)] = np.zeros((1,layer_dims[l]))\n", + " \n", + " return parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W1 = [[ 0.00441227 -0.0033087 0.02430771 -0.00252092 0.0010961 ]\n", + " [ 0.01582481 -0.00909232 -0.00591637 0.00187603 -0.0032987 ]\n", + " [-0.01192765 -0.00204877 -0.00358829 0.00603472 -0.01664789]\n", + " [-0.00700179 0.01151391 0.01857331 -0.0151118 0.00644848]]\n", + "b1 = [[0. 0. 0. 0.]]\n", + "W2 = [[-9.80607885e-03 -8.56853155e-03 -8.71879183e-03 -4.22507929e-03]\n", + " [ 9.96439827e-03 7.12421271e-03 5.91442432e-04 -3.63310878e-03]\n", + " [ 3.28884293e-05 -1.05930442e-03 7.93053319e-03 -6.31571630e-03]]\n", + "b2 = [[0. 0. 0.]]\n" + ] + } + ], + "source": [ + "parameters = initialization([5,4,3])\n", + "print(\"W1 = \" + str(parameters[\"W1\"]))\n", + "print(\"b1 = \" + str(parameters[\"b1\"]))\n", + "print(\"W2 = \" + str(parameters[\"W2\"]))\n", + "print(\"b2 = \" + str(parameters[\"b2\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Forward propagation\n", + "\n", + "The forward propagation has been split in different steps. Firstly, the linear forward module computes the following equations:\n", + "\n", + "$$Z^{[l]} = W^{[l]}A^{[l-1]} +b^{[l]}\\tag{4}$$\n", + "\n", + "where $A^{[0]} = X$. \n", + "\n", + "Define a function to compute $Z^{[l]}$" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "def linear_forward(A, W, b):\n", + " Z = W@A + b\n", + " cache = (A, W, b)\n", + " \n", + " return Z, cache" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Z = [[-0.67356113 0.67062057]]\n" + ] + } + ], + "source": [ + "A, W, b = linear_forward_test()\n", + "\n", + "Z, linear_cache = linear_forward(A, W, b)\n", + "print(\"Z = \" + str(Z))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected output**:\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
**Z** [[ -0.67356113 0.67062057]]
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Activation Functions\n", + "\n", + "In the first notebook you implemented the sigmoid function:\n", + "\n", + "- **Sigmoid**: $\\sigma(Z) = \\sigma(W A + b) = \\frac{1}{ 1 + e^{-(W A + b)}}$.\n", + "\n", + "In this notebook, you will need to implement the ReLU activation defined as:\n", + "\n", + "- **ReLU**: $A = RELU(Z) = max(0, Z)$. \n", + "\n", + "Complete the function below that computes the ReLU an activation fucntion." + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "def relu(Z):\n", + "\n", + " A = np.maximum(0,Z)\n", + " cache = Z \n", + " \n", + " return A, cache" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You have implemented a function that determines the linear foward step. You will now combine the output of this function with either a sigmoid() or a relu() activation function. " + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "def forward_one(A_prev, W, b, activation):\n", + " Z, linear_cache = linear_forward(A_prev, W, b)\n", + " if activation == 'relu' :\n", + " A, activation_cache = relu(Z)\n", + " \n", + " elif activation == 'sigmoid' :\n", + " A, activation_cache = sigmoid(Z)\n", + " \n", + " cache = (linear_cache, activation_cache)\n", + "\n", + " return A, cache" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "With sigmoid: A = [[0.96313579 0.22542973]]\n", + "With ReLU: A = [[3.26295337 0. ]]\n" + ] + } + ], + "source": [ + "A_prev, W, b = forward_one_test()\n", + "\n", + "A, linear_activation_cache = forward_one(A_prev, W, b, activation = \"sigmoid\")\n", + "print(\"With sigmoid: A = \" + str(A))\n", + "\n", + "A, linear_activation_cache = forward_one(A_prev, W, b, activation = \"relu\")\n", + "print(\"With ReLU: A = \" + str(A))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Forward propagation model\n", + "\n", + "The structure you will implement in this exercise consists on $L-1$ layers using a ReLU activation function and a last layer using a sigmoid.\n", + "Implement the forward propagation of the above model." + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "def forward_all(X, parameters):\n", + "\n", + " caches = []\n", + " A = X\n", + " L = len(parameters) // 2 \n", + " \n", + " for l in range(1, L):\n", + " A_prev = A \n", + " # Implement L-1 layers of RELU and for each layer add \"cache\" to the \"caches\" list.\n", + " A, cache = forward_one(A,parameters[\"W\"+ str(l)],parameters[\"b\"+ str(l)],\"relu\")\n", + " caches.append(cache)\n", + " AL, cache = forward_one(A,parameters[\"W\"+ str(L)],parameters[\"b\"+ str(L)],\"sigmoid\")\n", + " caches.append(cache)\n", + " \n", + " \n", + " return AL, caches" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AL = [[0.03921668 0.70498921 0.19734387 0.04728177]]\n", + "Length of caches list = 3\n" + ] + } + ], + "source": [ + "X, parameters = forward_all_test()\n", + "AL, caches = forward_all(X, parameters)\n", + "print(\"AL = \" + str(AL))\n", + "print(\"Length of caches list = \" + str(len(caches)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cost function\n", + "\n", + "You will now compute the cross-entropy cost $J$, for all the training set using the following formula: $$-\\frac{1}{m} \\sum\\limits_{i = 1}^{m} (y^{(i)}\\log\\left(a^{[L] (i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right)) \\tag{7}$$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "def cost_function(AL, Y):\n", + " \n", + " m = Y.shape[1]\n", + "\n", + " cost = (-1/m)*np.sum((np.dot(Y,np.log(AL.T))+np.dot((1-Y),np.log(1-AL).T))) \n", + " cost = np.squeeze(cost) # Eliminates useless dimensionality for the variable cost.\n", + " \n", + " return cost" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cost = 0.2797765635793422\n" + ] + } + ], + "source": [ + "Y, AL = compute_cost()\n", + "print(\"cost = \" + str(cost_function(AL, Y)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
**cost** 0.2797765635793422
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Backpropagation \n", + "\n", + "You will now implement the functions that will help you compute the gradient of the loss function with respect to the different parameters.\n", + "\n", + "To move backward in the computational graph you need to apply the chain rule." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Linear backward\n", + "\n", + "For each layer $l$, the linear part is: $Z^{[l]} = W^{[l]} A^{[l-1]} + b^{[l]}$ (followed by an activation).\n", + "\n", + "Suppose you have already calculated the derivative $dZ^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial Z^{[l]}}$. You want to get $(dW^{[l]}, db^{[l]}, dA^{[l-1]})$.\n", + "\n", + "\n", + "The three outputs $(dW^{[l]}, db^{[l]}, dA^{[l-1]})$ are computed using the input $dZ^{[l]}$. The formulas you saw in class are:\n", + "$$ dW^{[l]} = \\frac{\\partial \\mathcal{J} }{\\partial W^{[l]}} = \\frac{1}{m} dZ^{[l]} A^{[l-1] T} \\tag{8}$$\n", + "$$ db^{[l]} = \\frac{\\partial \\mathcal{J} }{\\partial b^{[l]}} = \\frac{1}{m} \\sum_{i = 1}^{m} dZ^{[l](i)}\\tag{9}$$\n", + "$$ dA^{[l-1]} = \\frac{\\partial \\mathcal{L} }{\\partial A^{[l-1]}} = W^{[l] T} dZ^{[l]} \\tag{10}$$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "def linear_backward(dZ, cache):\n", + " A_prev, W, b = cache\n", + " m = A_prev.shape[1]\n", + " \n", + " dW = 1/m * dZ@A_prev.T\n", + " db = 1/m * np.sum(dZ,1, keepdims = True)\n", + " dA_prev = W.T@dZ\n", + " \n", + " return dA_prev, dW, db\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dA_prev = [[ 1.62477986e-01 2.08119187e+00 -1.34890293e+00 -8.08822550e-01]\n", + " [ 1.25651742e-02 -2.21287224e-01 -5.90636554e-01 4.05614891e-03]\n", + " [ 1.98659671e-01 2.39946554e+00 -1.86852905e+00 -9.65910523e-01]\n", + " [ 3.18813678e-01 -9.92645222e-01 -6.57125623e-01 -1.46564901e-01]\n", + " [ 2.48593418e-01 -1.19723579e+00 -4.44132647e-01 -6.09748046e-04]]\n", + "dW = [[-1.05705158 -0.98560069 -0.54049797 0.10982291 0.53086144]\n", + " [ 0.71089562 1.01447326 -0.10518156 0.34944625 -0.12867032]\n", + " [ 0.46569162 0.31842359 0.30629837 -0.01104559 -0.19524287]]\n", + "db = [[ 0.5722591 ]\n", + " [ 0.04780547]\n", + " [-0.38497696]]\n" + ] + } + ], + "source": [ + "# Set up some test inputs\n", + "dZ, linear_cache = linear_backward_test()\n", + "\n", + "dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", + "\n", + "print (\"dA_prev = \"+ str(dA_prev))\n", + "print (\"dW = \" + str(dW))\n", + "print (\"db = \" + str(db))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "** Expected Output**:\n", + " \n", + "```\n", + "dA_prev = \n", + "[[ 1.62477986e-01 2.08119187e+00 -1.34890293e+00 -8.08822550e-01]\n", + " [ 1.25651742e-02 -2.21287224e-01 -5.90636554e-01 4.05614891e-03]\n", + " [ 1.98659671e-01 2.39946554e+00 -1.86852905e+00 -9.65910523e-01]\n", + " [ 3.18813678e-01 -9.92645222e-01 -6.57125623e-01 -1.46564901e-01]\n", + " [ 2.48593418e-01 -1.19723579e+00 -4.44132647e-01 -6.09748046e-04]]\n", + "dW = \n", + "[[-1.05705158 -0.98560069 -0.54049797 0.10982291 0.53086144]\n", + " [ 0.71089562 1.01447326 -0.10518156 0.34944625 -0.12867032]\n", + " [ 0.46569162 0.31842359 0.30629837 -0.01104559 -0.19524287]]\n", + "db = \n", + "[[ 0.5722591 ]\n", + " [ 0.04780547]\n", + " [-0.38497696]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Activation Functions\n", + "\n", + "Now you need to write the code that computes the derivatives for the activation functions. You have learned the derivatives for the sigmoid and the ReLU during theory class.\n", + "Complete the two function below." + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "def sigmoid_backward(dA, cache): \n", + " Z = cache\n", + " \n", + " s = Z*(1-Z)\n", + " dZ = dA*s\n", + " \n", + " return dZ" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [], + "source": [ + "def relu_backward(dA, cache):\n", + " \n", + " Z = cache \n", + " dZ = np.array(dA, copy=True) # convert dz to an array.\n", + " dZ = dZ*np.where(Z>0,1,0)\n", + " return dZ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### One backpropagation step\n", + "\n", + "Next, you will create a function that implements one step of backpropagation," + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [], + "source": [ + "def backward_one(dA, cache, activation):\n", + " linear_cache, activation_cache = cache \n", + " if activation == \"relu\":\n", + " dZ = relu_backward(dA,activation_cache)\n", + " dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", + " elif activation == \"sigmoid\":\n", + " dZ = sigmoid_backward(dA,activation_cache)\n", + " dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", + " \n", + " return dA_prev, dW, db" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sigmoid:\n", + "dA_prev = [[ 0.00410547 0.03685307]\n", + " [-0.01417887 -0.12727776]\n", + " [ 0.00764463 0.06862266]]\n", + "dW = [[ 0.03231386 -0.0904648 0.02919517]]\n", + "db = [[0.06163813]]\n", + "\n", + "relu:\n", + "dA_prev = [[ 0.01679913 0.16610885]\n", + " [-0.05801838 -0.57368247]\n", + " [ 0.031281 0.30930474]]\n", + "dW = [[ 0.14820532 -0.40668077 0.13325465]]\n", + "db = [[0.27525652]]\n" + ] + } + ], + "source": [ + "dAL, linear_activation_cache = linear_activation_backward_test()\n", + "\n", + "dA_prev, dW, db = backward_one(dAL, linear_activation_cache, \"sigmoid\")\n", + "print (\"sigmoid:\")\n", + "print (\"dA_prev = \"+ str(dA_prev))\n", + "print (\"dW = \" + str(dW))\n", + "print (\"db = \" + str(db) + \"\\n\")\n", + "\n", + "dA_prev, dW, db = backward_one(dAL, linear_activation_cache, activation = \"relu\")\n", + "print (\"relu:\")\n", + "print (\"dA_prev = \"+ str(dA_prev))\n", + "print (\"dW = \" + str(dW))\n", + "print (\"db = \" + str(db))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Backpropagation model\n", + "\n", + "Now you will put all together to compute the backward function for the whole network. \n", + "In the backpropagation step, you will use the variables you stored in cache in the `forward_all` function to compute the gradients. You will iterate from the last layer backwards to layer $1$.\n", + "\n", + "You need to start by computing the derivative of the loss function with respect to $A^{[L]}$. And propagate this gradient backward thourgh all the layers in the network.\n", + "\n", + "You need to save each dA, dW and db in the grads dictionary. " + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "def backward_all(AL, Y, caches):\n", + " grads = {}\n", + " L = len(caches) \n", + " m = AL.shape[1]\n", + " Y = Y.reshape(AL.shape) \n", + "\n", + " dZ = AL-Y\n", + " current_cache = caches[L-1]\n", + " grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_backward(dZ, current_cache[0])\n", + " dAL = grads[\"dA\" + str(L-1)]\n", + " for l in reversed(range(L-1)):\n", + " current_cache = caches[l]\n", + " dA_prev_temp, dW_temp, db_temp = backward_one(dAL, current_cache, \"relu\")\n", + " grads[\"dA\" + str(l)] = dA_prev_temp\n", + " grads[\"dW\" + str(l + 1)] = dW_temp\n", + " grads[\"db\" + str(l + 1)] = db_temp\n", + " return grads" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dW1 = [[0.41642713 0.07927654 0.14011329 0.10664197]\n", + " [0. 0. 0. 0. ]\n", + " [0.05365169 0.01021384 0.01805193 0.01373955]]\n", + "db1 = [[-0.22346593]\n", + " [ 0. ]\n", + " [-0.02879093]]\n", + "dA1 = [[-0.80745758 -0.44693186]\n", + " [ 0.88640102 0.49062745]\n", + " [-0.10403132 -0.05758186]]\n" + ] + } + ], + "source": [ + "AL, Y_assess, caches = backward_all_test()\n", + "grads = backward_all(AL, Y_assess, caches)\n", + "print_grads(grads)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected Output**\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "
dW1 [[ 0.41010002 0.07807203 0.13798444 0.10502167]\n", + " [ 0. 0. 0. 0. ]\n", + " [ 0.05283652 0.01005865 0.01777766 0.0135308 ]]
db1 [[-0.22007063]\n", + " [ 0. ]\n", + " [-0.02835349]]
dA1 [[ 0.12913162 -0.44014127]\n", + " [-0.14175655 0.48317296]\n", + " [ 0.01663708 -0.05670698]]
\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Gradient Descent\n", + "\n", + "Finally you can update the parameters of the model according: \n", + "\n", + "$$ W^{[l]} = W^{[l]} - \\alpha \\text{ } dW^{[l]} $$\n", + "$$ b^{[l]} = b^{[l]} - \\alpha \\text{ } db^{[l]} $$\n", + "\n", + "where $\\alpha$ is the learning rate. After computing the updated parameters, store them in the parameters dictionary. " + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "def gradient_descent(parameters, grads, learning_rate):\n", + " L = len(parameters) // 2 \n", + "\n", + " for l in range(L):\n", + " parameters[\"W\" + str(l+1)] = parameters[\"W\"+ str(l+1)] - learning_rate * grads[\"dW\"+ str(l+1)]\n", + " parameters[\"b\" + str(l+1)] = parameters[\"b\"+ str(l+1)] - learning_rate * grads[\"db\"+ str(l+1)]\n", + " return parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W1 = [[-0.59562069 -0.09991781 -2.14584584 1.82662008]\n", + " [-1.76569676 -0.80627147 0.51115557 -1.18258802]\n", + " [-1.0535704 -0.86128581 0.68284052 2.20374577]]\n", + "b1 = [[-0.04659241]\n", + " [-1.28888275]\n", + " [ 0.53405496]]\n", + "W2 = [[-0.55569196 0.0354055 1.32964895]]\n", + "b2 = [[-0.84610769]]\n" + ] + } + ], + "source": [ + "parameters, grads = gradient_descent_test_case()\n", + "parameters = gradient_descent(parameters, grads, 0.1)\n", + "\n", + "print (\"W1 = \"+ str(parameters[\"W1\"]))\n", + "print (\"b1 = \"+ str(parameters[\"b1\"]))\n", + "print (\"W2 = \"+ str(parameters[\"W2\"]))\n", + "print (\"b2 = \"+ str(parameters[\"b2\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected Output**:\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
W1 [[-0.59562069 -0.09991781 -2.14584584 1.82662008]\n", + " [-1.76569676 -0.80627147 0.51115557 -1.18258802]\n", + " [-1.0535704 -0.86128581 0.68284052 2.20374577]]
b1 [[-0.04659241]\n", + " [-1.28888275]\n", + " [ 0.53405496]]
W2 [[-0.55569196 0.0354055 1.32964895]]
b2 [[-0.84610769]]
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can now create a deep neural network combining all the functions defined above." + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "def dnn(X, Y, layers_dims, learning_rate = 0.009, num_iterations = 100, print_cost=True):#lr was 0.009\n", + " costs = [] \n", + " \n", + " parameters = initialization(layers_dims)\n", + " for i in range(0, num_iterations):\n", + " AL, caches = forward_all(X, parameters)\n", + " cost = cost_function(AL, Y)\n", + " costs.append(cost)\n", + " if print_cost:\n", + " print(cost)\n", + " grads = backward_all(AL, Y, caches)\n", + " parameters = gradient_descent(parameters, grads, learning_rate)\n", + " \n", + " \n", + " return parameters, costs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2 . Deep Neural Networks for Classification\n", + "\n", + "Consider now the dataset you used in the previous exercise. You solved the classification problem using Logistic Regression. Propose a Deep Neural Network architecture using the code you developed in the first part of this exercise that improves on the classification results of Logistic Regression." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import h5py\n", + "import scipy\n", + "from PIL import Image\n", + "from scipy import ndimage\n", + "from lr_utils import load_dataset\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, classes = load_dataset()" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "m_train = train_set_x_orig.shape[0]\n", + "m_test = test_set_x_orig.shape[0]\n", + "num_px = train_set_x_orig.shape[1] \n", + "train_set_x_flatten = train_set_x_orig.reshape(m_train,num_px * num_px * 3).T\n", + "test_set_x_flatten = test_set_x_orig.reshape(m_test,num_px * num_px * 3).T\n", + "train_set_x = train_set_x_flatten/255.\n", + "test_set_x = test_set_x_flatten/255." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(12288, 209) 64\n" + ] + }, + { + "ename": "TypeError", + "evalue": "_swapaxes_dispatcher() missing 2 required positional arguments: 'axis1' and 'axis2'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_set_x_flatten\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_px\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mswapaxes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_set_x_flatten\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[0mdnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_set_x_flatten\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_set_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtrain_set_x_flatten\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mswapaxes\u001b[1;34m(*args, **kwargs)\u001b[0m\n", + "\u001b[1;31mTypeError\u001b[0m: _swapaxes_dispatcher() missing 2 required positional arguments: 'axis1' and 'axis2'" + ] + } + ], + "source": [ + "print(train_set_x_flatten.shape, num_px)\n", + "np.swapaxes(train_set_x_flatten)\n", + "dnn(train_set_x_flatten, train_set_y, [train_set_x_flatten.shape[0], 4, 1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "coursera": { + "course_slug": "neural-networks-deep-learning", + "graded_item_id": "c4HO0", + "launcher_item_id": "lSYZM" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/assets/empty.ipynb b/tests/assets/empty.ipynb new file mode 100644 index 00000000..c6f02f6f --- /dev/null +++ b/tests/assets/empty.ipynb @@ -0,0 +1,32 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/assets/usual.ipynb b/tests/assets/usual.ipynb new file mode 100644 index 00000000..8f60756f --- /dev/null +++ b/tests/assets/usual.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# A report" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\_distributor_init.py:32: UserWarning: loaded more than 1 DLL from .libs:\n", + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.QVLO2T66WEPI7JZ63PS3HMOHFEY472BC.gfortran-win_amd64.dll\n", + "c:\\users\\efabr\\miniconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll\n", + " stacklevel=1)\n" + ] + } + ], + "source": [ + "# Imports\n", + "import numpy as np\n", + "import anotherlib as al" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we load some data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def loading_data(filename):\n", + " return np.random.random((20,20))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "data = loading_data(\"hello.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then do something with it" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.63948558, 3.27076204, 3.48612965, 3.04451501, 3.36451054,\n", + " 3.49666763, 3.45160491, 3.41537927, 3.07652413, 3.66197315,\n", + " 3.18283939, 3.24002107, 3.55144998, 3.29783081, 3.57207105,\n", + " 3.52894424, 3.55476906, 3.8049996 , 3.97005352, 3.01381803],\n", + " [3.36975411, 3.79889692, 3.98689836, 3.19976399, 3.07865854,\n", + " 3.0357039 , 3.36727864, 3.44675425, 3.1485551 , 3.32505796,\n", + " 3.94280723, 3.93294761, 3.69969612, 3.39441561, 3.52488239,\n", + " 3.77017711, 3.31080687, 3.13851642, 3.8914675 , 3.22217344],\n", + " [3.68869882, 3.49143552, 3.94120228, 3.29677181, 3.49179313,\n", + " 3.13340065, 3.19535721, 3.19036895, 3.51647584, 3.56500995,\n", + " 3.71223623, 3.20364635, 3.74915529, 3.60763252, 3.33939492,\n", + " 3.32512116, 3.77610616, 3.4022048 , 3.58662752, 3.89863465],\n", + " [3.33233293, 3.67890573, 3.34158316, 3.20828747, 3.50817954,\n", + " 3.5434048 , 3.40567346, 3.34097188, 3.33359796, 3.54765666,\n", + " 3.78149215, 3.82061228, 3.58202782, 3.21714103, 3.78187063,\n", + " 3.14331254, 3.21238203, 3.05876572, 3.23641512, 3.07314562],\n", + " [3.96469085, 3.34664564, 3.80053502, 3.35413646, 3.2456791 ,\n", + " 3.9323963 , 3.14123278, 3.08543803, 3.32433011, 3.53758282,\n", + " 3.56606284, 3.283641 , 3.35755769, 3.57456616, 3.47084384,\n", + " 3.59188003, 3.7366276 , 3.1607076 , 3.70733558, 3.89854312],\n", + " [3.37855368, 3.37342495, 3.74749753, 3.32611455, 3.04470219,\n", + " 3.26913714, 3.55734092, 3.15093813, 3.06262451, 3.33520461,\n", + " 3.53713674, 3.73679289, 3.00179027, 3.67067575, 3.4524812 ,\n", + " 3.13490282, 3.17967332, 3.27475765, 3.63188686, 3.73635671],\n", + " [3.79496576, 3.13701034, 3.9137442 , 3.81506409, 3.45731737,\n", + " 3.71956806, 3.33353583, 3.22600394, 3.320342 , 3.50387212,\n", + " 3.52740845, 3.1051689 , 3.51774963, 3.39219758, 3.76537776,\n", + " 3.13219734, 3.2705521 , 3.51957676, 3.74631399, 3.39932089],\n", + " [3.55779719, 3.41516178, 3.07029595, 3.22289207, 3.28158917,\n", + " 3.79661708, 3.71722156, 3.36995609, 3.33689999, 3.59126513,\n", + " 3.7079066 , 3.49525298, 3.56939265, 3.96935088, 3.12164002,\n", + " 3.676332 , 3.97468624, 3.51783025, 3.21887996, 3.30523752],\n", + " [3.84033676, 3.32525466, 3.71113927, 3.9279179 , 3.002697 ,\n", + " 3.15725527, 3.03005171, 3.47318527, 3.64722962, 3.28827003,\n", + " 3.48847088, 3.19699471, 3.80627061, 3.18231442, 3.54125576,\n", + " 3.34299146, 3.73841012, 3.67166747, 3.9092137 , 3.0335553 ],\n", + " [3.7058796 , 3.57808271, 3.39623753, 3.92757397, 3.1351561 ,\n", + " 3.48092558, 3.33881418, 3.07534734, 3.4116249 , 3.83838506,\n", + " 3.48396284, 3.89211238, 3.09995526, 3.57699336, 3.04028569,\n", + " 3.70796281, 3.82625111, 3.00335284, 3.82351291, 3.45597272],\n", + " [3.97176208, 3.2377911 , 3.35122552, 3.008061 , 3.42012245,\n", + " 3.80543675, 3.05006123, 3.82998196, 3.21035844, 3.9324146 ,\n", + " 3.05317098, 3.61833738, 3.63764824, 3.52124233, 3.87902481,\n", + " 3.50641822, 3.3221031 , 3.7521446 , 3.71614348, 3.18424037],\n", + " [3.85059993, 3.70474109, 3.67106526, 3.46888781, 3.51912545,\n", + " 3.37423032, 3.03162456, 3.46525341, 3.16811541, 3.08918478,\n", + " 3.21506861, 3.49130976, 3.20194597, 3.70852144, 3.83703797,\n", + " 3.46643191, 3.56993413, 3.01070241, 3.0345923 , 3.38354211],\n", + " [3.1067765 , 3.17986932, 3.39974581, 3.81503415, 3.40999637,\n", + " 3.40962731, 3.89842993, 3.68748917, 3.46850976, 3.72833772,\n", + " 3.84667009, 3.51419885, 3.34223842, 3.75641038, 3.09752562,\n", + " 3.10430882, 3.17031172, 3.68318615, 3.26852823, 3.55773047],\n", + " [3.84328038, 3.12204164, 3.04845877, 3.06138172, 3.72354385,\n", + " 3.55180905, 3.52664241, 3.51290538, 3.41487721, 3.5808519 ,\n", + " 3.52471368, 3.1293383 , 3.97481537, 3.53333148, 3.17242121,\n", + " 3.33274109, 3.66331831, 3.48478873, 3.86629266, 3.39943898],\n", + " [3.01380158, 3.91737603, 3.92958052, 3.04459408, 3.33989635,\n", + " 3.46058188, 3.26540603, 3.86486319, 3.0671772 , 3.55414925,\n", + " 3.29465078, 3.73982667, 3.64077128, 3.79522347, 3.64958283,\n", + " 3.62710174, 3.27515289, 3.46606658, 3.65970842, 3.56858124],\n", + " [3.81765364, 3.18478292, 3.80875158, 3.08533944, 3.96073804,\n", + " 3.88650821, 3.98854391, 3.75026235, 3.99059743, 3.20337304,\n", + " 3.29633007, 3.8081186 , 3.38162993, 3.94841158, 3.14155517,\n", + " 3.63415005, 3.09000952, 3.87785134, 3.4880709 , 3.57519841],\n", + " [3.24256969, 3.50708353, 3.41433647, 3.28962266, 3.83899191,\n", + " 3.39102446, 3.6750535 , 3.65617164, 3.5215255 , 3.40090808,\n", + " 3.39341075, 3.62686904, 3.78749222, 3.68898565, 3.6632341 ,\n", + " 3.6559378 , 3.65910078, 3.47150939, 3.42101486, 3.91651094],\n", + " [3.94385503, 3.99691801, 3.93431865, 3.62193945, 3.47153953,\n", + " 3.75111669, 3.00165189, 3.66504947, 3.7384211 , 3.20962032,\n", + " 3.10356305, 3.25014366, 3.22037248, 3.36181067, 3.30191277,\n", + " 3.88477453, 3.12995043, 3.13019189, 3.31214712, 3.13234449],\n", + " [3.21909987, 3.22103605, 3.01240075, 3.27869683, 3.3607669 ,\n", + " 3.73113868, 3.30960944, 3.50728795, 3.89795141, 3.62701628,\n", + " 3.61512764, 3.62268613, 3.72038233, 3.4902457 , 3.8297139 ,\n", + " 3.55794527, 3.43277611, 3.8092867 , 3.28863261, 3.62435088],\n", + " [3.05355563, 3.74357485, 3.12245218, 3.88984309, 3.7065529 ,\n", + " 3.83742772, 3.49424284, 3.44208031, 3.10441662, 3.26142749,\n", + " 3.41649061, 3.57395604, 3.60538383, 3.19872492, 3.2726298 ,\n", + " 3.47467517, 3.72478131, 3.8025068 , 3.54180911, 3.11842077]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = data + 3\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/unit/scene/test_ipynb_conversion.py b/tests/unit/scene/test_ipynb_conversion.py new file mode 100644 index 00000000..d73f357d --- /dev/null +++ b/tests/unit/scene/test_ipynb_conversion.py @@ -0,0 +1,128 @@ +"""Unit tests for the conversion from and to ipynb.""" + +from typing import OrderedDict +from pytest_mock import MockerFixture +import pytest_check as check +import json + +from opencodeblocks.scene.from_ipynb_conversion import ipynb_to_ipyg, is_title +from opencodeblocks.scene.ipynb_conversion_constants import BLOCK_TYPE_TO_NAME + + +class TestIpynbConversion: + + """Conversion from .ipynb""" + + def test_empty_data(self, mocker: MockerFixture): + """should return empty ipyg graph for empty data.""" + check.equal(ipynb_to_ipyg({}), {"blocks": [], "edges": []}) + + def test_empty_notebook_data(self, mocker: MockerFixture): + """should return expected graph for a real empty notebook data.""" + file_path = "./tests/assets/empty.ipynb" + real_notebook_conversion_is_coherent(file_path) + + def test_usual_notebook_data(self, mocker: MockerFixture): + """should return expected graph for a real usual notebook data.""" + file_path = "./tests/assets/usual.ipynb" + real_notebook_conversion_is_coherent(file_path) + + def test_complex_notebook_data(self, mocker: MockerFixture): + """should return expected graph for a real complex notebook data.""" + file_path = "./tests/assets/complex.ipynb" + real_notebook_conversion_is_coherent(file_path) + + def test_is_title(self, mocker: MockerFixture): + """should return True iff the given text can be used as a title for a block.""" + check.equal(is_title(string_to_markdown_block("")), False) + check.equal(is_title(string_to_markdown_block("Data Preprocessing")), True) + check.equal(is_title(string_to_markdown_block("Étude de cas")), True) + check.equal(is_title(string_to_markdown_block("# Report")), False) + check.equal( + is_title( + string_to_markdown_block( + "This is a very very very very very very very very very very very very very very very very very very long explanation" + ) + ), + False, + ) + check.equal(is_title(string_to_markdown_block("New line \n Next line")), False) + + +def real_notebook_conversion_is_coherent(file_path: str): + """Checks that the conversion of the ipynb notebook gives a coherent result. + + Args: + file_path: the path to a .ipynb file + """ + ipynb_data = load_json(file_path) + ipyg_data = ipynb_to_ipyg(ipynb_data) + check_conversion_coherence(ipynb_data, ipyg_data) + +def check_conversion_coherence(ipynb_data: OrderedDict, ipyg_data:OrderedDict): + """Checks that the ipyg data is coherent with the ipynb data. + + The conversion from ipynb to ipyg should return + 1. blocks and edges + 2. the right amount of code blocks and edges + 3. blocks and sockets with unique ids + 4. edges with existing ids + 5. code blocks that always have a source + 6. markdown blocks that always have text + """ + + # blocks and edges are present + check.equal("blocks" in ipyg_data and "edges" in ipyg_data, True) + + # the amount of code blocks and edges is right + code_blocks_in_ipynb: int = 0 + for cell in ipynb_data["cells"]: + if cell["cell_type"] == "code": + code_blocks_in_ipynb += 1 + code_blocks_in_ipyg: int = 0 + for block in ipyg_data["blocks"]: + if block["block_type"] == BLOCK_TYPE_TO_NAME["code"]: + code_blocks_in_ipyg += 1 + check.equal(code_blocks_in_ipyg, code_blocks_in_ipynb) + + # blocks and sockets have unique ids + block_id_set = set([]) + socket_id_set = set([]) + for block in ipyg_data["blocks"]: + if "id" in block: + check.equal(block["id"] in block_id_set, False) + block_id_set.add(block["id"]) + if "sockets" in block: + for socket in block["sockets"]: + if "id" in socket: + check.equal(socket["id"] in socket_id_set, False) + socket_id_set.add(socket["id"]) + + # edges are between objects with existing ids + for edge in ipyg_data["edges"]: + check.equal(edge["source"]["block"] in block_id_set, True) + check.equal(edge["destination"]["block"] in block_id_set, True) + check.equal(edge["source"]["socket"] in socket_id_set, True) + check.equal(edge["destination"]["socket"] in socket_id_set, True) + + # code blocks always have a source and markdown blocks always have a text + for block in ipyg_data["blocks"]: + if block["block_type"] == BLOCK_TYPE_TO_NAME["code"]: + check.equal("source" in block and type(block["source"]) == str, True) + if block["block_type"] == BLOCK_TYPE_TO_NAME["markdown"]: + check.equal("text" in block and type(block["text"]) == str, True) + + +def load_json(file_path: str): + """Helper function that returns the ipynb data in a given file.""" + with open(file_path, "r", encoding="utf-8") as file: + data = json.loads(file.read()) + return data + + +def string_to_markdown_block(string: str): + """Helper function that returns the ipyg data necessary for the is_title function to work.""" + return { + "block_type": BLOCK_TYPE_TO_NAME["markdown"], + "text": string, + }