diff --git a/demo/tutorials/llm_notebooks/HuggingFaceAPI_QA_Summarization_Testing_Notebook.ipynb b/demo/tutorials/llm_notebooks/HuggingFaceAPI_QA_Summarization_Testing_Notebook.ipynb new file mode 100644 index 000000000..af6993ccd --- /dev/null +++ b/demo/tutorials/llm_notebooks/HuggingFaceAPI_QA_Summarization_Testing_Notebook.ipynb @@ -0,0 +1,1580 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "e7PsSmy9sCoR" + }, + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUgAAABcCAYAAAAMJCwKAAAgAElEQVR4nOy9f5gcZ3Xn+znnra5pjcfKZCyNfqDIQgghZMdxZMfGxpbbwhjM2g4h2Ak/Nol3Aw5xEsLu5eHh8vCofNl9uFluLhiwhUi4zib3ZomcZBMgARsjt4RxbGIritcSsiyE0GpleSQLMYxHPd1V59w/qnq6Z6ZnNJJG/Ej6+zw9PW911fueeqvq1Pn9CucASZJokkzZaudirC666KKLcwWZ+y4TveyWJeW4/lKZYYD5mI2m8+YdH61Wk3Tux+uiiy66ODeYYwaZaKUysNSI7xSVtfj4MCPi9t8WLhzY+sADt9fndswuuuiii3ODaO66ShQSM7lvvYj8B6A8/pMIiM4/evToTuDI3I3ZRRdddHHuMIcMMocgC9ysFwx3DBzVyFzCQBpF8VyP10UXXXRxrjDnDBJygdFyl4wiTS3egJPnYrguuuiii3MCPRedem57NHBk3A6pwLxzMVwXXXTRxTnBnEmQSZJ/xP2gaDjhrv00vTSigB12tVqSJNrcf/p+uiFBXXTRxY8ec+7Fvuqq+f1RT/ktgl40PogwbKn/XQgv7KhUsJwBJjNIr10G2UUXXfzocU7iICsV9AfnL4k5nG85//zYKpXv1pMksStv+uT8eKy0RtyWqU9U8U1cU5e9Mb17qtU7anNPWxdddNHF7HEOGOTUTJpKBa1UsC271kYLjh79zyL6bnefP3F4b5JzxLEPvrhw4Z/v7sZMdtFFFz9CnBMGORW5On1V5YLVsUT/CNJrlnXcUzXg+JfU7c5K5ehQ1x7ZRRdd/KhwTsJ8JqMpTW7dzlJc+swykBZ3HpcdAfcMkVAGLVerKHl8UBdddNHFDx3nJMxn2sHMFYrEmrbtPyQxtosuuujitPBDlSDXbwgqDo4grUTtCRJkF1100cWPC+aIQc4uZMdMLAhtzDH/lo7KdhdddNHFjxZzwCATXbuWCNZO8/sWBgdfUvhuCh75hN8mM8P2djfKp4suuvjR4iwYZKLXvq7/YrGeD7jbIBxF3NskyZZ/JTc9LkyBBdP5XNxBwETV8OwwcKJSwarVM6ewiy666OJscEb6bJIkWq0uXOkS/ptqaZ1ZSqsoxQxwU/f28J7Jxzil6LwnG/aDD2zf+rtbz4S2Lrrooou5whlLkCa+LmjP8ix9KXUkEloWxBm+TaTwnDsmok+L6iHcIxcxaBzP0h98bnvlxe1szetLnu0JdtFFF12cKc6YQbprjLgiolKECzXlwVN9Fz2kmdumyPyhNLhGmRhEI9XqnceongFzLIpg0A0s76KLLuYILQaZJAobIZFZMphsgnQ4W7g7ICaAqp2oXHfs4K5dREePthsnZ2BySdPOWS2+K5bTvLG5rcsgu+iiizlBziCTRyIWDpY5ursO5PnPic8QunM3ofgvZ46T2eSp2tB04iRJYkmSpDOmFCau44x77e6II3GZ0s+U0bEyvq+PTc/2Ic8tw5fGJL5l9ky+iy666GJ65AxyydJVuN7OYh/lM88OIQwjz42QygjKMJ6OYlajhzqhd5Q7qFPJO/Ai7Lv5fx7VOHO7CfdZZPJsPtwLe9fxmb2D4H286IuJWYTqAvS8BbgsRmwAGCTL9gFb5mhuuuiii3/lyBlkqsuZN+8OsvogIaqhOgqhRikbJUtHca2TpaM0pE5afzBJNn5m/bb7VGkP8p74/3TtcSapBhODIjvDvj9I+fy7kbCGtF7GrBfPYtwUc8vXd3AIEdC5AEYXXXTRxZkgZ5Alt9yg6BH1sX5gfsHbNOdnriBQ7jVOvpRWqH72rHVYY3bGSytFNBqLkXSQrFFInN70hBffbmiYZYdddNFFF7NDIUECJcgZjytNxtiEA7iRpYqQTu2mubPMsi2AIGKz5LMCmOKmHeMtu3yxiy66OAeI2v6eIthbirVlRGGyq3imlMHJ7bbM60ICzMuatSrsTlmXRrFZqeNddNFFF3OIXEXtIBNOz5CauvfZQ0TqANXqRH47qyK5XYbZRRddnGNMlCDbMUWY7MyR2r3Ys4XjiKC4r61UPnMQsrJpi0lm+olDpfTE4Wo16cS6p6Gviy666GJuMZE1+mTD4/RcyFWsGcRzOpCWAKogHzGyjwATdPbg8QF06d2Vyv2fn75WRbc0WhdddHFuMclJAy3GM7lG4xSHSwp5QLa7W3uwT4t1easHkem1cqHVrWMi0XIXeY9Qa/LHtmOno+cnH801wydt6wa9d9HFjwgdVOxTOVya8N2W1YdE4wXi2YxH5BFERidm5u75/sVPDmAZIEsta/QC9YnHdex9GhrPHJ2YVbH9HDCsRG+6aaCvWg29k3+pVDanlcrzx//lMMr2eW2d08SVMP+lnOuPEdoz485Vptnk7LvTHSdxhbvJ04anw91nXm+hSV87XaeYl4kqdrsXe4oGOy7iWZWKVbJtu2HwfZlnG8VZPC1RCuLgbgMg/ePVfMaHLAZpfakI5gBxTOvHSUzwHGrY0zHHczXWU08tKZ8YyX4f918uwt5VwAwipfF0tbrkvUmS/EQzyZwBJkYClSo6NFRELly0FtjNll1Q1P+05vz/JJ9vF2eARGxqrYV2VIqaC8nE9ONT9lvUmWj2u2VXG9/bDbuHLO+bKf1Ob4OcUqpxIiOrVLAk+e2HIdl62WVLykuXTkfd8wCcGB78UAjRfzCrRyAzVBGapTR4jpjjbbdtiavVY+sybIUIRhaADIJHiB4DHprrMYeGxqK4HF6uIbrYLVMpXgiRBixr1EulenzKTn5skWilglarS/qvrty7LFTlNSby6gWLfJkg/Rw7rrB4FOG4kR1av97/6aGq7CXWw5VKcnxGR10Xs8Omb61A9l0OGXhQPv2tnfzOq/fOWf/JIxFLll2CPbsq3yCK6yj3f2c7d7z8xCmP37Ir5lhpGZEuxp5dCroAedl8JJQR78ElxTmJ7x0G389nnjuI7B0i8eP5+DMwysSVnzown/i5FaitI7rwSk74UpA+xFPcj7P0woPw3C42P/c0YfcBEj/R7HN6RuU+KS6yybgKKRVyzpwk9tRTjD711LQUKsC111nqba6Yyd7vZnvWPvEp9J09KpUkOjR8qC/WeXeKh7fnGToOLghR5GZPcg4Y5Lx5wTL31C2z3BSRM0jLR09H53rAHwKaUmC1urA3w25Q4ZYS4Ro3WyUiKqJ4YcMW0DyyIeBqtZLqARq+AwY/BTz+Iz2Rn2Q0JSd/7mpCuAejTKlkYB8C5oZBJolywZJBotIHSeVW8BSIEB2hkd4BfKHJJzof78rRby9nXvmjZI31CPNxi0GLpBAthCEDF0PCMCE6hNsOFu39Mg39exIfmZZJLn52HRq/DS29kbSxGhFFFEQUHBzDHUxSotJBTP+SZbs/1mSSE+MgRVpSZJP5TG5PqEp2ahWoZVcquivY38QCFq32KVleJ/rm0ATZM3aeQkCQCCd2J3aIEVVkJsn37CCtOyEPgZrgiPrJxBe/uKScuX44aM/HwX8NfBU47hlmDSyr5x+r45ZinoEQ46zGeKuJLYcfrsnjXxaaaqUoqhEiMVEMOoPD9ExQ0lVIuJjcfFYGIkLUj+hNwKn5hKS9qCwDGaD5rIWIfBGWDDzL81OiHiWEftzW4PZOeno/TmQbedm+pR2rj21+9hqi8iZEfhv31WgUIZr32RiDtFgJQRVEIpxVGOsIvdOo2DBVahxvnzkXShL42rai+0nGw9MNE+pM31w7aQzM8WbON27F2+aHgJ9873zTrnre+endIfT8dpaNxTiKoHnWapvtuWi3NRRxQ+WAethd9Ne1RZ4NJrAOn7uKqYkra3dHHLN1pPXlxeJTxRgZmN/A//vcfN75yuHpO7kb5J2FFJfm6cRwgKzxNwj/E6eGiaLWh6SvxFmPllbgBo2xBcQ9v0Wj3s/CAx8i8aFxO+aSfZcS9XycrL4OMyOUFLLDGF/CfRduI0BMlr4c90twW8d5fQsYPvY1vvuq4dxZNNmL3ZTOxnmYTGqfBQwIs+lqMmMYyw+cvEs7fXMNV/WiMlBLqJbTZ+b/SrFlF9HCkfR3Qii/O01PxiIStU+d5Kq1tiWdGoKKY/nLCEXYWS8xVKkkUdcOORdwxl/ycyk/vhAW0Ft+HZmVUVXS9CuUoktxHyREqxitryfxvwdmthU26z3kmtROTD7KC684NuWY+7/TT73+a2j0XsxXkDViSvHtZNn/4MIDnyHxlEXfHsDlA5hdipmhoY5nW8jC3bzn5QemjJ24sujAcn7w4luw7AtTnTQT4iCZJtJnbpjDqXtpqdo5q+yZ0OrYyU+usNUBk+M8f7JQLOi2lhDdlqVjfcJEdU5EUxE9CLbHPT3miKlIHxIGUF2M23KgTJb+c2znDXdXtpwrTHSyzgkSMe57bjlZdmmxxRC/n6h0F5ktQAOkfhNUv0Jy/Wm85DwizSKuQ0naH+674bsrhlny/B+TvZQSlT5CI+1HrZcQ3sBIbQtUh5CfWUccX06jDhqBsJVG9hGGXnFw2kLgL6w4SCL/9+TNp1Gs4sxQVAxXhe+rBMuQIrB8qoMGwAUTFBEZcer5pJ6qNNo5oHvSALPeczycZdK24vuslZvJ/Z+q79kEn7diECfHJZ4+vdUqmrpfEcxX57p06zeRAOJfERu7B0r76uXGcM+YGMRlPOuzLBuUwKVo6UqX8Pj1679bb94/pzqHs6F5ch/5N0yOx5yu/5lspDPRM/m4TmOeaozZn2+bdjgXKnYzHCYK1yC6ODdLZUOkPEpmr8eya8hSRaPXMPiy5SR+4LTjIrdhU45JNirPL6mx8MBfo+k7CKXX5GdkawjxAi5ccZyxxsWk9aW4QVwe4eTI3zH0qoP58dPQMA3j7BzmM9lDfJYe4yRJ7NprP/Gwp/V3hKh86cyKtqu51zJPv9DosSPAYO5JnkRnRw/73KEps+aUztx/O5NKinbTNzXl+5QPcbOo8ERUq2iSJIz3P8n5Nf3DO3176kOXKLPstxOSJNEvPzHQW66Fi9ysb9zmSG6gcLNhj/QDgeN7Ad5wVf6oVquMAMe2b0/23XbbliePHv3eFqE80hw3/y5oSzoO3U7EeJhFqyrU7BaBa55ra15a85Mk01/D6embpRNz/LgZmanl3uDmhsljnQpzrJWMMxq/CRUgMpxvsqh+jO/V/wcS1fAsJu5dRnbychLZf0rypqDDGlOJ5PNwdOMQS57bQ6nnNaR1cPqwrJ8fSMw8/Rncy+ApwgjoPujAbDuez0RMVLHbvdhNJjQeG3l2TOjrX//9pyuVe/+NWe0t7lZkjDTvvxZt4sFcbU9w2f7El39vhJvfNJinNLbR1ZG+uUXrwW6Xb6dWLE+SRLfsWhsNHj0yuH7Dp1bLtvCaRwivuA4WQBY/4jricOhasn/m2vt2fPnL6QFg+HSlnaEh9KuP9i+9Juu5YSty5XUbfCnmPLJN9nuWfSPL0scrleRwXhkp77dS2bQiwy/11FJVVVOxrdsye+3rP7Xz9a998UheZm7higy9/LrruQp0BdssAj3yCPbPlcq926vV3j1JktRnS2vISmURHURzb7XguIuJBpzs4Ne/dmRPMXPtqvN43xddtDtNkuRYs33ZZZt7zz+/foUZ860qputVATz69KEXLxh8ZvDobhsbmz9fe3rWbt2u16x3+XnB5rNBRrZW/cA1lU8+GNGzE5ITM9kyK5UkeuihRQPr19+76pFtevl118urcJaSe2VrW6scuZb0Wat86tFqNT5QqeT9VSr3l2H0cjMbaNJnKqbmCvcc2779vY91GqvOwou3bpPl11TMqIKuV0313oOPVe/aOXX/+8uZ1i6Rbb6Y9cWEVc2iikZZ+OTer3/t93af+so0X/fMnQ3yvj2X4H4NaUMRMdz/jtsvqrP52R2E6ABuq0nTAcRfxyef+wrHV00fjnMmj7Fbffx/kTpRGOWkKm5Riy+IgkzJUJstpqYaTpYUJ4f7nAWq1buOAPedar9WDF2HHzvSdy6NkNImQU50FiVJol/9av+yhfHRm116flHcLgcGkOZNEEAEcVdcUonCgbLKX1+74dN/Ua0e250kSZ0OaB9RALFQvmBwwVvUone523rRkN/iWkjiwm9GpWg7LL4HfusrkEuYW7dlG5Tojzx4DUHVzUTiUW003l+tLvxLM26UEL1PsHUQehGseY754pPRPhi9p1rt2wIc60DqjBhfkUhcPU9HXXbttYMXv+51Q8/kNHZUVydsmzcvW+we/YEIl6q4oYCLikd/0//9F38XLlhe6gn/HuRmcVla1CzNRxZXNfl3HvE3kl2wqVJJdnZikle94Y8HsrGxDaUe/SWMG9xYIKoTGEkeiqcaiR5w2Oos+KvLLttchXqvubwHid6q5PSpuEnQ2C3aWakkV7WPmSSJfvUbFwyW0ujDbtnNiqSIqASNStjDwE3ttFUqj0Rp2LU8ePRRd7+6SZO6mmsoq/EeYBYMsg1z5cVWuYFSOSIdM5BDYE8CUPf9SGMvImuwFOLyJdjoCrj7mbkZeCMs291PI1pNVoTqiB7ETx6j96U6dv4xJKQgkGXzwS7jwgMPkST1001TnL4e5GScczvfRJyWLekcO2m8k/yfJFqtXrA6RPGnIPrP4De4eb+54Vkzxq+BZ3XcU8AjsJUov68S3Zux4M1ffGpJOZfiOp9MMeWxpPZOJXwUZL27q2f1vN+sgWcNwMuOvxENH69U7nvNuBqdaU01KEgZJ0aIVUOs7ksz+A2Nev4Q/Grce90LWpv9muFuKyF8xCj/1k03fXL+bOIR43qtbm7H3a3wSkPLbCD9ov7Rr1YHr9iya+2kJYc7I4rE0JCiGmHEOLEEjZQwX+q22qV0r4j+O5ylbpm25iWPrQTvF5O3u0QfzbKB1ZP7r1TuXRzX7UMq0cfBf9VhgWOYNcav43if7ubmy8F/TSW+5/zz7feGFv70sKg+JSKG5/RhRSygyKpG44LBibdNYpr5MlFdKSqtawORO5dWKpsXTKRvm6mzGMIyEYnHx4AyeE1cpkioM6KIvT4rJIly/3f6gdcXy6AoIjtI64dJXHnx+SHcniCKR4EU95WIrJ05x7oN0wljSaLjtsK0VKHUs5YsNZAU9ypmx3j+sjruu4ii44hAWu8lKr2Z2tjVrL0tym2ns4+rzXecHObzI8aPX9zb1HmpVC9YnRE2icrNbul890wR0yYrLbJFtJ25upu6W+yZXy4e/vC8kcbNUyWacS++uhuOrBb0P7r7cstSLVxammcESB5bKK7uZu7Zmgzf+NBDixbkc+i1PI7eQUxx1KwRu8htKuH95o1lZinuZjjmbX2Cq3umjs8XLb3rByd1PcwmaPv7I0L2zyI6MjHeFXAzRG6MNHzugqGhjZXKp9aQd2rkJocpfTcaYybjBUscxNUtU7N0tbr/IcgVbhYVvNha8yKKgONq1oiRaL2WSu+f2HuirtHHReTd7tni/HwzBVcBXFAR1bbzUMSa46+QEH9w4dDQ73iWPSOqRxAMseJ6ZIjo/FJJV7aGK87RwnJ3W+qeX5e2/QfNGmsLm2lrPlJdhtsCt2J/DNEA5nvghT0zX49JmCsnTb1+MaXyGiw1oEaWfoOFHM+LSVyfYjwOHMctIksHiEpXMbCvb+blpAtMJ4s1+cLi564h6vkAWTqAqqL6NHbyAY4+MAoYFu3A/BmcCDMQ1hJKH+NY/MbChpnHSs6Clok7zCgl/ngwz444x8JtK+snI0kSrVQ2rXDCx1R0vecXILeL5a/nVELphIjsNfc9IcRDImEiE/RMRWWxEG2+9nX3XXLyZKaTw2HGz0noBe/L/1VUo1SQnKG17SqCmmdpFHpeE+L0LUmSqKnXJ3QoqHtWBrnULFuGmZL3aaKKeMs+JCKIiLplkWe2LEjpjmp14eBkp087kiSxSgUT9+2CPi46yd6UF0lWz7I1IcT/u0v0j9dtuO/Prq3c9+bXfnXJsi1b1kaTmWSppOZNHWe80ImD+EoRvcIsNQRVVUSDFT/bhIQrcfWsHrn7r61ff+/VkOhll23uXV8Z/AOV8KtZNtYLFo2fN2IaolGVsB9nt4TosGioC0W/goJFWVbrDaXeD6Csc2cvIupe3C3uphppBs0QGBLy1Etcf8GzbAGeL4ZXVLMy1aAeqOQ25MSqVbRaXdiL+s+6Zf15VpxAca+4yN9Xq0n6Q800ShKF65RM14MMgqRE8X5UHmf32nSciVn9ScZGnyaKQQKIVuixaSs2FCgW4ZMyJZayaPEyNn1rBfftXcnmZ9fw2b03sOQ7mwjRf8fSy9EIgj6O1d/LnWt35IxPjLtW7SPLPkb5vL2okku5cimBv+Wz+/8rn917Awt3D0JVT8UoO8dBdsT0XChx1yLwfE6QnKtyTKeBiT5yz62CrrlDRl+8WQjXFA/nuKoooiaqO71R36QavknGaCb1derhXaJhvVsWk8cwqVlmqqV+Se0DIZTeZ3gqjk728I8nZmrY75buMOe4qi4vJKeBPPOkuZdHZo35SrjuoccW/XUkmRVse1IuRe52EpW6oI+aNQ4gUtYQXeKWXTJZzc+7tyvAlkFy5NRe4Rf3Zb7gc0HjNe4sds90vB6ooI5hWcMQ6ROJ3i6kb45i/+bCRcf/qlod+AJwqOmpbzTESrGk3kZ38yxwN5HIVGSve7bTzU5I0NWIrMOy/lawQ26nVonVqN8CyWPnnffpimjp7WluP8sZjjuCGnAo8+xz5tnfSxSOq9sKcf6tiLzV3fpaHmGP0sbYAkF/CU+HNET1jCxu7w+4qDlfCfDahs0v9ZTWuhvuaZt06nlMs8vP33LL5t4vfvH5WrWKXX2j9pbSsAo3xX2cRvdsGPWvz3wXT4OzYqcb4WX7FuPhKtJ6nKuxjd00xiZ6qe+6aIRNzz6I6M1kYyC6CgmXksie6SvxCGCgcjla2gyhmTgQgffhtpigfWQpwGG88RUyPs6RVROl6MSVIzzEon0fpjzvD2iMrSgkXSPSd5Lpmyj1PsqSpV9G9lQ5fGR/EfIwTbmzM1GxN26EJOETu04ul2dH3+S/IhHuhoQzn37PDAKf+NWxR39/Tc/TZ9zPHKAV4tPGpAQbPHpk0CX+JfD5tN9qriYiJ9wb/3HDhmOPNjfv2rX20JEXXzyo5veAXOHuxUPratYwDfE1sTQuMbfc09tWetidIutEdpqnH80auj2ObbQRxgaiLHqnavR+t6y/RbXg5mgUrQhZulhdzCfFIgKIYwh1N/usRX5P5DIE9ahhsiYS+SOQi/OiGQV7dVPQxYJeDDyZJFPDh5oowmSoVuVLnjUGRMNHRaI+LyQ9mhlJuRqf21CFPjeviMrlaPn69Rs+/alq9dhjlQo0GuDixaJtE9ITTTQC829CfaNQ3yk6r4bbYkPuFA3vxrK+1jUS3DMQW1epbF7gkv0i7oMTcyDERMOwe/qpejn77BNfPj5S/HCgUhnYax56VUu3uzVyVb4ZDKa6yiwbVbeaIHFz3twzcF9dqfzU/GolGSZJrFTZNGDua5quxXH2KCi5mr36e99rLAP2QWKa3dcHvpKiDB5Cs97CHjLfe0axn2cjfiRibPrWKuKe1aR1I4pr1Eef4OjQMZKLWiXDAHTvw2SNEZBeNJSx7A3A508dD6n9aLSu+D9/EIpsXxr1lHweTiD+jwhD42M2+22mG76w6i9Z8u06qncRxVcDZRpjIKEfsVuReAORfpNFS/8W+/W/hOTI5MIas3fStIjPaSharqzE5f0CH0T0g4h/UNo+p9NG9QOi9gF3W3c6FJ17FGxSvJYSLnbzy3MnRpukpaqI/7Xasceq1evG4yIvumh3uviCC3YiPCAhGqG4PXMV1k1hIHO7HogmhDMB4KYhOu6SbQr0fimOXzherRwd/cbDJw6JN+7DssdEI9zb46QwdwZClg20r/Mz3qNDblPXrZbJPVE2dLBaPToK3x95fWXom5h/yt1TL9TUNptqZMgrZjNbuap9dHRkJPoTJ/tdYK+GWIubfeI5NhklmbpZn3t2q0rPPSkL3ghAb/uuzZNonoupB7sbjldh5ESlcnQUjh5Q5L+CPENbFXvH86ElLDUdW6caX+JmOm4eaaq41tiRxvqnN13ZZI5JEat5/DCBexxLc2bbJMrVzfpBBtzTWq5mA1DYFcNSiBZX8pU71Sxbi2XL3QxcwN3cyRMn3Ey1NKAlXdOkO8p8qbstd2tZs91NPfUdUDsx1ck3C5ypCJO4cv93yki4nLS+vAinOU4WHodKEaeZaDOPmedX78PZQVTKGZzZhsK5MzM8HSUdO0ha309aP0BaP0jWOIGIUe6NCAFCWM28+R/B5HMsfnbdxFqStOIan/+fX6KR3oll7ydLdxL1KFFJMQNPe0nTDcTzPkKJTWzad3F+bMtkMdFJMytPdfHMFXMgSorIqED+cUZo+0xoU7RpfSb9PuowKh3X3v7hYrKKXbzv64peJyrz80IWkjNJF3PLhh17II+N22btQc4PPLA7bbhvxX1IhOYDhLtoljV6Bb8cvJ/2cnCOiahmWX3Ig26tVr9br1aTwsaTWLX6vhMmfFk1dApk70uRPjWxKdIjmCg1cftiFA0drFQo+kvSJEksy6wqovtVWyFN7m6ImogOMkskSWK33PJ8bfsjd/1pGuQNZul/EtHdGnpG8WAgaev9InnxCnE1y2K37OJI40/Bomva+2wG0DuF9CiyY/vWux6qVpO0SX+lgp1/vu53T3eIaJ2mKNw80r2XNLrW8pTGCVCNMOVvH3voPUNF8HdxbP7/9q13PYbzpIQSTAjeFVWVsjsHRQPgzegzk1CanyKrxvcN4ToJIXYc1Qjwb6roweZS9OY+X+DSSmWccV+C+4LcOQOCpqLhmEn29Wrl+8OTVwSdHs2XPGcnQY6MDRDF16MaUeqBsZM7iE7sbDk/ig9AIinIA2SZkaVQ6lnOWHrD9J27FXRuh3Ataf3nSMd+lpPRzxHkZ2nUr4lUAr8AACAASURBVOXkS/8HIjuAlNEf9FMq3Uyp9//js/tvnVJkNxEjuT5l6JUHOLzyM8ThtaT1X6Y+9nlK8UE0GGZG/eR8gt5KpA+y6G2Xw8ZxJjnNu8QnqduT2y2IuYGnhtfBUnJ5tPPH2769rQ0pWNGWVPxUl3ASPefAf9SxSyNCfDWiJmBN+5yoIqqHTfwAdPbC+1jPQbf0cBFnaOMrO4orooOO9I+rn+MQBEZcs1pnlVYONetHTiyI45GgEaRtFq6m1wIDHcnwY3n17ok9RlGoC+SFSGWCGwiE0yrc25yHbzx858Ht1aGN4v4rno19VFQeEo0Oi2hK4RgaL3snglmmDstd+DCjcVSYGZjw2hJBjCPFSBPu48sue76myAtISPPzLc5B8nMQZRVu88enq/g2S8F9GtNOPoaITPrdEcFAyiqyF3dEirAmwRR6BVlRrWJr1xLltlyMgkE6uh2V/VLEznrWKLv5RbCkH8Al/KxoZDhWOHNURA+QsTe/dKeTauhn96wkYvREK/BsXe5gQlGG8f71fGbPGyd8Fu99I5959k14I8ZtBFFDxBC/iS27TnEfSUqqdY6uHeWui0Z438tP8K5XHuLoXzzO0OGP4GPvIEv/BNE6acOwdDUiG1my7JKOITxNafKOl9c48ud/g/a9i3r9DtLGnxLFJ9AI6jXQsJhS+WMs3bOqGZI0UcX2JuMZt8xPbY+jzSvj1BCpC1ITpCZyZh+EGlBDfHoJshN959SLPSFPPHZncOJdVgwucjzKQsfAb0isp+fQMHBMVWkvC+wO4tILEkNhMyzGbf2djjKvNfdoUz+104RMYbyGTX64kiTRRqTmkp9H03c/V2+gavWF3SLH/ou4v8fTsd8F+WNURmj6porxRFDPUhC9JoR0DWitKfw0YwUACFNfpM30wsyzurTJSs1XiLur4QvcPPY2ppFL9lkaEXUMiG97kRwZZw5FzwV6Ef8ndxsZZ+aOmmW94K+47JYl5YGBwWU4a1pFkQ1RnkD0ADC+sJ1GpeVZyJYmSaK4r83PurjOKlia7g2hdPA0pr5F55nGQTbVV/cKyCCWKY0xQ/RWouiPCD2fm/iJ/yj/lN6PWx9uSqMGGl/B96KVM4fYOJTHtPOyC9uMw2v2kcUfAdtCFEd5LCSXIvqOZsjYVPrb7J53Lh3lhVXbKcfvx+obCeEQGnImKXI5pu/gwgMxietEFRumMsJTqN2ipDmDo+ZCzdXqLlZ3L75ltm3qAjXwus2kBHSi7xxGII0/jrnEGkkeqNuyXTVvXJd6o6EdCysAVKuYIB0YqBgaVCZyiVlh5uq92Sn3mA06BsmfEZqmgSStVF44uGHDi19qjI1+yN3vEuFA4T0eH89xVKLY1K91UqWI5/TCwTPZMz89/cW3FDpsXso8br2AJrhL0jRk07zkmpCxcRW6SamBO+UU9uCyVzQycTcH3LNYkRXn/yCdLxGXiJb6MENENEsbdXWextLv5jZJDMHcWCoNX/zEE6v6EFbiha3U3VTDCGL/dGYLuZ3FszLOYPQNSGFL1qBEpQFgGSJLO390MSGKgNzuV4oW4375zI4agU5l9NvV96MrhsjsHiwbHY+Qc7uVe3f1zZgt01L/jRUHRvDz/gRr3IOEEUQhrZcpla9mNFsGc/AEpSmIWj2gGJh625uh+aKcZdudVHBcT9MGOUfPcLWKVSpphER9orlHeFzykkLddclVhZz28ZqGDr2lkk3jUUy0Urkwdk72NVlqy/nh6m41F6nLhBqJZ4hxlTLMvN8s0KJzbkX05hxVKsnw0MJlWwaODcVBo4+5Wb9IW9FVHHHWgMduTRUcaIsBPRXG59llvOakC3VEwFrsMZckJY4yZszbdbfzRbStXsr4CGnJ5TBBtnor9lFxjBAPYukCsNeqKJm4iUQK2d5K5ej+rdsu2Ccan3DL+t1dRWxQRFaMjIwckuCL3VtXwtyPoZxe9kzz/Jrc8UxtkPfuvRT8NWSN3K5kthfP9mAetdJrOw3tA2i4FKxMo94P0ev4+D99ie+fGMkXy/r26dHRYq5P80f7dhNK64qCFSuQsJIkyVMaT/UCuf76lOQRWPgzX6As/waXDQgpqsvRxjIS2TdRxT6ddMKNG4tDPBWRmkNNoO5IzZGaS/E5jTbqNReti4fTu4RzJEHmapSWaa7SKC0lU3Nj4xFROdQ+Ty0Hji2uYx09dEkCjdLIgIsvNjOgXfoUHDuheYXjlq3wNJhS59PPOM3whNPs/9Q4VQBztZqkg0d3W+S6WzU6RFtgeZ6P7gAxPiGb5bTombCvkJfTcx8SpD6+zEfBdTVEajbVeVOcSxF9wEpErKm+53lNggjHwWrm2T+4pXVENF9SRUxF+qGxGPe1ZllhRwSQJ5MkMXU9KKJDCCaCOl520VeGYKtVS3mWkGOiQS2r71Orn17udfPkzxYRNxKXI/KMpRouG3n+lb+Enn8bPaXpP0HuIpSeyV9KppTii+ntWwnbjLMNoHbJFwVzz71sQeaf4ohJqBiMHaFeP4Bqmj/O3otob37Krb9nhsjNTWuKmEEuR07Rfjrxu6nPjpF7XSU79xLkxLp/UKmgSZKk69dvWolk42EW446/nA8edOGo5OEhxc+Cu6mIDqpwCbBzciB1ksD6DaxRiRabp4wvN5BXuUnF0n2GRHqGrOicmmDPoP9OZdSa8zxRwk40l9qzMnh5siMwd1n5CYR+0dzHebr0tDQANHegaOruB1TCCcda0qKTB4wrVyVJ8qVOmkClcm+fua+T9vvZx42jB8BHXMMeNfYDa8wzlTy4e74RLhVhZV60Q3C31Mi+AZAGORwsPYSzGjBRAdFV7vYDFaWotI5IhEj69Wr1fSfOrIiwnNnNkiTKsn/fT+Pk68kaoAFE9yAndwDw/JJa5wML5jfwjv301J9Gw7p8jRlbidvFcN0cxDrnWWb5v2ago62c71nWg4t+2vAf1HKeZNY+SR1Y48RMjqntAm2MXyH1fGU6y4qU2BwtBaa1TSe1WxARyzNWbAYJshN9p4/JD0ClklCpJLr1Eb9LVPvNsjw+zwsmaKkiPEua7XMNI7j0uuQ5u7ntSGNxfxvwp8UImveLwoVRaiOvV2WBu1vTGC+CqZaGU8+eELefZ8JbY/bnNc0V4mwtKGf2LCVarS5a7mK3O/5MpXL/1mr1jmm88HDllQN9mcstkqYrEJ9EsIDotwS5zJuhQPlmbb+zZsbE2VEJqWm6C5FDIEvHexHUrAGU3vjwwwvur1SS/fnSxq2eTLhRJVpheXC7FhRansrOznovwyHzuro+jdvaptfZ3frEea2jA4ghqoAcDsiTAFHmQ+bZXtFSxTyFzFXUVpl5LJKNu/TMGmTIGdZXPxsv9kZo7LuEnvJqxk6ChgjsSYLlDq0Z6ywmyvFVIyx69h+Ie9/C2EvzcesnlK/ip1Z8gUsPjHB62eQth9GSvQO4ryJLc6btNkw9O3L65/eDXlwGsbQo2yajICMwOdVwfIXA5k0jrfY0T4umpRTSmqOWhzugrcfcaQmUxcbJAmZ72y0X1CSawYvdib7ZY+3aJB4cXHS1iS/1NN3nrieiKMRbt/pKUb9DVG81y3TcvuS5ucXhYObp0yX1Iy6lRxG/Ec8lcgTFUtMQ3bi+cu//1hjr+X96eg4VMWoLyyYnbw3S83bL0phchcpVJtHIspMHAjxs8PNeLHrkM7C8TpjgZsgdSLTbICevHHk6aB07OyRJYus33Ls60vPuzGxsmVntmfWVz2zH7B9V2Z8GhqJMLAvSGzJfaeLvwv1N7lY4UYq5QcnS2qiKPezwC+30nO55tJ+/4+oi+ywd+6ZoWGd56FbO7NxNlLUhkg/Coru3bHnhcJKQVqsXxnnNR/+ISRp5U5b1XMbVEO03sr+76crjI7t2ra0NHRv6Bwi34pTzQPJ0PrABsd7WlZKdwJE8E+aukfXXf/op1WjY0rQ/L4jhqwVZbtbIox60hFu2uyRHnzytk++E5vM203KsTSSee5Nl6XqcBagaGp2g0djG80PD8MDMYyWJkWxULNpO/eRhRPoRNczWMy9dyrZte1j0zkkHzeKhXvJ8GdffptSzgEbNiGIwHuPFVUdy73el5c2eaclZqkr2skvp6bmYRj1Pa/TsAMYhEtepSy6cUT1IrUsza2Py8ZM16RnahhgK0YTg3kk4i3qQuXTzU72m4VfE7TcJ0Ql1GTUhQhlAQtkss0lDGGAisr3k8QGIR8xH/0IlrMN1QdOp4DmTBJcPx3Hj1akt3HbttYxmLlep6O2epUvBtWlbaxaeyCz9XP1kOtRT1gjBcLS9HuRsMZVlZMW8hDNijNB8lGdPS5IkumULkWSsymx00N0jCdGlAusMUhOGg8mwo6mYlc19UDXEmRW1KNqcHqKKW/b5RoPDUezllg9b8NNw0sCkF4N7/gIJ/ldCuFHUV7lleYiNoG5ZJITbHR+8YHDwi1+r+rGgtVWWydtEdY2bjWsADiaqdcuyh+aVSzvzEKPd6QvbFz0j6BHwFYVwoUBuG3Mxx8zddo6OlIab8/a17faMWXZCkCKHXGKYGHcqKtXqI8k06uypZ2EqNkIyUzTARqCqLBlcisZXktbLedSF7CewO2dC15/aX5CIkTxygMVLHyOetzZP99OVqFxBkuxm0+3ka08V8OKZvo4iYHsjucpaqM6Lvr0Az94KelcRagRuJzC7H6rK4LLL0W/3k922k7suOjI1pKjoKxHj3r2XEOR3SRurwYxo3ijpS9tYYIcY6iRBTodpHDgaxtLM4xqSV0M5mzx4AcMhUzk9G+RpPC31uBzHKQs89zAOoDIghSrtZHnwdrPb3GZlInoos/pfBV48AZDFi/5eG/yChNJveFYvN1W+/CR8vov8RkDfCpK6WX9epqrlnRUXE1V1S78QGPt8Z4/zGbpG5Ix9lB26On0MDv5Ur6Gvxr0XUMtSy/3FROLaj0o/4uNOmMzSybdWKqqK2ZMe/F5ixnn9mUnAHc6jAcdeHHx84cKhTaLh4+QRNCYi6oJC1gv6JhWtAKPu3gfEZqZ5EXsHxDSUEOdxs9q9Dz74nuMA1eojkbL7oIscQFg5ZXwRUwnHzPyfb7nl+RrkNuqr3pDuK9X0gGi0sjBUNZlwbj7FasC2fP8zWXvHARRLI5yL2LT3ZngO/Fe1df81K+Y3289C9DLDWIPIxUVoD2SN3YTy1NUBZ0Jyfcpn9j6IZe/GHUKIsfQm4E8mO+EQYsT72D04zIW/njK6OyJ6Wxn2LiCTdZTC67HoTbgtAIworuPp54nqW7lwRR+mb0PCrdT9m2za8yD+rd2kpUMMMMxL56WE28qk+xZz395LifRdIFdjmVEqK86TpKUt7H5FSlIwtdmZqjo/sHWLLcJriMbkthhMMHVTkyh32bppvq1gPqKFimJKsX+zPwXIZggU74RZPjdJkthrX7u5TMziwnsMnqdw5fbrdkkjV/5D6BnNvPG5gD7ctpzB0A03fOIPGo3yAo3i2y2tNyWaXDV3U3fpQ9wQz+v3FZKPoIiqmttXAvLhavX7w5XKwl6bUUL/yUA+v5+YX4rDxS5mZm0vnPwFpLl0MEntzf/Ns0tCrJ6lzxD8w4svGHzm8IkXFnQebXbocGtYCKndfvvu9IknBv7kpZPyStHwW+T1N1NBiqfBcJMyeWFammuku+dZPSGU1PG9Da+//xtfP76nybSq1W122WVLDp/Xlz4jGq5xyyLaXroI6iIHVdnfnDOAN1yVnPhadeGOoGFDXui3FWCV2yzZL954uv2Y00I+x0paLxNKt1OK3zTrl3CWlUkb/eBQikcYe+kJDi87cdqLcIlvJ02PoNFg7qxhPZv2DY4vP49ofhvI5YSwGWSYWqNOiCKM+USlBZRKg2SNATzLmWpcTmmMfYGGf5yja0+waM9yovJrEF+KyFuJz9uAZ8fRxnFG/BiM1ElLfYQwSFxaSv1kwWR7FPchxkY/xNE1+5vnNlHgG1dX2yeu2e7MhcolTOCkZz7q4qPuPiomNXcZFfOamNda2/Lf3bzmxfb8t3w/cR91l9FsxjjITvTNHqVSvdexQciZFS4mxSdPe5O0CKlINcRDDat/eNEFA/8lL4TQujGvuebEIZEjv25p/ZOi4VirTmOzVqNT2NVM0BTHVCOTEB9yz/6vQPquavU9z7Q7AYq0RcPF2p+pjkGzraMoDMtN+ovtgbT15kvHf5dgrRTCTjjJeICqF7RIUQl4Fo9DVupRkFS1NKIarIitMRFJBTWcPG3O1fJ2HjKjoZRq6DnmWf2PLbLbtq8/+vBFF+1uuw/yfvL9i3Oc1eOpNK9JM60xyyIFuPLK4yPnzcs+hGXvFaI9QeNiPClSIL2Nkef0qqppKJ2wrLElqzdu+Ub1xR2txcEAEnvqqedruD2hWjohzb5a18c8G9sD9XEJrOn1D/A1MwMN7fsX9gd/cmysMTQ5rXLWEPL7BAHL+qifXEy9NrtPkzlqgLQxhPmjpx2ek7hy56uOoeEhQpQ7Yks9g3h6I9Rb9ImmqPQTQoWo52ZKpbcQ4lsJ0QbMLqZRGwSUuHcUZD+1l95Pze7k6CtypqZaJkQpUZybIhq1ftJ0JSJXEKI3EUpvRsONWHYJjbEBRCGeN4LZwzTGfpGjax5vJ7tDPcjJjHBm8axu5BWfFdP8T4H266gdtnVoN3OwZ7JBdqLvtKSvKBL0sKiWTaQPtzJ54QkDqSMyjPsQlu0Usb94tPrbDwM8MMkWXTwQtUrl/g+kfvKL6nabhJ5LgWW49UlegFVB6yI6jNgRS9OnTep/dnxo0WO33747bYZqnH9+ZN//QXZYNX7aMFQL35UEGo2TB0qlUsfsjgaMlDXeIRN0VDFERyRNR4AR1Z4draI2CrghOuI6Ntxxek6GNJSj/aj0mQYTXB1MpaSucqjt3Dvi8eoLB6+5ZvBOVasgvFajaK0QBtyZD152L7SWfC2WuiDH3bMhz+o7UR5UOfbQhmuxR5PEEhK9+sYoVQ0HBN1pmk2gJ5NakW43MaQqSUA0OhZC/DRCLG03mkjpsPjJ0eYSq0mSjFSrfLbuCx8LJreFKGxwD0vzXG0rjpVUJIwAx9zGnvEs+++qjYe2P/q+E52X+YVqlR0i4fEQlZY1tzuYalxv1EYeqX69FarTCpy/d6e7PR6intjVinPNXyBpdvJrPT3DwzOVmpsWlg0T9T4DVj4jI5ijBUNTRr/3GPN69p7u2i7jCPwVIaxFepSe82Cs9mpMHqdU3oPQh3kZiPHm85NnF0GooTJKo3GcNN2PNZ5ArMp7Xr13Qmrh86v3snTPHWR6IyLXEc9bBT6AWR9mEZiimiLRKBKOU39pH7XRv0PCF3jPq4YmO67yJ+uze2+g1LuZdGw5WTadwp3r6I3aX/Kq//W2ZFvFkkTs4986uQLxN6vPQV5b4eixzKvvW3teHmN1775V9ER/i9uaYvW0Dge6EfVAlj3N83922UwXr1K5v5yFk6s9s+UqMmDIAnWPwVLxMOyeHVHVg8C+SuXo6GzVmZtu+uT8kZFohUS+SmCxYX3iquJ+3NWPqLf6hElMJkn0tV/tX1YqlQbaOWFQVxdGouzY/k6LTV150yfnxyO6KgstVScGsiAWsrGDJ08Gi+Ppf69W33dicp+33bYlfv740Apx+jJrHRfU1cZKx77xjTtPmQPcZBqVyr19WQjLQ9YYNNEBy7yfQF4d3RkVYVjdh0APQe+havWOGsWSuW3ZNhEsXJGpz59MTzAZrlbv2teJhqtv3DQY123p1DeLpmPn6/6nvnjnuFzelOB27VobHTl+fJVYusKdpYL3g0YOI2I+BHJo3ryePQ8++JvHTzUHt922JT569IWVmUpvO90A3jN28B8e/A8d+kj06spPrw1ZiJvX7FTXa1b4410D1MMymqnFTWGoUXzP1G7/PxJljCF+75WHzogOgHt39SHzVhIKPpPKML3hEA1bTqO+gCjqwzxGPcI9ArW8iogWoTc+hDeGOLo2v36d1PymY2fZoX7Sl1biuhjxAdA+3CPUR3E5TqZH0Jf28Z6fG5qO3JzbbNqzgZ6+zaS1FTmX7Yj8DdKo/w090duS766oJ4nYJ58bXeaZ3+yEGMfOyktjBqpIJtX3ru3J04U2P7sGjf8WfNW0DNLdKPWAZzt41yt+YeoOE9G+/nG+ZOtLOjT0Xbv9dtL2dZFP19bTYgxJBBcW8/jdZimufK3safucSXWa/phKBW0vedUsk9XcNt3veYzf6fU78zEdeimqgrevTz15/NYa3zP1e/r05BELE49p+3WasI8Wc06SRHftIjp69EJtv4ZF37Ocg6nX9NTzOPGY2V2vU5Exi3VgZoWqwjY7Y+lxCj3NcJxpajlOe9wM+0zYv2CUrf4Vqkwc8+4ZUxJzbrP52Wso9W6mMbYan4FBaqRY+ijiv8Tzq4+TiG1+1hec9Nobxa0X1bP0oBpmmhJk+/f//P88kCSJsenZKwjRF4EFZOn0EmRpHmTpdt698vrZj9fK8ICm6jIXC4ZN7vfHbRGyHxXaM2pgbub63GFittWPN61dzAKniovsACFxZelzl1Cat5n62OXj3qGOfhkB1b1kY7/MC6/eTSJ27y7vS8NL17iEQU5Zx/HUUPfR1OZVhx/gRJKIsXnv2xG9H/N4gkNmAn1uxL2QNv6ad6+8bVYBsF100UUXp0CzWMUwaTact8fTuXJMKExrRqmnHymtgbtJ3PXoEDVTjoh7TfC647Uz/Yh4aipDw0O0ORDCL6AhHndZji9X10afA5aBUtjHZrn+bhdddNHFDMgZZNw4QTZ2pChZNFHymqzSZul84Cou/PU4AZLrJY0bHBHXE47XBK1LpnWh7XPKttcFr5tRH3Pbz7a7cxru/04ZYUPhYe6cqSPFtiyFzJ6d+ynqoosu/rUiZ5CH1p7A2UUUj+YS2jRhMyJKlsbEPeupp2uboVBHh847JioH1b2mntZUqam3fU7ZDjXB63h04OSreo/AxrwOx8n6G9FwMWld8WncP05RXUSOIeSOnblcg7aLLrr4V4vWUonC0+CdY+Pa4Q5ZuhbRm1m4u5ck0eR6SV+M4wOWlo5khLq518y9ZqH4tP/f3m7bniHHYi/tTUQsgTzfslS6sxhzyuJTEyGgYTcuh7r2xy666GKu0JLKgj5NOnaIEGkH70wbXHEvA/8WDVfkbnTX5OVSmzcW71NPjyleV3wio/S2Txtz1NTrkqbH5WR939G1jJK4suSpMpK9EwmvIa3TvnznFIgYuGHZDsbsBFw3RyENXXTRxb92FG5vMf7XoSNktpWoB5gpk4XcIQIr///27ifEruoO4Pj3d869972ZvsQYnTCRYEIYUpmFRBoGXdVAd13ZVpe1QWiKWVYLUkrvUIrYLooUq6YuFARtCy5aKaWbDLRKrS66KLY0dkwlZpKZMB3j+ObNfef+jov73sub/2/GSSPl94FhOMx973Bn8eOce3/n98P5H7L/vapgZR7d6RPS/O++xrRGuaROm1LGIJIUErQQ6fsJWlR/06IUuVxvNqY/Or7vWt7dGWvjXlz2CGW7AVvkcImAS66i5RvMjy2Sn7zpLWONMf8fVi4Vf/HPu3H+LYQM7ZSFiquu7tWHFCWtKaF4lVA8ztzs1W4CZh6jOzhDPSx/spdm0mg5XHSFYxnqaaaFoknQlk+GFubGaeYiSn4ugfuVQ++fILpniXo3ZTtZVeVj1ePRCN4r4v9AaJ3hyl0fbPsAvTHGbGDtXvr5f7+C9w91muC4zXfbUcnqBWX7t8TiKW6Nf+fd8dAfpPJzMeEIyUhzLoER5marPtj5SQnXM+MnYeTBYZyfIKs/g8a7KNsbTLpq/trwAq3mE8wee2GrrHhjjNmO6+Gv+3Lj7L++giQvEXWUUjcPkFW2tuLTgJbvoPpL2vIa82OLOZOdjhAb5CT2H/85cP5OvDyE84+AHKVsb/0cMaIkCSBTEB7mw7FLtno0xuymleEvzx2HH95LO/wY5Nuods4vbkkRgbQ2S2vpjzh+Ra35JqfuWVj3HGg3kD3z/ii++Bo++zqRE8Sy0TvJM8iczjtUH+Ty2GsrvtcYY3bB2kiUR8fBfxwn3fNzQjGBbljdp09nJQmQZAqySFieBvkLTt6mHS+RyiKxdJRxP94fBb5EZILa0CHay/XqxU/cOjjG7vPPuqLlr/mweQpWbuuNMWY3rB8gc1GeO/8NstrPCMVoFSQHLNsdY7Wa9KnDewgBNFR9dKvVaB2fgnMQ2lAG3TSNZ+0EikuA+FdieYqZV3Zem84YYzax/vY3jw75wu9pffIsiEOcDlyUVsQRoyMUyvKSom065wHrIBkxQnsZlpd08ODYPd0TOw165AKqP2UmTG/jXo0xZls2Xhbm0XHLhb0Mhadx8k1Uldh5ntjrM9qp5r3huG+K6+lBdBqUDPD5vjFU5eLTbJ6y/AHt1svMjTdta22MuVE2Xr3lonx05Bqe76O8iEsCzmkv6PWauMsm41U5jL1CE4N+vvsVUq0c01qL0H6C1L3I3G8sOBpjbqitHyzm0THy7gF88jhJ7Vto2IeuetPcW+XJjRgr3iuRi8T4JKfHzu74bo0xZhu2fv6XizI3PovwJGUxSZJdxGdVWbQYtfNWmV7zrN0aRxSRquct7k20/C4Mv3xD/xvGGNNnsLfHuSgzx+bJ0rOE9hkiUyRZwCeuU0OyIn1b452Pq+CbZHRSh14gLJ1hf/t1Zg62dnSXxhizA37gK6cmI/fcqnz8wHka8+dQvQJ6lNrQHlQFYlldGGVNy4beKrFroz7bUqXwJGmLMryDxu8RWs8xO36JuRG1Z47GmP+lwQMkwNRU5H4RFh+4xmO3vcFXH/0dZXsJn9ZIa/Wqx7QH5yIinf1ylPWDo4A4xbkqenrfojZ0haL1JzT8BIk/4jvH3mbiQCA/qUxNbqf5tTHGfGYDZn+vo9eshxRnXwAAALtJREFU+8uOO0aPojIBch/p8HGkPEQobyfGYbzXNdNEdagqIk18chHVC4Tib0TewvNnTn/xam8OSwI3xtwkOw+QcD2Adc9b73+vQcYhXLyDUu9E/GHSZBTxDaJmAGhs4uICoZyB+AGlTEOcxV+7zMzrrV4fW2OMuck+W4Bcrb8Rd34u4fCRhI9Dxp7EsdC5xgfFF8rwcOA/RwK5hF4tSAuMxpjPkd0NkP16W3BYWfJssjPu/LagaIz5nPoUBSp4D1AF9yMAAAAASUVORK5CYII=)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "3o5sAOfwL5qd" + }, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/HuggingFaceAPI_QA_Summarization_Testing_Notebook.ipynb)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "WJJzt3RWhEc6" + }, + "source": [ + "**LangTest** is an open-source python library designed to help developers deliver safe and effective Natural Language Processing (NLP) models. Whether you are using **John Snow Labs, Hugging Face, Spacy** models or **OpenAI, Cohere, AI21, Hugging Face Inference API and Azure-OpenAI** based LLMs, it has got you covered. You can test any Named Entity Recognition (NER), Text Classification model using the library. We also support testing LLMS for Question-Answering and Summarization tasks on benchmark datasets. The library supports 50+ out of the box tests. These tests fall into robustness, accuracy, bias, representation and fairness test categories.\n", + "\n", + "Metrics are calculated by comparing the model's extractions in the original list of sentences against the extractions carried out in the noisy list of sentences. The original annotated labels are not used at any point, we are simply comparing the model against itself in a 2 settings." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "26qXWhCYhHAt" + }, + "source": [ + "# Getting started with LangTest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "azUb114QhOsY", + "outputId": "45a0a692-5076-4fcf-ceb4-b5eb914d9aed" + }, + "outputs": [], + "source": [ + "!pip install \"langtest[evaluate,langchain,transformers]\" " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Harness and Its Parameters\n", + "\n", + "The Harness class is a testing class for Natural Language Processing (NLP) models. It evaluates the performance of a NLP model on a given task using test data and generates a report with test results.Harness can be imported from the LangTest library in the following way." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lTzSJpMlhgq5" + }, + "outputs": [], + "source": [ + "#Import Harness from the LangTest library\n", + "from langtest import Harness" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "sBcZjwJBhkOw" + }, + "source": [ + "It imports the Harness class from within the module, that is designed to provide a blueprint or framework for conducting NLP testing, and that instances of the Harness class can be customized or configured for different testing scenarios or environments.\n", + "\n", + "Here is a list of the different parameters that can be passed to the Harness function:\n", + "\n", + "
\n", + "\n", + "\n", + "| Parameter | Description | \n", + "| - | - | \n", + "|**task** |Task for which the model is to be evaluated (question-answering or summarization)|\n", + "| **model** | Specifies the model(s) to be evaluated. Can be a dictionary or a list of dictionaries. Each dictionary should contain 'model' and 'hub' keys. If a path is specified, the dictionary must contain 'model' and 'hub' keys.|\n", + "| **data** | The data to be used for evaluation. A dictionary providing flexibility and options for data sources. It should include the following keys: |\n", + "| **config** | Configuration for the tests to be performed, specified in the form of a YAML file. |\n", + "\n", + "
\n", + "
" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "JFhJ9CcbsKqN" + }, + "source": [ + "# Model Testing from HuggingFace Hub For Question Answering\n", + "\n", + "In this section, we dive into testing of HuggingFace Hub models in Question Answering task.\n", + "\n", + "LangTest supports robustness tests for LLM testing for now." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "kKgXC7cvuyar" + }, + "source": [ + "### Set environment for HuggingFace Hub" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MHqlSjFLuy7o" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"HUGGINGFACEHUB_API_TOKEN\"] = \"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BoolQ-test-tiny dataset testing" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "swaYPW-wPlku" + }, + "source": [ + "### Setup and Configure Harness" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "p_5nO14bvTzt", + "outputId": "cee6c5f4-6f32-4f72-e9db-440a410b59c7" + }, + "outputs": [], + "source": [ + "harness = Harness(task=\"question-answering\", model={\"model\": \"google/flan-t5-small\",\"hub\": \"huggingface-inference-api\"}, data={\"data_source\": 'BoolQ-test-tiny'})" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "jWPAw9q0PwD1" + }, + "source": [ + "We have specified task as QA, hub as `huggingface-inference-api` and model as `google/flan-t5-small`, `google/flan-t5-xl`, `stabilityai/stablelm-tuned-alpha-3b`, or `databricks/dolly-v2-3b`.\n", + "\n", + "For dataset we used `BoolQ-test-tiny` which includes 50 lines from BoolQ-test. Other available datasets are:\n", + "\n", + "#### BoolQ\n", + "* `BoolQ-test-tiny`\n", + "* `BoolQ-test`\n", + "* `BoolQ-combined`\n", + "#### NQ-open\n", + "* `NQ-open-test`\n", + "* `NQ-open-combined`\n", + "* `NQ-open-test-tiny`\n", + "#### TruthfulQA\n", + "* `TruthfulQA-combined`\n", + "* `TruthfulQA-test`\n", + "* `TruthfulQA-tiny`\n", + "#### MMLU\n", + "* `MMLU-test`\n", + "* `MMLU-test-tiny`\n", + "#### OpenBookQA\n", + "* `OpenBookQA-test`\n", + "* `OpenBookQA-test-tiny`\n", + "#### QUAC\n", + "* `Quac-test`\n", + "* `Quac-test-tiny`\n", + "#### NarrativeQA\n", + "* `NarrativeQA-test`\n", + "* `NarrativeQA-test-tiny`\n", + "#### HellaSwag\n", + "* `HellaSwag-test`\n", + "* `HellaSwag-test-tiny`\n", + "#### BBQ\n", + "* `BBQ-test`\n", + "* `BBQ-test-tiny`\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For tests we used lowercase and uppercase. Other available robustness tests for QA task are:\n", + "* `add_context`\n", + "* `add_contraction`\n", + "* `add_punctuation`\n", + "* `add_typo`\n", + "* `add_ocr_typo`\n", + "* `american_to_british`\n", + "* `british_to_american`\n", + "* `lowercase`\n", + "* `strip_punctuation`\n", + "* `titlecase`\n", + "* `uppercase`\n", + "* `number_to_word`\n", + "* `add_abbreviation`\n", + "* `add_speech_to_text_typo`\n", + "* `add_slangs`\n", + "* `dyslexia_word_swap`\n", + "* `multiple_perturbations`\n", + "* `adjective_synonym_swap`\n", + "* `adjective_antonym_swap`\n", + "* `strip_all_punctuation`" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Available Bias tests for QA task are:\n", + "\n", + "* `replace_to_male_pronouns`\n", + "* `replace_to_female_pronouns`\n", + "* `replace_to_neutral_pronouns`\n", + "* `replace_to_high_income_country`\n", + "* `replace_to_low_income_country`\n", + "* `replace_to_upper_middle_income_country`\n", + "* `replace_to_lower_middle_income_country`\n", + "* `replace_to_white_firstnames`\n", + "* `replace_to_black_firstnames`\n", + "* `replace_to_hispanic_firstnames`\n", + "* `replace_to_asian_firstnames`\n", + "* `replace_to_white_lastnames`\n", + "* `replace_to_sikh_names`\n", + "* `replace_to_christian_names`\n", + "* `replace_to_hindu_names`\n", + "* `replace_to_muslim_names`\n", + "* `replace_to_inter_racial_lastnames`\n", + "* `replace_to_native_american_lastnames`\n", + "* `replace_to_asian_lastnames`\n", + "* `replace_to_hispanic_lastnames`\n", + "* `replace_to_black_lastnames`\n", + "* `replace_to_parsi_names`\n", + "* `replace_to_jain_names`\n", + "* `replace_to_buddhist_names`\n", + "\n", + "Available Representation tests for QA task are:\n", + "\n", + "* `min_gender_representation_count`\n", + "* `min_ethnicity_name_representation_count`\n", + "* `min_religion_name_representation_count`\n", + "* `min_country_economic_representation_count`\n", + "* `min_gender_representation_proportion`\n", + "* `min_ethnicity_name_representation_proportion`\n", + "* `min_religion_name_representation_proportion`\n", + "* `min_country_economic_representation_proportion`\n", + "\n", + "\n", + "Available Accuracy tests for QA task are:\n", + "\n", + "* `min_exact_match_score`\n", + "* `min_bleu_score`\n", + "* `min_rouge1_score`\n", + "* `min_rouge2_score`\n", + "* `min_rougeL_score`\n", + "* `min_rougeLsum_score`\n", + "\n", + "\n", + "Available Fairness tests for QA task are:\n", + "\n", + "* `max_gender_rouge1_score`\n", + "* `max_gender_rouge2_score`\n", + "* `max_gender_rougeL_score`\n", + "* `max_gender_rougeLsum_score`\n", + "* `min_gender_rouge1_score`\n", + "* `min_gender_rouge2_score`\n", + "* `min_gender_rougeL_score`\n", + "* `min_gender_rougeLsum_score`" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also set prompts and other model parameters in config. Possible parameters are:\n", + "* `user_promt:` Promt to be given to the model.\n", + "* `temperature:` Temperature of the model.\n", + "* `max_length:` Maximum number of output tokens allowed for model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Q-ghKYLpPqtt", + "outputId": "20f1a53b-2745-4028-bbe7-8ce4dccb745e" + }, + "outputs": [], + "source": [ + "harness.configure({\n", + " 'model_parameters': {\n", + " 'temperature': 0,\n", + " 'max_length': 64\n", + " },\n", + " \n", + " 'tests': {\n", + " 'defaults':{\n", + " 'min_pass_rate': 1.00\n", + " },\n", + "\n", + " 'robustness':{\n", + " 'lowercase': {'min_pass_rate': 0.70},\n", + " 'uppercase': {'min_pass_rate': 0.70}\n", + " }\n", + " }\n", + "})" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "ZPU46A7WigFr" + }, + "source": [ + "Here we have configured the harness to perform two robustness tests (uppercase and lowercase) and defined the minimum pass rate for each test." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "➤ You can adjust the level of transformation in the sentence by using the \"`prob`\" parameter, which controls the proportion of words to be changed during robustness tests.\n", + "\n", + "➤ **NOTE** : \"`prob`\" defaults to 1.0, which means all words will be transformed.\n", + "```\n", + "harness.configure(\n", + "{\n", + " 'tests': {\n", + " 'defaults': {'min_pass_rate': 0.65},\n", + " 'robustness': {\n", + " 'lowercase': {'min_pass_rate': 0.66, 'prob': 0.50}, \n", + " 'uppercase':{'min_pass_rate': 0.60, 'prob': 0.70},\n", + " }\n", + " }\n", + "})\n", + "\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "i6kPvA13F7cr" + }, + "source": [ + "\n", + "### Generating the test cases." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mdNH3wCKF9fn", + "outputId": "cd348490-7ade-40fa-d870-dc059f5aa647" + }, + "outputs": [], + "source": [ + "harness.generate()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "nyjDdYLeGCmM" + }, + "source": [ + "harness.generate() method automatically generates the test cases (based on the provided configuration)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 641 + }, + "id": "c0jL1_G7F_p6", + "outputId": "502c1525-9000-4041-823b-3b04f6650892" + }, + "outputs": [], + "source": [ + "harness.testcases()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "NOJ8BAU2GGzd" + }, + "source": [ + "harness.testcases() method displays the produced test cases in form of a pandas data frame." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "3CwhQw6hGR9S" + }, + "source": [ + "### Running the tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aguX6-aFGOnP", + "outputId": "bb014811-522b-4f07-fa8a-bf3d1c906d7f" + }, + "outputs": [], + "source": [ + "harness.run()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "191O2oaUGWrH" + }, + "source": [ + "Called after harness.generate() and is to used to run all the tests. Returns a pass/fail flag for each test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XDbd1mpREWR5" + }, + "outputs": [], + "source": [ + "harness.generated_results()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "TKB8Rsr2GZME" + }, + "source": [ + "This method returns the generated results in the form of a pandas dataframe, which provides a convenient and easy-to-use format for working with the test results. You can use this method to quickly identify the test cases that failed and to determine where fixes are needed." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "PBSlpWnUU55G" + }, + "source": [ + "### Final Results" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 112 + }, + "id": "gp57HcF9yxi7", + "outputId": "b893072f-102a-45a6-be03-d737996e659c" + }, + "outputs": [], + "source": [ + "harness.report()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "5N0cKfKiLsiQ" + }, + "source": [ + "#### NQ-open-test dataset testing" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also use another dataset, NQ-open for testing the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oDh3Zaa9EDfZ", + "outputId": "10443ac6-8c92-4e86-ef4e-7050962c4255" + }, + "outputs": [], + "source": [ + "harness = Harness(task=\"question-answering\", model={\"model\": \"google/flan-t5-small\",\"hub\":\"huggingface-inference-api\"}, data={\"data_source\": 'NQ-open-test-tiny'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZnLWJkPVEDmg", + "outputId": "92ca0633-a1c6-4de3-f9fd-c77e6bcb5374" + }, + "outputs": [], + "source": [ + "harness.configure({\n", + " 'model_parameters': {\n", + " 'max_length': 512\n", + " },\n", + " 'tests': {\n", + " 'defaults':{\n", + " 'min_pass_rate': 1.00\n", + " },\n", + "\n", + " 'robustness':{\n", + " 'lowercase': {'min_pass_rate': 0.70},\n", + " 'uppercase': {'min_pass_rate': 0.70}\n", + " }\n", + " }\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "A3U0kM62EG6B", + "outputId": "1ad54c30-3371-41b6-e85c-4dc69ffcd8aa" + }, + "outputs": [], + "source": [ + "harness.generate()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 423 + }, + "id": "0ipwMsy0EHYY", + "outputId": "a3e9c1fb-22dc-416d-8cf6-043ac4b28432" + }, + "outputs": [], + "source": [ + "harness.testcases()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0Nic5HRZEJu5", + "outputId": "dbbf911a-413e-479c-996b-98430920f0b5" + }, + "outputs": [], + "source": [ + "harness.run()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BjZc-ZcCELbU" + }, + "outputs": [], + "source": [ + "harness.generated_results()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 112 + }, + "id": "PlrAxK1eENmh", + "outputId": "7fd59473-20ac-402b-a39b-e5e3e29cf1f4" + }, + "outputs": [], + "source": [ + "harness.report()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HuggingFace Hub Model Testing For Summarization\n", + "\n", + "In this section, we dive into testing of HuggingFace Hub models in Summarization task.\n", + "\n", + "LangTest supports robustness tests for LLM testing for now." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## XSum-test-tiny dataset testing" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup and configure harness" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "harness = Harness(task=\"summarization\", model={\"model\": \"google/pegasus-newsroom\", \"hub\":\"huggingface-inference-api\"}, data={\"data_source\": 'XSum-test-tiny'})" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have specified task as Summarization, hub as `huggingface-inference-api` and any summarization model from huggingface model hub such as `google/pegasus-newsroom`, `facebook/bart-large-cnn`, `google/pegasus-cnn_dailymail`, `google/bigbird-pegasus-large-bigpatent` and many more.\n", + "\n", + "For dataset we used XSum-test-tiny which includes 50 lines from XSum-test. Available datasets for summarization are:\n", + "\n", + "* `XSum-test`\n", + "* `XSum-test-tiny`" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For tests we used lowercase and uppercase. Other available robustness tests for summarization task are:\n", + "\n", + "* `add_context`\n", + "* `add_contraction`\n", + "* `add_punctuation`\n", + "* `add_typo`\n", + "* `add_ocr_typo`\n", + "* `american_to_british`\n", + "* `british_to_american`\n", + "* `lowercase`\n", + "* `strip_punctuation`\n", + "* `titlecase`\n", + "* `uppercase`\n", + "* `number_to_word`\n", + "* `add_abbreviation`\n", + "* `add_speech_to_text_typo`\n", + "* `add_slangs`\n", + "* `dyslexia_word_swap`\n", + "* `multiple_perturbations`\n", + "* `adjective_synonym_swap`\n", + "* `adjective_antonym_swap`\n", + "* `strip_all_punctuation`\n", + "\n", + "Available Bias tests for summarization task are:\n", + "\n", + "* `replace_to_male_pronouns`\n", + "* `replace_to_female_pronouns`\n", + "* `replace_to_neutral_pronouns`\n", + "* `replace_to_high_income_country`\n", + "* `replace_to_low_income_country`\n", + "* `replace_to_upper_middle_income_country`\n", + "* `replace_to_lower_middle_income_country`\n", + "* `replace_to_white_firstnames`\n", + "* `replace_to_black_firstnames`\n", + "* `replace_to_hispanic_firstnames`\n", + "* `replace_to_asian_firstnames`\n", + "* `replace_to_white_lastnames`\n", + "* `replace_to_sikh_names`\n", + "* `replace_to_christian_names`\n", + "* `replace_to_hindu_names`\n", + "* `replace_to_muslim_names`\n", + "* `replace_to_inter_racial_lastnames`\n", + "* `replace_to_native_american_lastnames`\n", + "* `replace_to_asian_lastnames`\n", + "* `replace_to_hispanic_lastnames`\n", + "* `replace_to_black_lastnames`\n", + "* `replace_to_parsi_names`\n", + "* `replace_to_jain_names`\n", + "* `replace_to_buddhist_names`\n", + "\n", + "Available Representation tests for summarization task are:\n", + "\n", + "* `min_gender_representation_count`\n", + "* `min_ethnicity_name_representation_count`\n", + "* `min_religion_name_representation_count`\n", + "* `min_country_economic_representation_count`\n", + "* `min_gender_representation_proportion`\n", + "* `min_ethnicity_name_representation_proportion`\n", + "* `min_religion_name_representation_proportion`\n", + "* `min_country_economic_representation_proportion`\n", + "\n", + "\n", + "Available Accuracy tests for summarization task are:\n", + "\n", + "* `min_exact_match_score`\n", + "* `min_bleu_score`\n", + "* `min_rouge1_score`\n", + "* `min_rouge2_score`\n", + "* `min_rougeL_score`\n", + "* `min_rougeLsum_score`\n", + "\n", + "\n", + "Available Fairness tests for summarization task are:\n", + "\n", + "* `max_gender_rouge1_score`\n", + "* `max_gender_rouge2_score`\n", + "* `max_gender_rougeL_score`\n", + "* `max_gender_rougeLsum_score`\n", + "* `min_gender_rouge1_score`\n", + "* `min_gender_rouge2_score`\n", + "* `min_gender_rougeL_score`\n", + "* `min_gender_rougeLsum_score`" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tests': {'defaults': {'min_pass_rate': 0.65},\n", + " 'robustness': {'uppercase': {'min_pass_rate': 0.66},\n", + " 'lowercase': {'min_pass_rate': 0.6}}}}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.configure(\n", + "{\n", + " 'tests': {'defaults': {'min_pass_rate': 0.65},\n", + " 'robustness': {'uppercase': {'min_pass_rate': 0.66}, \n", + " 'lowercase':{'min_pass_rate': 0.60},\n", + " }\n", + " }\n", + " }\n", + " )" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`Here we have configured the harness to perform two robustness tests (uppercase and lowercase) and defined the minimum pass rate for each test.`" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generating the Test Cases" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Generating testcases...: 100%|██████████| 1/1 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typeoriginaltest_case
0robustnessuppercaseThe ex-Reading defender denied fraudulent trad...THE EX-READING DEFENDER DENIED FRAUDULENT TRAD...
1robustnessuppercaseVoges was forced to retire hurt on 86 after su...VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SU...
2robustnessuppercaseSeven photographs taken in the Norfolk country...SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRY...
3robustnessuppercaseChris Poole - known as \"moot\" online - created...CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED...
4robustnessuppercaseFour police officers were injured in the incid...FOUR POLICE OFFICERS WERE INJURED IN THE INCID...
5robustnessuppercaseThe injured pedestrian - a young man - is thou...THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOU...
6robustnessuppercaseBarca will be investigated for alleged misappr...BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPR...
7robustnessuppercaseThe think tank said the city's 1,536 schools n...THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS N...
8robustnesslowercaseThe ex-Reading defender denied fraudulent trad...the ex-reading defender denied fraudulent trad...
9robustnesslowercaseVoges was forced to retire hurt on 86 after su...voges was forced to retire hurt on 86 after su...
10robustnesslowercaseSeven photographs taken in the Norfolk country...seven photographs taken in the norfolk country...
11robustnesslowercaseChris Poole - known as \"moot\" online - created...chris poole - known as \"moot\" online - created...
12robustnesslowercaseFour police officers were injured in the incid...four police officers were injured in the incid...
13robustnesslowercaseThe injured pedestrian - a young man - is thou...the injured pedestrian - a young man - is thou...
14robustnesslowercaseBarca will be investigated for alleged misappr...barca will be investigated for alleged misappr...
15robustnesslowercaseThe think tank said the city's 1,536 schools n...the think tank said the city's 1,536 schools n...
\n", + "" + ], + "text/plain": [ + " category test_type original \\\n", + "0 robustness uppercase The ex-Reading defender denied fraudulent trad... \n", + "1 robustness uppercase Voges was forced to retire hurt on 86 after su... \n", + "2 robustness uppercase Seven photographs taken in the Norfolk country... \n", + "3 robustness uppercase Chris Poole - known as \"moot\" online - created... \n", + "4 robustness uppercase Four police officers were injured in the incid... \n", + "5 robustness uppercase The injured pedestrian - a young man - is thou... \n", + "6 robustness uppercase Barca will be investigated for alleged misappr... \n", + "7 robustness uppercase The think tank said the city's 1,536 schools n... \n", + "8 robustness lowercase The ex-Reading defender denied fraudulent trad... \n", + "9 robustness lowercase Voges was forced to retire hurt on 86 after su... \n", + "10 robustness lowercase Seven photographs taken in the Norfolk country... \n", + "11 robustness lowercase Chris Poole - known as \"moot\" online - created... \n", + "12 robustness lowercase Four police officers were injured in the incid... \n", + "13 robustness lowercase The injured pedestrian - a young man - is thou... \n", + "14 robustness lowercase Barca will be investigated for alleged misappr... \n", + "15 robustness lowercase The think tank said the city's 1,536 schools n... \n", + "\n", + " test_case \n", + "0 THE EX-READING DEFENDER DENIED FRAUDULENT TRAD... \n", + "1 VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SU... \n", + "2 SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRY... \n", + "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED... \n", + "4 FOUR POLICE OFFICERS WERE INJURED IN THE INCID... \n", + "5 THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOU... \n", + "6 BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPR... \n", + "7 THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS N... \n", + "8 the ex-reading defender denied fraudulent trad... \n", + "9 voges was forced to retire hurt on 86 after su... \n", + "10 seven photographs taken in the norfolk country... \n", + "11 chris poole - known as \"moot\" online - created... \n", + "12 four police officers were injured in the incid... \n", + "13 the injured pedestrian - a young man - is thou... \n", + "14 barca will be investigated for alleged misappr... \n", + "15 the think tank said the city's 1,536 schools n... " + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.testcases()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Running the tests" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running testcases... : 100%|██████████| 16/16 [01:46<00:00, 6.67s/it]\n" + ] + }, + { + "data": { + "text/plain": [] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.run()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generated Results" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typeoriginaltest_caseexpected_resultactual_resulteval_scorepass
0robustnessuppercaseThe ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail.THE EX-READING DEFENDER DENIED FRAUDULENT TRADING CHARGES RELATING TO THE SODJE SPORTS FOUNDATION - A CHARITY TO RAISE MONEY FOR NIGERIAN SPORT.\\nMR SODJE, 37, IS JOINTLY CHARGED WITH ELDER BROTHERS EFE, 44, BRIGHT, 50 AND STEPHEN, 42.\\nAPPEARING AT THE OLD BAILEY EARLIER, ALL FOUR DENIED THE OFFENCE.\\nTHE CHARGE RELATES TO OFFENCES WHICH ALLEGEDLY TOOK PLACE BETWEEN 2008 AND 2014.\\nSAM, FROM KENT, EFE AND BRIGHT, OF GREATER MANCHESTER, AND STEPHEN, FROM BEXLEY, ARE DUE TO STAND TRIAL IN JULY.\\nTHEY WERE ALL RELEASED ON BAIL.The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, allFormer Liverpool defender Steven Caulker has pleaded guilty to one count of conspiracy to commit fraud and one count of conspiracy to commit money laundering in connection with the alleged manipulation of foreign exchange markets.0.000000False
1robustnessuppercaseVoges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests.VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SUFFERING THE INJURY WHILE BATTING DURING THE COUNTY CHAMPIONSHIP DRAW WITH SOMERSET ON 4 JUNE.\\nMIDDLESEX HOPE TO HAVE THE AUSTRALIAN BACK FOR THEIR T20 BLAST GAME AGAINST HAMPSHIRE AT LORD'S ON 3 AUGUST.\\nTHE 37-YEAR-OLD HAS SCORED 230 RUNS IN FOUR FIRST-CLASS GAMES THIS SEASON AT AN AVERAGE OF 57.50.\\n\"LOSING ADAM IS NATURALLY A BLOW AS HE CONTRIBUTES SIGNIFICANTLY TO EVERYTHING WE DO,\" DIRECTOR OF CRICKET ANGUS FRASER SAID.\\n\"HIS ABSENCE, HOWEVER, DOES GIVE OPPORTUNITIES TO OTHER PLAYERS WHO ARE DESPERATE TO PLAY IN THE FIRST XI.\\n\"IN THE PAST WE HAVE COPED WELL WITHOUT AN OVERSEAS PLAYER AND I EXPECT US TO DO SO NOW.\"\\nDEFENDING COUNTY CHAMPIONS MIDDLESEX ARE SIXTH IN THE DIVISION ONE TABLE, HAVING DRAWN ALL FOUR OF THEIR MATCHES THIS SEASON.\\nVOGES RETIRED FROM INTERNATIONAL CRICKET IN FEBRUARY WITH A TEST BATTING AVERAGE OF 61.87 FROM 31 INNINGS, SECOND ONLY TO AUSTRALIAN GREAT SIR DONALD BRADMAN'S CAREER AVERAGE OF 99.94 FROM 52 TESTS.Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced.Former Australia batsman Adam Voges has been forced to retire from first-class cricket after injuring his ankle during Middlesex's draw with Somerset on 4 June.0.301887False
2robustnessuppercaseSeven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said.SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRYSIDE BY PHOTOGRAPHER JOSH OLINS WILL APPEAR IN THE JUNE EDITION.\\nIN HER FIRST SITTING FOR A MAGAZINE, THE DUCHESS IS SEEN LOOKING RELAXED AND WEARING CASUAL CLOTHES.\\nTHE SHOOT WAS IN COLLABORATION WITH THE NATIONAL PORTRAIT GALLERY, WHERE TWO IMAGES ARE BEING DISPLAYED IN THE VOGUE 100: A CENTURY OF STYLE EXHIBITION.\\nTHE DUCHESS, WHO HAS A KEEN INTEREST IN PHOTOGRAPHY, HAS BEEN PATRON OF THE NATIONAL PORTRAIT GALLERY SINCE 2012.\\nNICHOLAS CULLINAN, DIRECTOR OF THE NATIONAL PORTRAIT GALLERY, SAID: \"JOSH HAS CAPTURED THE DUCHESS EXACTLY AS SHE IS - FULL OF LIFE, WITH A GREAT SENSE OF HUMOUR, THOUGHTFUL AND INTELLIGENT, AND IN FACT, VERY BEAUTIFUL.\"\\nHE SAID THE IMAGES ALSO ENCAPSULATED WHAT VOGUE HAD DONE OVER THE PAST 100 YEARS - \"TO PAIR THE BEST PHOTOGRAPHERS WITH THE GREAT PERSONALITIES OF THE DAY, IN ORDER TO REFLECT BROADER SHIFTS IN CULTURE AND SOCIETY\".\\nALEXANDRA SHULMAN, EDITOR-IN-CHIEF OF BRITISH VOGUE, SAID: \"TO BE ABLE TO PUBLISH A PHOTOGRAPHIC SHOOT WITH THE DUCHESS OF CAMBRIDGE HAS BEEN ONE OF MY GREATEST AMBITIONS FOR THE MAGAZINE.\"\\nTHE COLLABORATION FOR THE JUNE EDITION HAD RESULTED IN \"A TRUE CELEBRATION OF OUR CENTENARY AS WELL AS A FITTING TRIBUTE TO A YOUNG WOMAN WHOSE INTEREST IN BOTH PHOTOGRAPHY AND THE COUNTRYSIDE IS WELL KNOWN\", SHE SAID.\\nOTHER ROYAL PORTRAITS TO HAVE FEATURED IN THE FASHION MAGAZINE INCLUDE DIANA, PRINCESS OF WALES - WHO GRACED THE COVER FOUR TIMES - AND PRINCESS ANNE.\\nTHE DUCHESS IS TO VISIT THE EXHIBITION AT THE NATIONAL PORTRAIT GALLERY ON WEDNESDAY, KENSINGTON PALACE SAID.Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100The Duchess of Cambridge will appear on the cover of British Vogue for the first time in the magazine's 100th anniversary issue, it has been announced.0.121951False
3robustnessuppercaseChris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on FacebookCHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003.\\nIT HAS GONE ON TO BE CLOSELY ASSOCIATED WITH OFFENSIVE AND OFTEN ILLEGAL ACTIVITY, INCLUDING INSTANCES WHERE THE IMAGES OF CHILD ABUSE WERE SHARED.\\nIT WAS WIDELY CREDITED AS BEING THE FIRST PLACE WHERE LEAKED IMAGES OF NUDE CELEBRITIES WERE POSTED FOLLOWING 2014'S WELL-PUBLICISED SECURITY BREACH AFFECTING APPLE'S ICLOUD SERVICE. THAT INCIDENT PROMPTED A POLICY CHANGE ON THE SITE.\\nHOWEVER, 4CHAN HAS ALSO BEEN THE RALLYING POINT FOR MANY INSTANCES OF ONLINE ACTIVISM FROM THE LIKES OF ANONYMOUS, THE LOOSELY ORGANIZED HACKTIVISM GROUP.\\nMR POOLE SHARED NEWS OF HIS NEW POSITION ON BLOGGING SITE TUMBLR.\\n\"WHEN MEETING WITH CURRENT AND FORMER GOOGLERS, I CONTINUALLY FIND MYSELF DRAWN TO THEIR INTELLIGENCE, PASSION, AND ENTHUSIASM - AS WELL AS A UNIVERSAL DESIRE TO SHARE IT WITH OTHERS.\"\\n\"I'M ALSO IMPRESSED BY GOOGLE'S COMMITMENT TO ENABLING THESE SAME TALENTED PEOPLE TO TACKLE SOME OF THE WORLD'S MOST INTERESTING AND IMPORTANT PROBLEMS.\\nHE ADDED: \"I CAN'T WAIT TO CONTRIBUTE MY OWN EXPERIENCE FROM A DOZEN YEARS OF BUILDING ONLINE COMMUNITIES, AND TO BEGIN THE NEXT CHAPTER OF MY CAREER AT SUCH AN INCREDIBLE COMPANY.\"\\nMR POOLE STEPPED DOWN AS THE ADMINISTRATOR OF 4CHAN IN JANUARY 2015. NOW HE IS EXPECTED TO TURN HIS ATTENTIONS TO GOOGLE'S SOCIAL NETWORKING EFFORTS.\\nHIS ARRIVAL WAS WELCOMED BY BRADLEY HOROWITZ, THE HEAD OF \"STREAMS, PHOTOS AND SHARING\" AT THE SEARCH GIANT'S FLOUNDERING SOCIAL NETWORK, GOOGLE+.\\n\"I'M THRILLED HE'S JOINING OUR TEAM HERE AT GOOGLE,\" MR HOROWITZ SAID.\\n\"WELCOME CHRIS!Ï»¿\"\\nSEVERAL COMMENTATORS DESCRIBED THE APPOINTMENT AS \"UNEXPECTED\" BUT NOTED THAT MR POOLE'S EXPERTISE WITH SOCIAL MEDIA COULD PROVE USEFUL TO THE SEARCH FIRM.\\nFOLLOW DAVE LEE ON TWITTER @DAVELEEBBC AND ON FACEBOOKThe man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports.CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003. It has gone on to be CLOSELY ASSOCIATED with OFFENSIVE and OFTEN ILLEGAL ACTIVITY, including inSTANCES where the IMAGES of Child ABUSE were shared. It was0.030769False
4robustnessuppercaseFour police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance.FOUR POLICE OFFICERS WERE INJURED IN THE INCIDENT ON FRIDAY NIGHT.\\nA MAN, AGED 19, AND A BOY, AGED 16, HAVE BEEN CHARGED WITH SIX COUNTS OF AGGRAVATED VEHICLE TAKING.\\nTHEY ARE DUE TO APPEAR BEFORE BELFAST MAGISTRATES' COURT ON MONDAY.\\nTHE 19-YEAR-OLD MAN HAS ALSO BEEN CHARGED WITH DRIVING WHILE DISQUALIFIED AND USING A MOTOR VEHICLE WITHOUT INSURANCE.Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday.A 19-year-old man and a 16-year-old boy have been charged after four police officers were injured in a car crash in Belfast on Friday night.0.294118False
5robustnessuppercaseThe injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured.THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOUGHT TO HAVE BEEN WALKING WITH A GROUP OF PEOPLE FROM A GRADUATION CEREMONY AT THE CAIRD HALL.\\nTHE INCIDENT TOOK PLACE ON HIGH STREET AT ABOUT 18:00.\\nTHE MAN'S INJURIES ARE BELIEVED NOT TO BE LIFE-THREATENING. THE DRIVER OF THE TAXI IS THOUGHT TO BE UNINJURED.The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought toA man in his 20s has been seriously injured after he was hit by a TAXI in central London. He is thought to have been walking with a group of people from a graduation ceremony at the CAIRD HALL.0.391304False
6robustnessuppercaseBarca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005.BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS.\\nTHE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES\\nROSELL, SPEAKING AT A NEWS CONFERENCE AFTER A BARCA BOARD MEETING, INSISTED HE HAD \"ACTED CORRECTLY\".\\nVICE-PRESIDENT JOSEP MARIA BARTOMEU NOW TAKES OVER FROM THE 49-YEAR-OLD ROSELL, WHO CAME TO POWER IN 2010.\\nROSELL'S FUTURE HAS BEEN A REAL SOURCE OF CONCERN EVER SINCE A SPANISH NATIONAL COURT JUDGE ACCEPTED A LAWSUIT THIS WEEK FROM BARCELONA CLUB MEMBER JORDI CASES, WHO ALLEGED THAT THE AMOUNT PAID FOR NEYMAR WAS MORE THAN THE REPORTED FEE.\\nROSELL MAINTAINS THE ACCUSATION IS \"UNFAIR AND RECKLESS\".\\nHE ADDED: \"FOR SOME TIME, MY FAMILY AND MYSELF HAVE SUFFERED THREATS AND ATTACKS IN SILENCE. THESE THREATS AND ATTACKS HAVE MADE ME WONDER IF BEING PRESIDENT MEANS HAVING TO JEOPARDISE MY FAMILY.\\n\"FROM THE BEGINNING, I HAVE SAID THE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES.\"\\nROSELL SAID HE WAS RESIGNING TO SPARE THE CLUB'S BOARD OF DIRECTORS FROM \"UNFAIR ATTACKS\" THAT COULD \"NEGATIVELY AFFECT THEIR MANAGEMENT OR THE IMAGE OF THE CLUB\".\\nHE ADDED: \"IT HAS BEEN AN HONOUR TO SERVE THE BARCELONISTAS. IT'S BEEN A PRIVILEGE TO BE THE PRESIDENT OF FC BARCELONA.\"\\nROSELL WAS NAMED BARCA PRESIDENT IN THE SUMMER OF 2010 AFTER WINNING A LANDSLIDE VOTE TO SUCCEED JOAN LAPORTA, EARNING 61.34% OF A RECORD TURN-OUT OF 57,088 VOTERS.\\nHE HAD ALSO BEEN VICE-PRESIDENT UNDER LAPORTA BEFORE RESIGNING IN 2005.Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar.Barcelona will be INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS.0.487805False
7robustnessuppercaseThe think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\"THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS NEEDED TO SAVE £360M IN THE FIRST YEAR IF THE GOVERNMENT'S NATIONAL FUNDING FORMULA (NFF) PLAN GOES AHEAD.\\nTHE AMOUNT IS THE EQUIVALENT OF 12,857 QUALIFIED TEACHERS, ON AN AVERAGE SALARY OF £28,000.\\nTHE GOVERNMENT SAID LONDON WAS THE HIGHEST FUNDED PART OF THE COUNTRY.\\nIT ADDED THAT UNDER THE PLANS, WHICH ARE UNDER CONSULTATION, INNER-CITY SCHOOLS WOULD BE ALLOCATED 30% MORE MONEY PER PUPIL THAN THE NATIONAL AVERAGE.\\nBUT LONDON COUNCILS, WHICH REPRESENTS THE CITY'S 32 BOROUGHS AND THE CITY, SAID NO SCHOOL WOULD GAIN ENOUGH FUNDING FROM THE NFF TO COMPENSATE FOR INCREASED COST PRESSURES FROM INFLATION, HIGHER PENSION CONTRIBUTIONS AND NATIONAL INSURANCE.\\nMINISTERS SAID THE NEW FORMULA WAS NEEDED TO TACKLE UNEVEN LEVELS OF FUNDING ACROSS ENGLAND, WITH THE BEST FUNDED AREAS GETTING MORE THAN £6,300 PER PUPIL PER YEAR, WHILE THE WORST-FUNDED AVERAGING £4,200.\\nIT SAID THE FUNDING CUT WAS ON TOP OF NATIONAL AUDIT OFFICE FIGURES WHICH SHOWED ENGLAND SCHOOLS FACED AN EIGHT PER CENT REAL-TERMS CUT PER PUPIL BY 2019-20 BECAUSE IT WIDER COST PRESSURES.\\nIN A STATEMENT, LONDON COUNCILS SAID: \"AT A TIME WHEN UK SCHOOLS ARE SEEN AS UNDERPERFORMING BY INTERNATIONAL STANDARDS, AND WHEN BUSINESSES BASED IN LONDON ARE FACING MASSIVE UNCERTAINTY ABOUT RECRUITING SKILLED STAFF, THERE IS AN URGENT NEED TO INVEST IN SCHOOLS IN LONDON AND ACROSS THE REST OF THE COUNTRY.\"\\nIT ADDED: \"WITHOUT THE RIGHT QUALIFICATIONS AND SKILLS, LONDON'S CHILDREN WILL BE UNABLE TO ACCESS JOBS AND CONTRIBUTE TO THE NATIONAL ECONOMY. OVER 60% OF JOBS IN INNER LONDON REQUIRE A DEGREE AND AROUND 45% OF JOBS IN THE REST OF THE CAPITAL REQUIRE A DEGREE.\"The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government saidLondon's 1,536 schools need to save £360M in the first year if the government's national funding plan goes ahead, says think tank.0.486486False
8robustnesslowercaseThe ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail.the ex-reading defender denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for nigerian sport.\\nmr sodje, 37, is jointly charged with elder brothers efe, 44, bright, 50 and stephen, 42.\\nappearing at the old bailey earlier, all four denied the offence.\\nthe charge relates to offences which allegedly took place between 2008 and 2014.\\nsam, from kent, efe and bright, of greater manchester, and stephen, from bexley, are due to stand trial in july.\\nthey were all released on bail.The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, allEx-Reading defender Samuel Eto'o has denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for Nigerian sport.0.558824True
9robustnesslowercaseVoges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests.voges was forced to retire hurt on 86 after suffering the injury while batting during the county championship draw with somerset on 4 june.\\nmiddlesex hope to have the australian back for their t20 blast game against hampshire at lord's on 3 august.\\nthe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"losing adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket angus fraser said.\\n\"his absence, however, does give opportunities to other players who are desperate to play in the first xi.\\n\"in the past we have coped well without an overseas player and i expect us to do so now.\"\\ndefending county champions middlesex are sixth in the division one table, having drawn all four of their matches this season.\\nvoges retired from international cricket in february with a test batting average of 61.87 from 31 innings, second only to australian great sir donald bradman's career average of 99.94 from 52 tests.Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced.Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season after injuring his hamstring during the county championship draw with Somerset on June 4.0.545455True
10robustnesslowercaseSeven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said.seven photographs taken in the norfolk countryside by photographer josh olins will appear in the june edition.\\nin her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nthe shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue 100: a century of style exhibition.\\nthe duchess, who has a keen interest in photography, has been patron of the national portrait gallery since 2012.\\nnicholas cullinan, director of the national portrait gallery, said: \"josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nhe said the images also encapsulated what vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nalexandra shulman, editor-in-chief of british vogue, said: \"to be able to publish a photographic shoot with the duchess of cambridge has been one of my greatest ambitions for the magazine.\"\\nthe collaboration for the june edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nother royal portraits to have featured in the fashion magazine include diana, princess of wales - who graced the cover four times - and princess anne.\\nthe duchess is to visit the exhibition at the national portrait gallery on wednesday, kensington palace said.Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100Seven photographs taken in the norfolk countryside by photographer josholins will appear in the june edition of British Vogue. in her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. the shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue0.945455True
11robustnesslowercaseChris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on Facebookchris poole - known as \"moot\" online - created the site in 2003.\\nit has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nit was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting apple's icloud service. that incident prompted a policy change on the site.\\nhowever, 4chan has also been the rallying point for many instances of online activism from the likes of anonymous, the loosely organized hacktivism group.\\nmr poole shared news of his new position on blogging site tumblr.\\n\"when meeting with current and former googlers, i continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"i'm also impressed by google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nhe added: \"i can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nmr poole stepped down as the administrator of 4chan in january 2015. now he is expected to turn his attentions to google's social networking efforts.\\nhis arrival was welcomed by bradley horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, google+.\\n\"i'm thrilled he's joining our team here at google,\" mr horowitz said.\\n\"welcome chris!\"\\nseveral commentators described the appointment as \"unexpected\" but noted that mr poole's expertise with social media could prove useful to the search firm.\\nfollow dave lee on twitter @daveleebbc and on facebookThe man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports.Google has appointed the former administrator of 4chan, one of the world's most notorious online forums, to its social media team. Dave Lee reports.0.113208False
12robustnesslowercaseFour police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance.four police officers were injured in the incident on friday night.\\na man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nthey are due to appear before belfast magistrates' court on monday.\\nthe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance.Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday.Four police officers were injured in the incident on Friday night. a man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. they are due to appear before Belfast magistrates' court on Monday.1.000000True
13robustnesslowercaseThe injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured.the injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the caird hall.\\nthe incident took place on high street at about 18:00.\\nthe man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured.The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought toA pedestrian is injured after being hit by a taxi in central London. The incident took place on high street at about 18:00. the man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured.0.591837True
14robustnesslowercaseBarca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005.barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos.\\nthe signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nrosell, speaking at a news conference after a barca board meeting, insisted he had \"acted correctly\".\\nvice-president josep maria bartomeu now takes over from the 49-year-old rosell, who came to power in 2010.\\nrosell's future has been a real source of concern ever since a spanish national court judge accepted a lawsuit this week from barcelona club member jordi cases, who alleged that the amount paid for neymar was more than the reported fee.\\nrosell maintains the accusation is \"unfair and reckless\".\\nhe added: \"for some time, my family and myself have suffered threats and attacks in silence. these threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"from the beginning, i have said the signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nrosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nhe added: \"it has been an honour to serve the barcelonistas. it's been a privilege to be the president of fc barcelona.\"\\nrosell was named barca president in the summer of 2010 after winning a landslide vote to succeed joan laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nhe had also been vice-president under laporta before resigning in 2005.Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar.Barcelona will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos, the club has announced.0.444444False
15robustnesslowercaseThe think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\"the think tank said the city's 1,536 schools needed to save £360m in the first year if the government's national funding formula (nff) plan goes ahead.\\nthe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nthe government said london was the highest funded part of the country.\\nit added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nbut london councils, which represents the city's 32 boroughs and the city, said no school would gain enough funding from the nff to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nministers said the new formula was needed to tackle uneven levels of funding across england, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nit said the funding cut was on top of national audit office figures which showed england schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nin a statement, london councils said: \"at a time when uk schools are seen as underperforming by international standards, and when businesses based in london are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in london and across the rest of the country.\"\\nit added: \"without the right qualifications and skills, london's children will be unable to access jobs and contribute to the national economy. over 60% of jobs in inner london require a degree and around 45% of jobs in the rest of the capital require a degree.\"The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government saidLondon's 1,536 schools need to save £360m in the first year if the government's national funding formula (nff) plan goes ahead0.547945True
\n", + "
" + ], + "text/plain": [ + " category test_type \\\n", + "0 robustness uppercase \n", + "1 robustness uppercase \n", + "2 robustness uppercase \n", + "3 robustness uppercase \n", + "4 robustness uppercase \n", + "5 robustness uppercase \n", + "6 robustness uppercase \n", + "7 robustness uppercase \n", + "8 robustness lowercase \n", + "9 robustness lowercase \n", + "10 robustness lowercase \n", + "11 robustness lowercase \n", + "12 robustness lowercase \n", + "13 robustness lowercase \n", + "14 robustness lowercase \n", + "15 robustness lowercase \n", + "\n", + " original \\\n", + "0 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail. \n", + "1 Voges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests. \n", + "2 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said. \n", + "3 Chris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on Facebook \n", + "4 Four police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", + "5 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured. \n", + "6 Barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005. \n", + "7 The think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", + "8 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail. \n", + "9 Voges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests. \n", + "10 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said. \n", + "11 Chris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on Facebook \n", + "12 Four police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", + "13 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured. \n", + "14 Barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005. \n", + "15 The think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", + "\n", + " test_case \\\n", + "0 THE EX-READING DEFENDER DENIED FRAUDULENT TRADING CHARGES RELATING TO THE SODJE SPORTS FOUNDATION - A CHARITY TO RAISE MONEY FOR NIGERIAN SPORT.\\nMR SODJE, 37, IS JOINTLY CHARGED WITH ELDER BROTHERS EFE, 44, BRIGHT, 50 AND STEPHEN, 42.\\nAPPEARING AT THE OLD BAILEY EARLIER, ALL FOUR DENIED THE OFFENCE.\\nTHE CHARGE RELATES TO OFFENCES WHICH ALLEGEDLY TOOK PLACE BETWEEN 2008 AND 2014.\\nSAM, FROM KENT, EFE AND BRIGHT, OF GREATER MANCHESTER, AND STEPHEN, FROM BEXLEY, ARE DUE TO STAND TRIAL IN JULY.\\nTHEY WERE ALL RELEASED ON BAIL. \n", + "1 VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SUFFERING THE INJURY WHILE BATTING DURING THE COUNTY CHAMPIONSHIP DRAW WITH SOMERSET ON 4 JUNE.\\nMIDDLESEX HOPE TO HAVE THE AUSTRALIAN BACK FOR THEIR T20 BLAST GAME AGAINST HAMPSHIRE AT LORD'S ON 3 AUGUST.\\nTHE 37-YEAR-OLD HAS SCORED 230 RUNS IN FOUR FIRST-CLASS GAMES THIS SEASON AT AN AVERAGE OF 57.50.\\n\"LOSING ADAM IS NATURALLY A BLOW AS HE CONTRIBUTES SIGNIFICANTLY TO EVERYTHING WE DO,\" DIRECTOR OF CRICKET ANGUS FRASER SAID.\\n\"HIS ABSENCE, HOWEVER, DOES GIVE OPPORTUNITIES TO OTHER PLAYERS WHO ARE DESPERATE TO PLAY IN THE FIRST XI.\\n\"IN THE PAST WE HAVE COPED WELL WITHOUT AN OVERSEAS PLAYER AND I EXPECT US TO DO SO NOW.\"\\nDEFENDING COUNTY CHAMPIONS MIDDLESEX ARE SIXTH IN THE DIVISION ONE TABLE, HAVING DRAWN ALL FOUR OF THEIR MATCHES THIS SEASON.\\nVOGES RETIRED FROM INTERNATIONAL CRICKET IN FEBRUARY WITH A TEST BATTING AVERAGE OF 61.87 FROM 31 INNINGS, SECOND ONLY TO AUSTRALIAN GREAT SIR DONALD BRADMAN'S CAREER AVERAGE OF 99.94 FROM 52 TESTS. \n", + "2 SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRYSIDE BY PHOTOGRAPHER JOSH OLINS WILL APPEAR IN THE JUNE EDITION.\\nIN HER FIRST SITTING FOR A MAGAZINE, THE DUCHESS IS SEEN LOOKING RELAXED AND WEARING CASUAL CLOTHES.\\nTHE SHOOT WAS IN COLLABORATION WITH THE NATIONAL PORTRAIT GALLERY, WHERE TWO IMAGES ARE BEING DISPLAYED IN THE VOGUE 100: A CENTURY OF STYLE EXHIBITION.\\nTHE DUCHESS, WHO HAS A KEEN INTEREST IN PHOTOGRAPHY, HAS BEEN PATRON OF THE NATIONAL PORTRAIT GALLERY SINCE 2012.\\nNICHOLAS CULLINAN, DIRECTOR OF THE NATIONAL PORTRAIT GALLERY, SAID: \"JOSH HAS CAPTURED THE DUCHESS EXACTLY AS SHE IS - FULL OF LIFE, WITH A GREAT SENSE OF HUMOUR, THOUGHTFUL AND INTELLIGENT, AND IN FACT, VERY BEAUTIFUL.\"\\nHE SAID THE IMAGES ALSO ENCAPSULATED WHAT VOGUE HAD DONE OVER THE PAST 100 YEARS - \"TO PAIR THE BEST PHOTOGRAPHERS WITH THE GREAT PERSONALITIES OF THE DAY, IN ORDER TO REFLECT BROADER SHIFTS IN CULTURE AND SOCIETY\".\\nALEXANDRA SHULMAN, EDITOR-IN-CHIEF OF BRITISH VOGUE, SAID: \"TO BE ABLE TO PUBLISH A PHOTOGRAPHIC SHOOT WITH THE DUCHESS OF CAMBRIDGE HAS BEEN ONE OF MY GREATEST AMBITIONS FOR THE MAGAZINE.\"\\nTHE COLLABORATION FOR THE JUNE EDITION HAD RESULTED IN \"A TRUE CELEBRATION OF OUR CENTENARY AS WELL AS A FITTING TRIBUTE TO A YOUNG WOMAN WHOSE INTEREST IN BOTH PHOTOGRAPHY AND THE COUNTRYSIDE IS WELL KNOWN\", SHE SAID.\\nOTHER ROYAL PORTRAITS TO HAVE FEATURED IN THE FASHION MAGAZINE INCLUDE DIANA, PRINCESS OF WALES - WHO GRACED THE COVER FOUR TIMES - AND PRINCESS ANNE.\\nTHE DUCHESS IS TO VISIT THE EXHIBITION AT THE NATIONAL PORTRAIT GALLERY ON WEDNESDAY, KENSINGTON PALACE SAID. \n", + "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003.\\nIT HAS GONE ON TO BE CLOSELY ASSOCIATED WITH OFFENSIVE AND OFTEN ILLEGAL ACTIVITY, INCLUDING INSTANCES WHERE THE IMAGES OF CHILD ABUSE WERE SHARED.\\nIT WAS WIDELY CREDITED AS BEING THE FIRST PLACE WHERE LEAKED IMAGES OF NUDE CELEBRITIES WERE POSTED FOLLOWING 2014'S WELL-PUBLICISED SECURITY BREACH AFFECTING APPLE'S ICLOUD SERVICE. THAT INCIDENT PROMPTED A POLICY CHANGE ON THE SITE.\\nHOWEVER, 4CHAN HAS ALSO BEEN THE RALLYING POINT FOR MANY INSTANCES OF ONLINE ACTIVISM FROM THE LIKES OF ANONYMOUS, THE LOOSELY ORGANIZED HACKTIVISM GROUP.\\nMR POOLE SHARED NEWS OF HIS NEW POSITION ON BLOGGING SITE TUMBLR.\\n\"WHEN MEETING WITH CURRENT AND FORMER GOOGLERS, I CONTINUALLY FIND MYSELF DRAWN TO THEIR INTELLIGENCE, PASSION, AND ENTHUSIASM - AS WELL AS A UNIVERSAL DESIRE TO SHARE IT WITH OTHERS.\"\\n\"I'M ALSO IMPRESSED BY GOOGLE'S COMMITMENT TO ENABLING THESE SAME TALENTED PEOPLE TO TACKLE SOME OF THE WORLD'S MOST INTERESTING AND IMPORTANT PROBLEMS.\\nHE ADDED: \"I CAN'T WAIT TO CONTRIBUTE MY OWN EXPERIENCE FROM A DOZEN YEARS OF BUILDING ONLINE COMMUNITIES, AND TO BEGIN THE NEXT CHAPTER OF MY CAREER AT SUCH AN INCREDIBLE COMPANY.\"\\nMR POOLE STEPPED DOWN AS THE ADMINISTRATOR OF 4CHAN IN JANUARY 2015. NOW HE IS EXPECTED TO TURN HIS ATTENTIONS TO GOOGLE'S SOCIAL NETWORKING EFFORTS.\\nHIS ARRIVAL WAS WELCOMED BY BRADLEY HOROWITZ, THE HEAD OF \"STREAMS, PHOTOS AND SHARING\" AT THE SEARCH GIANT'S FLOUNDERING SOCIAL NETWORK, GOOGLE+.\\n\"I'M THRILLED HE'S JOINING OUR TEAM HERE AT GOOGLE,\" MR HOROWITZ SAID.\\n\"WELCOME CHRIS!Ï»¿\"\\nSEVERAL COMMENTATORS DESCRIBED THE APPOINTMENT AS \"UNEXPECTED\" BUT NOTED THAT MR POOLE'S EXPERTISE WITH SOCIAL MEDIA COULD PROVE USEFUL TO THE SEARCH FIRM.\\nFOLLOW DAVE LEE ON TWITTER @DAVELEEBBC AND ON FACEBOOK \n", + "4 FOUR POLICE OFFICERS WERE INJURED IN THE INCIDENT ON FRIDAY NIGHT.\\nA MAN, AGED 19, AND A BOY, AGED 16, HAVE BEEN CHARGED WITH SIX COUNTS OF AGGRAVATED VEHICLE TAKING.\\nTHEY ARE DUE TO APPEAR BEFORE BELFAST MAGISTRATES' COURT ON MONDAY.\\nTHE 19-YEAR-OLD MAN HAS ALSO BEEN CHARGED WITH DRIVING WHILE DISQUALIFIED AND USING A MOTOR VEHICLE WITHOUT INSURANCE. \n", + "5 THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOUGHT TO HAVE BEEN WALKING WITH A GROUP OF PEOPLE FROM A GRADUATION CEREMONY AT THE CAIRD HALL.\\nTHE INCIDENT TOOK PLACE ON HIGH STREET AT ABOUT 18:00.\\nTHE MAN'S INJURIES ARE BELIEVED NOT TO BE LIFE-THREATENING. THE DRIVER OF THE TAXI IS THOUGHT TO BE UNINJURED. \n", + "6 BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS.\\nTHE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES\\nROSELL, SPEAKING AT A NEWS CONFERENCE AFTER A BARCA BOARD MEETING, INSISTED HE HAD \"ACTED CORRECTLY\".\\nVICE-PRESIDENT JOSEP MARIA BARTOMEU NOW TAKES OVER FROM THE 49-YEAR-OLD ROSELL, WHO CAME TO POWER IN 2010.\\nROSELL'S FUTURE HAS BEEN A REAL SOURCE OF CONCERN EVER SINCE A SPANISH NATIONAL COURT JUDGE ACCEPTED A LAWSUIT THIS WEEK FROM BARCELONA CLUB MEMBER JORDI CASES, WHO ALLEGED THAT THE AMOUNT PAID FOR NEYMAR WAS MORE THAN THE REPORTED FEE.\\nROSELL MAINTAINS THE ACCUSATION IS \"UNFAIR AND RECKLESS\".\\nHE ADDED: \"FOR SOME TIME, MY FAMILY AND MYSELF HAVE SUFFERED THREATS AND ATTACKS IN SILENCE. THESE THREATS AND ATTACKS HAVE MADE ME WONDER IF BEING PRESIDENT MEANS HAVING TO JEOPARDISE MY FAMILY.\\n\"FROM THE BEGINNING, I HAVE SAID THE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES.\"\\nROSELL SAID HE WAS RESIGNING TO SPARE THE CLUB'S BOARD OF DIRECTORS FROM \"UNFAIR ATTACKS\" THAT COULD \"NEGATIVELY AFFECT THEIR MANAGEMENT OR THE IMAGE OF THE CLUB\".\\nHE ADDED: \"IT HAS BEEN AN HONOUR TO SERVE THE BARCELONISTAS. IT'S BEEN A PRIVILEGE TO BE THE PRESIDENT OF FC BARCELONA.\"\\nROSELL WAS NAMED BARCA PRESIDENT IN THE SUMMER OF 2010 AFTER WINNING A LANDSLIDE VOTE TO SUCCEED JOAN LAPORTA, EARNING 61.34% OF A RECORD TURN-OUT OF 57,088 VOTERS.\\nHE HAD ALSO BEEN VICE-PRESIDENT UNDER LAPORTA BEFORE RESIGNING IN 2005. \n", + "7 THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS NEEDED TO SAVE £360M IN THE FIRST YEAR IF THE GOVERNMENT'S NATIONAL FUNDING FORMULA (NFF) PLAN GOES AHEAD.\\nTHE AMOUNT IS THE EQUIVALENT OF 12,857 QUALIFIED TEACHERS, ON AN AVERAGE SALARY OF £28,000.\\nTHE GOVERNMENT SAID LONDON WAS THE HIGHEST FUNDED PART OF THE COUNTRY.\\nIT ADDED THAT UNDER THE PLANS, WHICH ARE UNDER CONSULTATION, INNER-CITY SCHOOLS WOULD BE ALLOCATED 30% MORE MONEY PER PUPIL THAN THE NATIONAL AVERAGE.\\nBUT LONDON COUNCILS, WHICH REPRESENTS THE CITY'S 32 BOROUGHS AND THE CITY, SAID NO SCHOOL WOULD GAIN ENOUGH FUNDING FROM THE NFF TO COMPENSATE FOR INCREASED COST PRESSURES FROM INFLATION, HIGHER PENSION CONTRIBUTIONS AND NATIONAL INSURANCE.\\nMINISTERS SAID THE NEW FORMULA WAS NEEDED TO TACKLE UNEVEN LEVELS OF FUNDING ACROSS ENGLAND, WITH THE BEST FUNDED AREAS GETTING MORE THAN £6,300 PER PUPIL PER YEAR, WHILE THE WORST-FUNDED AVERAGING £4,200.\\nIT SAID THE FUNDING CUT WAS ON TOP OF NATIONAL AUDIT OFFICE FIGURES WHICH SHOWED ENGLAND SCHOOLS FACED AN EIGHT PER CENT REAL-TERMS CUT PER PUPIL BY 2019-20 BECAUSE IT WIDER COST PRESSURES.\\nIN A STATEMENT, LONDON COUNCILS SAID: \"AT A TIME WHEN UK SCHOOLS ARE SEEN AS UNDERPERFORMING BY INTERNATIONAL STANDARDS, AND WHEN BUSINESSES BASED IN LONDON ARE FACING MASSIVE UNCERTAINTY ABOUT RECRUITING SKILLED STAFF, THERE IS AN URGENT NEED TO INVEST IN SCHOOLS IN LONDON AND ACROSS THE REST OF THE COUNTRY.\"\\nIT ADDED: \"WITHOUT THE RIGHT QUALIFICATIONS AND SKILLS, LONDON'S CHILDREN WILL BE UNABLE TO ACCESS JOBS AND CONTRIBUTE TO THE NATIONAL ECONOMY. OVER 60% OF JOBS IN INNER LONDON REQUIRE A DEGREE AND AROUND 45% OF JOBS IN THE REST OF THE CAPITAL REQUIRE A DEGREE.\" \n", + "8 the ex-reading defender denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for nigerian sport.\\nmr sodje, 37, is jointly charged with elder brothers efe, 44, bright, 50 and stephen, 42.\\nappearing at the old bailey earlier, all four denied the offence.\\nthe charge relates to offences which allegedly took place between 2008 and 2014.\\nsam, from kent, efe and bright, of greater manchester, and stephen, from bexley, are due to stand trial in july.\\nthey were all released on bail. \n", + "9 voges was forced to retire hurt on 86 after suffering the injury while batting during the county championship draw with somerset on 4 june.\\nmiddlesex hope to have the australian back for their t20 blast game against hampshire at lord's on 3 august.\\nthe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"losing adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket angus fraser said.\\n\"his absence, however, does give opportunities to other players who are desperate to play in the first xi.\\n\"in the past we have coped well without an overseas player and i expect us to do so now.\"\\ndefending county champions middlesex are sixth in the division one table, having drawn all four of their matches this season.\\nvoges retired from international cricket in february with a test batting average of 61.87 from 31 innings, second only to australian great sir donald bradman's career average of 99.94 from 52 tests. \n", + "10 seven photographs taken in the norfolk countryside by photographer josh olins will appear in the june edition.\\nin her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nthe shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue 100: a century of style exhibition.\\nthe duchess, who has a keen interest in photography, has been patron of the national portrait gallery since 2012.\\nnicholas cullinan, director of the national portrait gallery, said: \"josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nhe said the images also encapsulated what vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nalexandra shulman, editor-in-chief of british vogue, said: \"to be able to publish a photographic shoot with the duchess of cambridge has been one of my greatest ambitions for the magazine.\"\\nthe collaboration for the june edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nother royal portraits to have featured in the fashion magazine include diana, princess of wales - who graced the cover four times - and princess anne.\\nthe duchess is to visit the exhibition at the national portrait gallery on wednesday, kensington palace said. \n", + "11 chris poole - known as \"moot\" online - created the site in 2003.\\nit has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nit was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting apple's icloud service. that incident prompted a policy change on the site.\\nhowever, 4chan has also been the rallying point for many instances of online activism from the likes of anonymous, the loosely organized hacktivism group.\\nmr poole shared news of his new position on blogging site tumblr.\\n\"when meeting with current and former googlers, i continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"i'm also impressed by google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nhe added: \"i can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nmr poole stepped down as the administrator of 4chan in january 2015. now he is expected to turn his attentions to google's social networking efforts.\\nhis arrival was welcomed by bradley horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, google+.\\n\"i'm thrilled he's joining our team here at google,\" mr horowitz said.\\n\"welcome chris!\"\\nseveral commentators described the appointment as \"unexpected\" but noted that mr poole's expertise with social media could prove useful to the search firm.\\nfollow dave lee on twitter @daveleebbc and on facebook \n", + "12 four police officers were injured in the incident on friday night.\\na man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nthey are due to appear before belfast magistrates' court on monday.\\nthe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", + "13 the injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the caird hall.\\nthe incident took place on high street at about 18:00.\\nthe man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured. \n", + "14 barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos.\\nthe signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nrosell, speaking at a news conference after a barca board meeting, insisted he had \"acted correctly\".\\nvice-president josep maria bartomeu now takes over from the 49-year-old rosell, who came to power in 2010.\\nrosell's future has been a real source of concern ever since a spanish national court judge accepted a lawsuit this week from barcelona club member jordi cases, who alleged that the amount paid for neymar was more than the reported fee.\\nrosell maintains the accusation is \"unfair and reckless\".\\nhe added: \"for some time, my family and myself have suffered threats and attacks in silence. these threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"from the beginning, i have said the signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nrosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nhe added: \"it has been an honour to serve the barcelonistas. it's been a privilege to be the president of fc barcelona.\"\\nrosell was named barca president in the summer of 2010 after winning a landslide vote to succeed joan laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nhe had also been vice-president under laporta before resigning in 2005. \n", + "15 the think tank said the city's 1,536 schools needed to save £360m in the first year if the government's national funding formula (nff) plan goes ahead.\\nthe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nthe government said london was the highest funded part of the country.\\nit added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nbut london councils, which represents the city's 32 boroughs and the city, said no school would gain enough funding from the nff to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nministers said the new formula was needed to tackle uneven levels of funding across england, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nit said the funding cut was on top of national audit office figures which showed england schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nin a statement, london councils said: \"at a time when uk schools are seen as underperforming by international standards, and when businesses based in london are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in london and across the rest of the country.\"\\nit added: \"without the right qualifications and skills, london's children will be unable to access jobs and contribute to the national economy. over 60% of jobs in inner london require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", + "\n", + " expected_result \\\n", + "0 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, all \n", + "1 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced. \n", + "2 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100 \n", + "3 The man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports. \n", + "4 Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday. \n", + "5 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought to \n", + "6 Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar. \n", + "7 The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government said \n", + "8 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, all \n", + "9 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced. \n", + "10 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100 \n", + "11 The man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports. \n", + "12 Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday. \n", + "13 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought to \n", + "14 Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar. \n", + "15 The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government said \n", + "\n", + " actual_result \\\n", + "0 Former Liverpool defender Steven Caulker has pleaded guilty to one count of conspiracy to commit fraud and one count of conspiracy to commit money laundering in connection with the alleged manipulation of foreign exchange markets. \n", + "1 Former Australia batsman Adam Voges has been forced to retire from first-class cricket after injuring his ankle during Middlesex's draw with Somerset on 4 June. \n", + "2 The Duchess of Cambridge will appear on the cover of British Vogue for the first time in the magazine's 100th anniversary issue, it has been announced. \n", + "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003. It has gone on to be CLOSELY ASSOCIATED with OFFENSIVE and OFTEN ILLEGAL ACTIVITY, including inSTANCES where the IMAGES of Child ABUSE were shared. It was \n", + "4 A 19-year-old man and a 16-year-old boy have been charged after four police officers were injured in a car crash in Belfast on Friday night. \n", + "5 A man in his 20s has been seriously injured after he was hit by a TAXI in central London. He is thought to have been walking with a group of people from a graduation ceremony at the CAIRD HALL. \n", + "6 Barcelona will be INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS. \n", + "7 London's 1,536 schools need to save £360M in the first year if the government's national funding plan goes ahead, says think tank. \n", + "8 Ex-Reading defender Samuel Eto'o has denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for Nigerian sport. \n", + "9 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season after injuring his hamstring during the county championship draw with Somerset on June 4. \n", + "10 Seven photographs taken in the norfolk countryside by photographer josholins will appear in the june edition of British Vogue. in her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. the shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue \n", + "11 Google has appointed the former administrator of 4chan, one of the world's most notorious online forums, to its social media team. Dave Lee reports. \n", + "12 Four police officers were injured in the incident on Friday night. a man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. they are due to appear before Belfast magistrates' court on Monday. \n", + "13 A pedestrian is injured after being hit by a taxi in central London. The incident took place on high street at about 18:00. the man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured. \n", + "14 Barcelona will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos, the club has announced. \n", + "15 London's 1,536 schools need to save £360m in the first year if the government's national funding formula (nff) plan goes ahead \n", + "\n", + " eval_score pass \n", + "0 0.000000 False \n", + "1 0.301887 False \n", + "2 0.121951 False \n", + "3 0.030769 False \n", + "4 0.294118 False \n", + "5 0.391304 False \n", + "6 0.487805 False \n", + "7 0.486486 False \n", + "8 0.558824 True \n", + "9 0.545455 True \n", + "10 0.945455 True \n", + "11 0.113208 False \n", + "12 1.000000 True \n", + "13 0.591837 True \n", + "14 0.444444 False \n", + "15 0.547945 True " + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.generated_results()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate Report" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag." + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typefail_countpass_countpass_rateminimum_pass_ratepass
0robustnessuppercase800%66%False
1robustnesslowercase2675%60%True
\n", + "
" + ], + "text/plain": [ + " category test_type fail_count pass_count pass_rate minimum_pass_rate \\\n", + "0 robustness uppercase 8 0 0% 66% \n", + "1 robustness lowercase 2 6 75% 60% \n", + "\n", + " pass \n", + "0 False \n", + "1 True " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.report()" + ] + } + ], + "metadata": { + "accelerator": "TPU", + "colab": { + "machine_shape": "hm", + "provenance": [], + "toc_visible": true + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/demo/tutorials/llm_notebooks/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb b/demo/tutorials/llm_notebooks/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb index 8a7815c62..8765288e1 100644 --- a/demo/tutorials/llm_notebooks/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb +++ b/demo/tutorials/llm_notebooks/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "e7PsSmy9sCoR" @@ -11,7 +10,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "3o5sAOfwL5qd" @@ -21,7 +19,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "WJJzt3RWhEc6" @@ -33,7 +30,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "26qXWhCYhHAt" @@ -46,20 +42,18 @@ "cell_type": "code", "execution_count": null, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "azUb114QhOsY", - "outputId": "45a0a692-5076-4fcf-ceb4-b5eb914d9aed" + "id": "azUb114QhOsY" }, "outputs": [], "source": [ - "!pip install \"langtest[evaluate,langchain,transformers]\" " + "!pip install \"langtest[evaluate,langchain,transformers]\"" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "yR6kjOaiheKN" + }, "source": [ "# Harness and Its Parameters\n", "\n", @@ -68,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "lTzSJpMlhgq5" }, @@ -79,7 +73,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "sBcZjwJBhkOw" @@ -93,7 +86,7 @@ "\n", "\n", "| Parameter | Description | \n", - "| - | - | \n", + "| - | - |\n", "|**task** |Task for which the model is to be evaluated (question-answering or summarization)|\n", "| **model** | Specifies the model(s) to be evaluated. Can be a dictionary or a list of dictionaries. Each dictionary should contain 'model' and 'hub' keys. If a path is specified, the dictionary must contain 'model' and 'hub' keys.|\n", "| **data** | The data to be used for evaluation. A dictionary providing flexibility and options for data sources. It should include the following keys: |\n", @@ -104,123 +97,117 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "JFhJ9CcbsKqN" }, "source": [ - "# Model Testing from HuggingFace Hub For Question Answering\n", + "# Hugging Face Text-Generation Model Testing\n", "\n", - "In this section, we dive into testing of HuggingFace Hub models in Question Answering task.\n", - "\n", - "LangTest supports robustness tests for LLM testing for now." + "In this section, we dive into testing of Hugging Face models in summarization." ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { - "id": "kKgXC7cvuyar" - }, - "source": [ - "### Set environment for HuggingFace Hub" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MHqlSjFLuy7o" + "id": "ksI05WvIf8TV" }, - "outputs": [], "source": [ - "import os\n", - "os.environ[\"HUGGINGFACEHUB_API_TOKEN\"] = \"\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## BoolQ-test-tiny dataset testing" + "## XSum-test-tiny dataset testing" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { - "id": "swaYPW-wPlku" + "id": "65WtgltZf8TV" }, "source": [ - "### Setup and Configure Harness" + "### Setup and configure harness" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, - "id": "p_5nO14bvTzt", - "outputId": "cee6c5f4-6f32-4f72-e9db-440a410b59c7" + "id": "8P2DuIyLf8TV", + "outputId": "b0230c98-4d0e-4913-8ba6-41561dddf3a9" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test Configuration : \n", + " {\n", + " \"model_parameters\": {\n", + " \"max_new_tokens\": 32\n", + " },\n", + " \"tests\": {\n", + " \"defaults\": {\n", + " \"min_pass_rate\": 1.0\n", + " },\n", + " \"robustness\": {\n", + " \"lowercase\": {\n", + " \"min_pass_rate\": 0.7\n", + " },\n", + " \"add_typo\": {\n", + " \"min_pass_rate\": 0.7\n", + " }\n", + " }\n", + " }\n", + "}\n" + ] + } + ], "source": [ - "harness = Harness(task=\"question-answering\", model={\"model\": \"google/flan-t5-small\",\"hub\": \"huggingface-inference-api\"}, data={\"data_source\": 'BoolQ-test-tiny'})" + "harness = Harness(\n", + " task=\"summarization\",\n", + " model={\"model\": \"facebook/opt-1.3b\", \"hub\":\"huggingface\"},\n", + " data={\"data_source\": 'XSum-test-tiny'},\n", + " config={\n", + " 'model_parameters': {\n", + " 'max_new_tokens': 32\n", + " },\n", + "\n", + " 'tests': {\n", + " 'defaults':{\n", + " 'min_pass_rate': 1.00\n", + " },\n", + "\n", + " 'robustness':{\n", + " 'lowercase': {'min_pass_rate': 0.70},\n", + " 'add_typo': {'min_pass_rate': 0.70}\n", + " }\n", + " }\n", + " })" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { - "id": "jWPAw9q0PwD1" + "id": "Gfbv90Lef8TV" }, "source": [ - "We have specified task as QA, hub as `huggingface-inference-api` and model as `google/flan-t5-small`, `google/flan-t5-xl`, `stabilityai/stablelm-tuned-alpha-3b`, or `databricks/dolly-v2-3b`.\n", + "We have specified task as summarization, hub as Hugging Face and model as `facebook/opt-1.3b`. Model can be accessed [here](https://huggingface.co/facebook/opt-1.3b)\n", "\n", - "For dataset we used `BoolQ-test-tiny` which includes 50 lines from BoolQ-test. Other available datasets are:\n", "\n", - "#### BoolQ\n", - "* `BoolQ-test-tiny`\n", - "* `BoolQ-test`\n", - "* `BoolQ-combined`\n", - "#### NQ-open\n", - "* `NQ-open-test`\n", - "* `NQ-open-combined`\n", - "* `NQ-open-test-tiny`\n", - "#### TruthfulQA\n", - "* `TruthfulQA-combined`\n", - "* `TruthfulQA-test`\n", - "* `TruthfulQA-tiny`\n", - "#### MMLU\n", - "* `MMLU-test`\n", - "* `MMLU-test-tiny`\n", - "#### OpenBookQA\n", - "* `OpenBookQA-test`\n", - "* `OpenBookQA-test-tiny`\n", - "#### QUAC\n", - "* `Quac-test`\n", - "* `Quac-test-tiny`\n", - "#### NarrativeQA\n", - "* `NarrativeQA-test`\n", - "* `NarrativeQA-test-tiny`\n", - "#### HellaSwag\n", - "* `HellaSwag-test`\n", - "* `HellaSwag-test-tiny`\n", - "#### BBQ\n", - "* `BBQ-test`\n", - "* `BBQ-test-tiny`\n" + "For dataset we used XSum-test-tiny which includes 50 lines from XSum-test. Available datasets for summarization are:\n", + "\n", + "* `XSum-test`\n", + "* `XSum-test-tiny`" ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "cq-lnk8ef8TV" + }, "source": [ - "For tests we used lowercase and uppercase. Other available robustness tests for QA task are:\n", + "For tests we used lowercase and add_typo. Other available robustness tests for summarization task are:\n", + "\n", "* `add_context`\n", "* `add_contraction`\n", "* `add_punctuation`\n", @@ -240,15 +227,9 @@ "* `multiple_perturbations`\n", "* `adjective_synonym_swap`\n", "* `adjective_antonym_swap`\n", - "* `strip_all_punctuation`" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Available Bias tests for QA task are:\n", + "* `strip_all_punctuation`\n", + "\n", + "Available Bias tests for summarization task are:\n", "\n", "* `replace_to_male_pronouns`\n", "* `replace_to_female_pronouns`\n", @@ -275,7 +256,7 @@ "* `replace_to_jain_names`\n", "* `replace_to_buddhist_names`\n", "\n", - "Available Representation tests for QA task are:\n", + "Available Representation tests for summarization task are:\n", "\n", "* `min_gender_representation_count`\n", "* `min_ethnicity_name_representation_count`\n", @@ -287,7 +268,7 @@ "* `min_country_economic_representation_proportion`\n", "\n", "\n", - "Available Accuracy tests for QA task are:\n", + "Available Accuracy tests for summarization task are:\n", "\n", "* `min_exact_match_score`\n", "* `min_bleu_score`\n", @@ -297,7 +278,7 @@ "* `min_rougeLsum_score`\n", "\n", "\n", - "Available Fairness tests for QA task are:\n", + "Available Fairness tests for summarization task are:\n", "\n", "* `max_gender_rouge1_score`\n", "* `max_gender_rouge2_score`\n", @@ -310,424 +291,1606 @@ ] }, { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also set prompts and other model parameters in config. Possible parameters are:\n", - "* `user_promt:` Promt to be given to the model.\n", - "* `temperature:` Temperature of the model.\n", - "* `max_length:` Maximum number of output tokens allowed for model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Q-ghKYLpPqtt", - "outputId": "20f1a53b-2745-4028-bbe7-8ce4dccb745e" - }, - "outputs": [], - "source": [ - "harness.configure({\n", - " 'model_parameters': {\n", - " 'temperature': 0,\n", - " 'max_length': 64\n", - " },\n", - " \n", - " 'tests': {\n", - " 'defaults':{\n", - " 'min_pass_rate': 1.00\n", - " },\n", - "\n", - " 'robustness':{\n", - " 'lowercase': {'min_pass_rate': 0.70},\n", - " 'uppercase': {'min_pass_rate': 0.70}\n", - " }\n", - " }\n", - "})" - ] - }, - { - "attachments": {}, "cell_type": "markdown", "metadata": { - "id": "ZPU46A7WigFr" + "id": "z1sAo1_pf8TW" }, "source": [ - "Here we have configured the harness to perform two robustness tests (uppercase and lowercase) and defined the minimum pass rate for each test." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "➤ You can adjust the level of transformation in the sentence by using the \"`prob`\" parameter, which controls the proportion of words to be changed during robustness tests.\n", - "\n", - "➤ **NOTE** : \"`prob`\" defaults to 1.0, which means all words will be transformed.\n", - "```\n", - "harness.configure(\n", - "{\n", - " 'tests': {\n", - " 'defaults': {'min_pass_rate': 0.65},\n", - " 'robustness': {\n", - " 'lowercase': {'min_pass_rate': 0.66, 'prob': 0.50}, \n", - " 'uppercase':{'min_pass_rate': 0.60, 'prob': 0.70},\n", - " }\n", - " }\n", - "})\n", - "\n", - "```" + "### Generating the Test Cases" ] }, { - "attachments": {}, - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 22, "metadata": { - "id": "i6kPvA13F7cr" + "id": "Pr1mWxDkf8TW" }, + "outputs": [], "source": [ - "\n", - "### Generating the test cases." + "harness.data = harness.data[:10]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, - "id": "mdNH3wCKF9fn", - "outputId": "cd348490-7ade-40fa-d870-dc059f5aa647" + "id": "U6vksWamf8TW", + "outputId": "782700df-547f-44f7-f42e-488105f805dd" }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Generating testcases...: 100%|██████████| 1/1 [00:00<00:00, 6909.89it/s]\n" + ] + }, + { + "data": { + "text/plain": [] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "harness.generate()" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "nyjDdYLeGCmM" - }, - "source": [ - "harness.generate() method automatically generates the test cases (based on the provided configuration)" - ] - }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 641 - }, - "id": "c0jL1_G7F_p6", - "outputId": "502c1525-9000-4041-823b-3b04f6650892" - }, - "outputs": [], - "source": [ - "harness.testcases()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "NOJ8BAU2GGzd" - }, - "source": [ - "harness.testcases() method displays the produced test cases in form of a pandas data frame." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "3CwhQw6hGR9S" - }, - "source": [ - "### Running the tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "height": 676 }, - "id": "aguX6-aFGOnP", - "outputId": "bb014811-522b-4f07-fa8a-bf3d1c906d7f" - }, - "outputs": [], - "source": [ - "harness.run()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "191O2oaUGWrH" - }, - "source": [ - "Called after harness.generate() and is to used to run all the tests. Returns a pass/fail flag for each test." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XDbd1mpREWR5" - }, - "outputs": [], - "source": [ - "harness.generated_results()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "TKB8Rsr2GZME" - }, - "source": [ - "This method returns the generated results in the form of a pandas dataframe, which provides a convenient and easy-to-use format for working with the test results. You can use this method to quickly identify the test cases that failed and to determine where fixes are needed." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "PBSlpWnUU55G" + "id": "FcXeJY40f8TW", + "outputId": "6c547433-13ae-4148-bb85-a3aaee545506" }, - "source": [ - "### Final Results" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag." - ] - }, - { + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typeoriginaltest_case
0robustnesslowercaseThe ex-Reading defender denied fraudulent trad...the ex-reading defender denied fraudulent trad...
1robustnesslowercaseVoges was forced to retire hurt on 86 after su...voges was forced to retire hurt on 86 after su...
2robustnesslowercaseSeven photographs taken in the Norfolk country...seven photographs taken in the norfolk country...
3robustnesslowercaseChris Poole - known as \"moot\" online - created...chris poole - known as \"moot\" online - created...
4robustnesslowercaseFour police officers were injured in the incid...four police officers were injured in the incid...
5robustnesslowercaseThe injured pedestrian - a young man - is thou...the injured pedestrian - a young man - is thou...
6robustnesslowercaseBarca will be investigated for alleged misappr...barca will be investigated for alleged misappr...
7robustnesslowercaseThe think tank said the city's 1,536 schools n...the think tank said the city's 1,536 schools n...
8robustnesslowercaseHis 110 means he has scored 323 runs in a week...his 110 means he has scored 323 runs in a week...
9robustnesslowercaseMs Kendall told the BBC Labour risked sending ...ms kendall told the bbc labour risked sending ...
10robustnessadd_typoThe ex-Reading defender denied fraudulent trad...The ex-Reading defender denied fraudulent trad...
11robustnessadd_typoVoges was forced to retire hurt on 86 after su...Voges was forced to retire hurt on 86 after su...
12robustnessadd_typoSeven photographs taken in the Norfolk country...Seven photographs taken in the Norfolk country...
13robustnessadd_typoChris Poole - known as \"moot\" online - created...Chris Poole - known as \"moot\" online - created...
14robustnessadd_typoFour police officers were injured in the incid...Four police officers were injured in the incid...
15robustnessadd_typoThe injured pedestrian - a young man - is thou...The injured pedestroan - a young man - is thou...
16robustnessadd_typoBarca will be investigated for alleged misappr...Barca will be investigated for alleged misappr...
17robustnessadd_typoThe think tank said the city's 1,536 schools n...The think tank said the city's 1,536 schools n...
18robustnessadd_typoHis 110 means he has scored 323 runs in a week...His 110 means he has scored 323 runs in a week...
19robustnessadd_typoMs Kendall told the BBC Labour risked sending ...Ms Kendall told the BBC Labour risked sending ...
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ], + "text/plain": [ + " category test_type original \\\n", + "0 robustness lowercase The ex-Reading defender denied fraudulent trad... \n", + "1 robustness lowercase Voges was forced to retire hurt on 86 after su... \n", + "2 robustness lowercase Seven photographs taken in the Norfolk country... \n", + "3 robustness lowercase Chris Poole - known as \"moot\" online - created... \n", + "4 robustness lowercase Four police officers were injured in the incid... \n", + "5 robustness lowercase The injured pedestrian - a young man - is thou... \n", + "6 robustness lowercase Barca will be investigated for alleged misappr... \n", + "7 robustness lowercase The think tank said the city's 1,536 schools n... \n", + "8 robustness lowercase His 110 means he has scored 323 runs in a week... \n", + "9 robustness lowercase Ms Kendall told the BBC Labour risked sending ... \n", + "10 robustness add_typo The ex-Reading defender denied fraudulent trad... \n", + "11 robustness add_typo Voges was forced to retire hurt on 86 after su... \n", + "12 robustness add_typo Seven photographs taken in the Norfolk country... \n", + "13 robustness add_typo Chris Poole - known as \"moot\" online - created... \n", + "14 robustness add_typo Four police officers were injured in the incid... \n", + "15 robustness add_typo The injured pedestrian - a young man - is thou... \n", + "16 robustness add_typo Barca will be investigated for alleged misappr... \n", + "17 robustness add_typo The think tank said the city's 1,536 schools n... \n", + "18 robustness add_typo His 110 means he has scored 323 runs in a week... \n", + "19 robustness add_typo Ms Kendall told the BBC Labour risked sending ... \n", + "\n", + " test_case \n", + "0 the ex-reading defender denied fraudulent trad... \n", + "1 voges was forced to retire hurt on 86 after su... \n", + "2 seven photographs taken in the norfolk country... \n", + "3 chris poole - known as \"moot\" online - created... \n", + "4 four police officers were injured in the incid... \n", + "5 the injured pedestrian - a young man - is thou... \n", + "6 barca will be investigated for alleged misappr... \n", + "7 the think tank said the city's 1,536 schools n... \n", + "8 his 110 means he has scored 323 runs in a week... \n", + "9 ms kendall told the bbc labour risked sending ... \n", + "10 The ex-Reading defender denied fraudulent trad... \n", + "11 Voges was forced to retire hurt on 86 after su... \n", + "12 Seven photographs taken in the Norfolk country... \n", + "13 Chris Poole - known as \"moot\" online - created... \n", + "14 Four police officers were injured in the incid... \n", + "15 The injured pedestroan - a young man - is thou... \n", + "16 Barca will be investigated for alleged misappr... \n", + "17 The think tank said the city's 1,536 schools n... \n", + "18 His 110 means he has scored 323 runs in a week... \n", + "19 Ms Kendall told the BBC Labour risked sending ... " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.testcases()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FMrwqFDqf8TX" + }, + "source": [ + "### Running the tests" + ] + }, + { "cell_type": "code", - "execution_count": null, + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5ONenQsVf8TX", + "outputId": "2574c825-0344-44ad-d44e-06dc96584e18" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running testcases... : 100%|██████████| 20/20 [03:02<00:00, 9.14s/it]\n" + ] + }, + { + "data": { + "text/plain": [] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "harness.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9rYevMn1f8TX" + }, + "source": [ + "### Generated Results" + ] + }, + { + "cell_type": "code", + "execution_count": 26, "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 112 + "height": 1000, + "referenced_widgets": [ + "336a4708137f4fe4b9aa551ab127227d", + "44086a4a5aaf4e4a912f4b26551d8f51", + "4e0a3727b36d43f2ba34f1ca24df9be5", + "d2078e9c1dfe4e7aa97332baf9473dd5", + "5e731174370041d4bbe536886f139948", + "ca1ba76b0dd04d4885a42091872d6a62", + "f594ac19e3a344d2b64aa55af953d115", + "b952c931792240e6b23bec6dd3ec49d5", + "796e43018c574e8b89e32ab7d9e29b84", + "daa0487fb8e24f76afc361bb1202479e", + "8d3102653db94e5aaf576c6059c98960" + ] }, - "id": "gp57HcF9yxi7", - "outputId": "b893072f-102a-45a6-be03-d737996e659c" + "id": "j9j92IJvf8TX", + "outputId": "cea5414d-0efb-4d8e-a4d0-9e938f192af0" }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "336a4708137f4fe4b9aa551ab127227d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading builder script: 0%| | 0.00/6.27k [00:00\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typeoriginaltest_caseexpected_resultactual_resulteval_scorepass
0robustnesslowercaseThe ex-Reading defender denied fraudulent trad...the ex-reading defender denied fraudulent trad...The ex-Reading defender denied fraudulent tra...the ex-reading defender denied fraudulent tra...0.978723True
1robustnesslowercaseVoges was forced to retire hurt on 86 after su...voges was forced to retire hurt on 86 after su...Voges was forced to retire hurt on 86 after s...vernon morgan has been ruled out of the first ...0.000000False
2robustnesslowercaseSeven photographs taken in the Norfolk country...seven photographs taken in the norfolk country...The duchess is seen looking relaxed and weari...the duchess of cambridge is photographed in t...0.416667False
3robustnesslowercaseChris Poole - known as \"moot\" online - created...chris poole - known as \"moot\" online - created... Chris Poole - known as \"moot\" online - creat...ian schwartz - known as \"the hacker\" - is a fo...0.044444False
4robustnesslowercaseFour police officers were injured in the incid...four police officers were injured in the incid...The man was arrested on suspicion of aggravat...the man and boy were arrested after a police ...0.072727False
5robustnesslowercaseThe injured pedestrian - a young man - is thou...the injured pedestrian - a young man - is thou...The injured pedestrian - a young man - is tho...the injured pedestrian - a young man - is tho...0.981132True
6robustnesslowercaseBarca will be investigated for alleged misappr...barca will be investigated for alleged misappr...The Spanish judge has accepted a lawsuit from...0.000000False
7robustnesslowercaseThe think tank said the city's 1,536 schools n...the think tank said the city's 1,536 schools n...The think tank said the city's 1,536 schools ...the think tank said the city's 1,536 schools ...1.000000True
8robustnesslowercaseHis 110 means he has scored 323 runs in a week...his 110 means he has scored 323 runs in a week...England batsman Roy was named man of the matc...urn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:0.000000False
9robustnesslowercaseMs Kendall told the BBC Labour risked sending ...ms kendall told the bbc labour risked sending ...ÂThe BBC's political correspondent Ross Hawkin...               Â0.000000False
10robustnessadd_typoThe ex-Reading defender denied fraudulent trad...The ex-Reading defender denied fraudulent trad...The ex-Reading defender denied fraudulent tra...The ex-Reading defender denied fraudulent tra...1.000000True
11robustnessadd_typoVoges was forced to retire hurt on 86 after su...Voges was forced to retire hurt on 86 after su...Voges was forced to retire hurt on 86 after s...Voges was forced to retire hurt on 86 after s...0.901961True
12robustnessadd_typoSeven photographs taken in the Norfolk country...Seven photographs taken in the Norfolk country...The duchess is seen looking relaxed and weari...The duchess is seen looking relaxed and weari...1.000000True
13robustnessadd_typoChris Poole - known as \"moot\" online - created...Chris Poole - known as \"moot\" online - created... Chris Poole - known as \"moot\" online - creat... Chris Poole - known as \"moot\" online - creat...1.000000True
14robustnessadd_typoFour police officers were injured in the incid...Four police officers were injured in the incid...The man was arrested on suspicion of aggravat...The man was arrested on suspicion of aggravat...1.000000True
15robustnessadd_typoThe injured pedestrian - a young man - is thou...The injured pedestroan - a young man - is thou...The injured pedestrian - a young man - is tho...The injured pedestroan - a young man - is tho...0.880000True
16robustnessadd_typoBarca will be investigated for alleged misappr...Barca will be investigated for alleged misappr...The Spanish judge has accepted a lawsuit from...Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?0.000000False
17robustnessadd_typoThe think tank said the city's 1,536 schools n...The think tank said the city's 1,536 schools n...The think tank said the city's 1,536 schools ...The think tank said the city's 1,536 schools ...1.000000True
18robustnessadd_typoHis 110 means he has scored 323 runs in a week...His 110 means he has scored 323 runs in a week...England batsman Roy was named man of the matc...England batsman Roy was named man of the matc...1.000000True
19robustnessadd_typoMs Kendall told the BBC Labour risked sending ...Ms Kendall told the BBC Labour risked sending ...ÂThe BBC's political correspondent Ross Hawkin...ÂThe BBC's political correspondent Ross Hawkin...1.000000True
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + " \n" + ], + "text/plain": [ + " category test_type original \\\n", + "0 robustness lowercase The ex-Reading defender denied fraudulent trad... \n", + "1 robustness lowercase Voges was forced to retire hurt on 86 after su... \n", + "2 robustness lowercase Seven photographs taken in the Norfolk country... \n", + "3 robustness lowercase Chris Poole - known as \"moot\" online - created... \n", + "4 robustness lowercase Four police officers were injured in the incid... \n", + "5 robustness lowercase The injured pedestrian - a young man - is thou... \n", + "6 robustness lowercase Barca will be investigated for alleged misappr... \n", + "7 robustness lowercase The think tank said the city's 1,536 schools n... \n", + "8 robustness lowercase His 110 means he has scored 323 runs in a week... \n", + "9 robustness lowercase Ms Kendall told the BBC Labour risked sending ... \n", + "10 robustness add_typo The ex-Reading defender denied fraudulent trad... \n", + "11 robustness add_typo Voges was forced to retire hurt on 86 after su... \n", + "12 robustness add_typo Seven photographs taken in the Norfolk country... \n", + "13 robustness add_typo Chris Poole - known as \"moot\" online - created... \n", + "14 robustness add_typo Four police officers were injured in the incid... \n", + "15 robustness add_typo The injured pedestrian - a young man - is thou... \n", + "16 robustness add_typo Barca will be investigated for alleged misappr... \n", + "17 robustness add_typo The think tank said the city's 1,536 schools n... \n", + "18 robustness add_typo His 110 means he has scored 323 runs in a week... \n", + "19 robustness add_typo Ms Kendall told the BBC Labour risked sending ... \n", + "\n", + " test_case \\\n", + "0 the ex-reading defender denied fraudulent trad... \n", + "1 voges was forced to retire hurt on 86 after su... \n", + "2 seven photographs taken in the norfolk country... \n", + "3 chris poole - known as \"moot\" online - created... \n", + "4 four police officers were injured in the incid... \n", + "5 the injured pedestrian - a young man - is thou... \n", + "6 barca will be investigated for alleged misappr... \n", + "7 the think tank said the city's 1,536 schools n... \n", + "8 his 110 means he has scored 323 runs in a week... \n", + "9 ms kendall told the bbc labour risked sending ... \n", + "10 The ex-Reading defender denied fraudulent trad... \n", + "11 Voges was forced to retire hurt on 86 after su... \n", + "12 Seven photographs taken in the Norfolk country... \n", + "13 Chris Poole - known as \"moot\" online - created... \n", + "14 Four police officers were injured in the incid... \n", + "15 The injured pedestroan - a young man - is thou... \n", + "16 Barca will be investigated for alleged misappr... \n", + "17 The think tank said the city's 1,536 schools n... \n", + "18 His 110 means he has scored 323 runs in a week... \n", + "19 Ms Kendall told the BBC Labour risked sending ... \n", + "\n", + " expected_result \\\n", + "0 The ex-Reading defender denied fraudulent tra... \n", + "1 Voges was forced to retire hurt on 86 after s... \n", + "2 The duchess is seen looking relaxed and weari... \n", + "3 Â Chris Poole - known as \"moot\" online - creat... \n", + "4 The man was arrested on suspicion of aggravat... \n", + "5 The injured pedestrian - a young man - is tho... \n", + "6 The Spanish judge has accepted a lawsuit from... \n", + "7 The think tank said the city's 1,536 schools ... \n", + "8 England batsman Roy was named man of the matc... \n", + "9 ÂThe BBC's political correspondent Ross Hawkin... \n", + "10 The ex-Reading defender denied fraudulent tra... \n", + "11 Voges was forced to retire hurt on 86 after s... \n", + "12 The duchess is seen looking relaxed and weari... \n", + "13 Â Chris Poole - known as \"moot\" online - creat... \n", + "14 The man was arrested on suspicion of aggravat... \n", + "15 The injured pedestrian - a young man - is tho... \n", + "16 The Spanish judge has accepted a lawsuit from... \n", + "17 The think tank said the city's 1,536 schools ... \n", + "18 England batsman Roy was named man of the matc... \n", + "19 ÂThe BBC's political correspondent Ross Hawkin... \n", + "\n", + " actual_result eval_score pass \n", + "0 the ex-reading defender denied fraudulent tra... 0.978723 True \n", + "1 vernon morgan has been ruled out of the first ... 0.000000 False \n", + "2 the duchess of cambridge is photographed in t... 0.416667 False \n", + "3 ian schwartz - known as \"the hacker\" - is a fo... 0.044444 False \n", + "4 the man and boy were arrested after a police ... 0.072727 False \n", + "5 the injured pedestrian - a young man - is tho... 0.981132 True \n", + "6 0.000000 False \n", + "7 the think tank said the city's 1,536 schools ... 1.000000 True \n", + "8 urn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn:nbn: 0.000000 False \n", + "9 Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â 0.000000 False \n", + "10 The ex-Reading defender denied fraudulent tra... 1.000000 True \n", + "11 Voges was forced to retire hurt on 86 after s... 0.901961 True \n", + "12 The duchess is seen looking relaxed and weari... 1.000000 True \n", + "13 Â Chris Poole - known as \"moot\" online - creat... 1.000000 True \n", + "14 The man was arrested on suspicion of aggravat... 1.000000 True \n", + "15 The injured pedestroan - a young man - is tho... 0.880000 True \n", + "16 Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â?Â? 0.000000 False \n", + "17 The think tank said the city's 1,536 schools ... 1.000000 True \n", + "18 England batsman Roy was named man of the matc... 1.000000 True \n", + "19 ÂThe BBC's political correspondent Ross Hawkin... 1.000000 True " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "harness.report()" + "harness.generated_results()" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { - "id": "5N0cKfKiLsiQ" + "id": "Me8qllMBf8TX" }, "source": [ - "#### NQ-open-test dataset testing" + "### Generate Report" ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "GxNSaFEwf8TX" + }, "source": [ - "We can also use another dataset, NQ-open for testing the model." + "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": { "colab": { - "base_uri": "https://localhost:8080/" + "base_uri": "https://localhost:8080/", + "height": 112 }, - "id": "oDh3Zaa9EDfZ", - "outputId": "10443ac6-8c92-4e86-ef4e-7050962c4255" + "id": "MUEJ2vnmf8TX", + "outputId": "6f1970d2-d3a8-4879-e545-22d3c8605176" }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
categorytest_typefail_countpass_countpass_rateminimum_pass_ratepass
0robustnesslowercase7330%70%False
1robustnessadd_typo1990%70%True
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ], + "text/plain": [ + " category test_type fail_count pass_count pass_rate minimum_pass_rate \\\n", + "0 robustness lowercase 7 3 30% 70% \n", + "1 robustness add_typo 1 9 90% 70% \n", + "\n", + " pass \n", + "0 False \n", + "1 True " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "harness = Harness(task=\"question-answering\", model={\"model\": \"google/flan-t5-small\",\"hub\":\"huggingface-inference-api\"}, data={\"data_source\": 'NQ-open-test-tiny'})" + "harness.report()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ZnLWJkPVEDmg", - "outputId": "92ca0633-a1c6-4de3-f9fd-c77e6bcb5374" + "id": "APH6t3Jef8TV" }, - "outputs": [], "source": [ - "harness.configure({\n", - " 'model_parameters': {\n", - " 'max_length': 512\n", - " },\n", - " 'tests': {\n", - " 'defaults':{\n", - " 'min_pass_rate': 1.00\n", - " },\n", + "# HF Model Testing For Summarization\n", "\n", - " 'robustness':{\n", - " 'lowercase': {'min_pass_rate': 0.70},\n", - " 'uppercase': {'min_pass_rate': 0.70}\n", - " }\n", - " }\n", - "})" + "In this section, we dive into testing of HF models in question answering task." ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "A3U0kM62EG6B", - "outputId": "1ad54c30-3371-41b6-e85c-4dc69ffcd8aa" + "id": "LDYv-iKWf8TL" }, - "outputs": [], "source": [ - "harness.generate()" + "## BoolQ-test-tiny dataset testing" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 423 - }, - "id": "0ipwMsy0EHYY", - "outputId": "a3e9c1fb-22dc-416d-8cf6-043ac4b28432" + "id": "swaYPW-wPlku" }, - "outputs": [], "source": [ - "harness.testcases()" + "### Setup and Configure Harness" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "colab": { - "base_uri": "https://localhost:8080/" + "base_uri": "https://localhost:8080/", + "height": 571, + "referenced_widgets": [ + "f16fa357e41946eeab0a00930b898379", + "4ac2635d7b954f1bb8658490ce4b102c", + "1b8a289aeb51470a8c1a2950f8c57aa7", + "5b650dafed624813b692c84d24657cfe", + "29fbd43db74740e4b9af627253cc0bdb", + "7b6b5d4199cc4f8785c8fabcae9a8036", + "cf93f2c9cb294637aef9766c794f7fb2", + "1a1f78619dd144baa168713c59ffc737", + "4a5677beefb242c583ce0bd3af5e2a95", + "3e86321f68d54720803a357c49212d60", + "9cd8390472734d8f89ad8ea058127fd0", + "5ef205051e974687ae134cc575d8b41e", + "fbfbdc0a14c04e2fb03a64ad467b95d5", + "4e10f045a75f4665bac2cf9a539f7b0b", + "923b976ddb594ddaa37578088f5ff87c", + "4400c80adb044860a2607681da776437", + "674dc3d1f55841ea9cd682c863df6d9f", + "afb8a9e85c4a40dabf112268d7e9196c", + "1d28ccffbccf43e080aa39c6a6f53b1c", + "8633a4a4ee2543caa02cfdbc4f21846d", + "0341ee74e543493ca3765809e41611e3", + "c04797768f1149d7a97adb994b45c09e", + "b8a4848be055497da44a449e7f2c672b", + "a3b4eb7bb33c4d39966c4ffdad654170", + "325256d7089640e694151e389d7a2df5", + "c614a0ee97364305a835f3a902d6e939", + "a61f506ecd39454c9c108eb1f53d268a", + "5c3c82880ed44efbb03cae396bc72918", + "68591dcca5d84e5fac075a2d0f4d0cdf", + "f7f320c30f4245c89544a1959ea323c1", + "0fd6422fbb7747aa9983aa0ad77390a3", + "8b1ab800a4b6466b8f982c57653bb456", + "44201622077d46b3beb20b665fa2e191", + "e3772ec851be44cf838067837104f36d", + "e00a999af46a4d898f3f9ca1083b68d1", + "5ea1c82925dc47cc9b672cdb92029f74", + "8b3333afb5d54c65b6cfce98f97d16b6", + "98e76b406ed6487094f78d5559d2d764", + "1fa614546c0848dab5984d98ab2aff6e", + "6021523b6760479f89367d38fffe06f8", + "ec0afe2bcb234063a15525bb4d2dbee2", + "b546d25c1f1e4f6ea18d33af8a8d4359", + "a0383864826b4d1db8e080510e1853c4", + "4d9d1da4041d4a529df11e08c9089142", + "e92f799378dd4be697cb05ad64e070ab", + "9716d1570ef74ed0a67660f9379c278c", + "7cb46e8a3c974d24864314fa17b79d85", + "1d6b9a06881d451dbbc342917bb3bdfc", + "f7f9a53cdbee4497955bec5d2441e56c", + "43580b4168f442139012428ba0e6c759", + "9ff53ca79e9a4379b51b61773571a159", + "294762c36d48446890ec7bca8a1f5591", + "7cd725d017384a90be458e35799b1279", + "8e3f068f9ad441bda11ac18f216b1f0d", + "fccb920a50114e859e0d3b9c176f53d5", + "c1987349992a40e7833027843ec52c79", + "11dd94d811424b41ab7c332074aecbfa", + "723ecc2c963541828a408f829b4b883c", + "cde87e4029ce469a9d048742ea06cea9", + "86a1bf7601b949ecbc7fd8b53936da0c", + "2304b5d9f4904ce4b133943a6770d276", + "ae8a33f996d64ab3a409a9d7d8599ba8", + "bc61b35936984566821f40758b155069", + "8cbad88f229949149c894e1a8a71f9de", + "2eb51073cd02468b946163f433f3962a", + "95cf3f8f98124519b9d9224b94743959", + "6c3ab47327624fc0877b9f8bb63bb167", + "c8bca45bffcb407aa9e287423323930a", + "9e7ef7a229c44ebb9903c2d2bc7a9b07", + "098411bb18d9448b9a9592294e1ce81b", + "89082c41aee84ab99e411ab5db986302", + "145895cb381e48a7beb2bccbb14f0795", + "4bb819239c4748fb9de90f5c66aab70d", + "4c109f682d4f4d699f3f509248bbf804", + "2d690e77373945ba8652dd260b760bcf", + "e9e8db2529d7452b946c3ca0151d6d55", + "f35ce4f6d998440e8f04a3cbc8394e11" + ] }, - "id": "0Nic5HRZEJu5", - "outputId": "dbbf911a-413e-479c-996b-98430920f0b5" + "id": "p_5nO14bvTzt", + "outputId": "4b690de6-958c-4972-89b2-4960e84b4d37" }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f16fa357e41946eeab0a00930b898379", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading (…)lve/main/config.json: 0%| | 0.00/653 [00:00\n", + "\n", + "
\n", + "
\n", "\n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + " \n" ], "text/plain": [ - " category test_type original \\\n", - "0 robustness uppercase The ex-Reading defender denied fraudulent trad... \n", - "1 robustness uppercase Voges was forced to retire hurt on 86 after su... \n", - "2 robustness uppercase Seven photographs taken in the Norfolk country... \n", - "3 robustness uppercase Chris Poole - known as \"moot\" online - created... \n", - "4 robustness uppercase Four police officers were injured in the incid... \n", - "5 robustness uppercase The injured pedestrian - a young man - is thou... \n", - "6 robustness uppercase Barca will be investigated for alleged misappr... \n", - "7 robustness uppercase The think tank said the city's 1,536 schools n... \n", - "8 robustness lowercase The ex-Reading defender denied fraudulent trad... \n", - "9 robustness lowercase Voges was forced to retire hurt on 86 after su... \n", - "10 robustness lowercase Seven photographs taken in the Norfolk country... \n", - "11 robustness lowercase Chris Poole - known as \"moot\" online - created... \n", - "12 robustness lowercase Four police officers were injured in the incid... \n", - "13 robustness lowercase The injured pedestrian - a young man - is thou... \n", - "14 robustness lowercase Barca will be investigated for alleged misappr... \n", - "15 robustness lowercase The think tank said the city's 1,536 schools n... \n", + " category test_type original_context \\\n", + "0 robustness lowercase 20 euro note -- Until now there has been only ... \n", + "1 robustness lowercase 2018–19 UEFA Champions League -- The final wil... \n", + "2 robustness lowercase Bullsnake -- Bullsnakes are very powerful cons... \n", + "3 robustness add_typo 20 euro note -- Until now there has been only ... \n", + "4 robustness add_typo 2018–19 UEFA Champions League -- The final wil... \n", + "5 robustness add_typo Bullsnake -- Bullsnakes are very powerful cons... \n", "\n", - " test_case \n", - "0 THE EX-READING DEFENDER DENIED FRAUDULENT TRAD... \n", - "1 VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SU... \n", - "2 SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRY... \n", - "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED... \n", - "4 FOUR POLICE OFFICERS WERE INJURED IN THE INCID... \n", - "5 THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOU... \n", - "6 BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPR... \n", - "7 THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS N... \n", - "8 the ex-reading defender denied fraudulent trad... \n", - "9 voges was forced to retire hurt on 86 after su... \n", - "10 seven photographs taken in the norfolk country... \n", - "11 chris poole - known as \"moot\" online - created... \n", - "12 four police officers were injured in the incid... \n", - "13 the injured pedestrian - a young man - is thou... \n", - "14 barca will be investigated for alleged misappr... \n", - "15 the think tank said the city's 1,536 schools n... " + " original_question \\\n", + "0 is the first series 20 euro note still legal t... \n", + "1 do the champions league winners get automatic ... \n", + "2 can a bull snake kill a small dog \n", + "3 is the first series 20 euro note still legal t... \n", + "4 do the champions league winners get automatic ... \n", + "5 can a bull snake kill a small dog \n", + "\n", + " perturbed_context \\\n", + "0 20 euro note -- until now there has been only ... \n", + "1 2018–19 uefa champions league -- the final wil... \n", + "2 bullsnake -- bullsnakes are very powerful cons... \n", + "3 20 euro note -- Until now tuere has been only ... \n", + "4 2018–19 UEFA Champions League -- The final wil... \n", + "5 Bullsnake -- Bullsnakes are very powerful cons... \n", + "\n", + " perturbed_question \n", + "0 is the first series 20 euro note still legal t... \n", + "1 do the champions league winners get automatic ... \n", + "2 can a bull snake kill a small dog \n", + "3 is the flrst series 20 euro note still legal t... \n", + "4 do the champions league winners gey automatic ... \n", + "5 can a bull snake kill a xmall dog " ] }, - "execution_count": 34, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -1078,30 +2372,46 @@ ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "NOJ8BAU2GGzd" + }, + "source": [ + "harness.testcases() method displays the produced test cases in form of a pandas data frame." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3CwhQw6hGR9S" + }, "source": [ "### Running the tests" ] }, { "cell_type": "code", - "execution_count": 36, - "metadata": {}, + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aguX6-aFGOnP", + "outputId": "223486c5-2655-492d-af27-a9c5090ae79b" + }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Running testcases... : 100%|██████████| 16/16 [01:46<00:00, 6.67s/it]\n" + "Running testcases... : 100%|██████████| 6/6 [00:45<00:00, 7.61s/it]\n" ] }, { "data": { "text/plain": [] }, - "execution_count": 36, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -1111,22 +2421,32 @@ ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "191O2oaUGWrH" + }, "source": [ - "### Generated Results" + "Called after harness.generate() and is to used to run all the tests. Returns a pass/fail flag for each test." ] }, { "cell_type": "code", - "execution_count": 39, - "metadata": {}, + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 445 + }, + "id": "-cXkdnihGYke", + "outputId": "c97a53c3-9928-458a-be77-d596c4d3d418" + }, "outputs": [ { "data": { "text/html": [ - "
\n", + "\n", + "
\n", + "
\n", "\n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], "text/plain": [ - " category test_type \\\n", - "0 robustness uppercase \n", - "1 robustness uppercase \n", - "2 robustness uppercase \n", - "3 robustness uppercase \n", - "4 robustness uppercase \n", - "5 robustness uppercase \n", - "6 robustness uppercase \n", - "7 robustness uppercase \n", - "8 robustness lowercase \n", - "9 robustness lowercase \n", - "10 robustness lowercase \n", - "11 robustness lowercase \n", - "12 robustness lowercase \n", - "13 robustness lowercase \n", - "14 robustness lowercase \n", - "15 robustness lowercase \n", - "\n", - " original \\\n", - "0 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail. \n", - "1 Voges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests. \n", - "2 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said. \n", - "3 Chris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on Facebook \n", - "4 Four police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", - "5 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured. \n", - "6 Barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005. \n", - "7 The think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", - "8 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport.\\nMr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42.\\nAppearing at the Old Bailey earlier, all four denied the offence.\\nThe charge relates to offences which allegedly took place between 2008 and 2014.\\nSam, from Kent, Efe and Bright, of Greater Manchester, and Stephen, from Bexley, are due to stand trial in July.\\nThey were all released on bail. \n", - "9 Voges was forced to retire hurt on 86 after suffering the injury while batting during the County Championship draw with Somerset on 4 June.\\nMiddlesex hope to have the Australian back for their T20 Blast game against Hampshire at Lord's on 3 August.\\nThe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"Losing Adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket Angus Fraser said.\\n\"His absence, however, does give opportunities to other players who are desperate to play in the first XI.\\n\"In the past we have coped well without an overseas player and I expect us to do so now.\"\\nDefending county champions Middlesex are sixth in the Division One table, having drawn all four of their matches this season.\\nVoges retired from international cricket in February with a Test batting average of 61.87 from 31 innings, second only to Australian great Sir Donald Bradman's career average of 99.94 from 52 Tests. \n", - "10 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition.\\nIn her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nThe shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100: A Century of Style exhibition.\\nThe duchess, who has a keen interest in photography, has been patron of the National Portrait Gallery since 2012.\\nNicholas Cullinan, director of the National Portrait Gallery, said: \"Josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nHe said the images also encapsulated what Vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nAlexandra Shulman, editor-in-chief of British Vogue, said: \"To be able to publish a photographic shoot with the Duchess of Cambridge has been one of my greatest ambitions for the magazine.\"\\nThe collaboration for the June edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nOther royal portraits to have featured in the fashion magazine include Diana, Princess of Wales - who graced the cover four times - and Princess Anne.\\nThe duchess is to visit the exhibition at the National Portrait Gallery on Wednesday, Kensington Palace said. \n", - "11 Chris Poole - known as \"moot\" online - created the site in 2003.\\nIt has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nIt was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting Apple's iCloud service. That incident prompted a policy change on the site.\\nHowever, 4chan has also been the rallying point for many instances of online activism from the likes of Anonymous, the loosely organized hacktivism group.\\nMr Poole shared news of his new position on blogging site Tumblr.\\n\"When meeting with current and former Googlers, I continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"I'm also impressed by Google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nHe added: \"I can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nMr Poole stepped down as the administrator of 4chan in January 2015. Now he is expected to turn his attentions to Google's social networking efforts.\\nHis arrival was welcomed by Bradley Horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, Google+.\\n\"I'm thrilled he's joining our team here at Google,\" Mr Horowitz said.\\n\"Welcome Chris!\"\\nSeveral commentators described the appointment as \"unexpected\" but noted that Mr Poole's expertise with social media could prove useful to the search firm.\\nFollow Dave Lee on Twitter @DaveLeeBBC and on Facebook \n", - "12 Four police officers were injured in the incident on Friday night.\\nA man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nThey are due to appear before Belfast Magistrates' Court on Monday.\\nThe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", - "13 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall.\\nThe incident took place on High Street at about 18:00.\\nThe man's injuries are believed not to be life-threatening. The driver of the taxi is thought to be uninjured. \n", - "14 Barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with Santos.\\nThe signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nRosell, speaking at a news conference after a Barca board meeting, insisted he had \"acted correctly\".\\nVice-president Josep Maria Bartomeu now takes over from the 49-year-old Rosell, who came to power in 2010.\\nRosell's future has been a real source of concern ever since a Spanish national court judge accepted a lawsuit this week from Barcelona club member Jordi Cases, who alleged that the amount paid for Neymar was more than the reported fee.\\nRosell maintains the accusation is \"unfair and reckless\".\\nHe added: \"For some time, my family and myself have suffered threats and attacks in silence. These threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"From the beginning, I have said the signing of Neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nRosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nHe added: \"It has been an honour to serve the Barcelonistas. It's been a privilege to be the president of FC Barcelona.\"\\nRosell was named Barca president in the summer of 2010 after winning a landslide vote to succeed Joan Laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nHe had also been vice-president under Laporta before resigning in 2005. \n", - "15 The think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nThe government said London was the highest funded part of the country.\\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", - "\n", - " test_case \\\n", - "0 THE EX-READING DEFENDER DENIED FRAUDULENT TRADING CHARGES RELATING TO THE SODJE SPORTS FOUNDATION - A CHARITY TO RAISE MONEY FOR NIGERIAN SPORT.\\nMR SODJE, 37, IS JOINTLY CHARGED WITH ELDER BROTHERS EFE, 44, BRIGHT, 50 AND STEPHEN, 42.\\nAPPEARING AT THE OLD BAILEY EARLIER, ALL FOUR DENIED THE OFFENCE.\\nTHE CHARGE RELATES TO OFFENCES WHICH ALLEGEDLY TOOK PLACE BETWEEN 2008 AND 2014.\\nSAM, FROM KENT, EFE AND BRIGHT, OF GREATER MANCHESTER, AND STEPHEN, FROM BEXLEY, ARE DUE TO STAND TRIAL IN JULY.\\nTHEY WERE ALL RELEASED ON BAIL. \n", - "1 VOGES WAS FORCED TO RETIRE HURT ON 86 AFTER SUFFERING THE INJURY WHILE BATTING DURING THE COUNTY CHAMPIONSHIP DRAW WITH SOMERSET ON 4 JUNE.\\nMIDDLESEX HOPE TO HAVE THE AUSTRALIAN BACK FOR THEIR T20 BLAST GAME AGAINST HAMPSHIRE AT LORD'S ON 3 AUGUST.\\nTHE 37-YEAR-OLD HAS SCORED 230 RUNS IN FOUR FIRST-CLASS GAMES THIS SEASON AT AN AVERAGE OF 57.50.\\n\"LOSING ADAM IS NATURALLY A BLOW AS HE CONTRIBUTES SIGNIFICANTLY TO EVERYTHING WE DO,\" DIRECTOR OF CRICKET ANGUS FRASER SAID.\\n\"HIS ABSENCE, HOWEVER, DOES GIVE OPPORTUNITIES TO OTHER PLAYERS WHO ARE DESPERATE TO PLAY IN THE FIRST XI.\\n\"IN THE PAST WE HAVE COPED WELL WITHOUT AN OVERSEAS PLAYER AND I EXPECT US TO DO SO NOW.\"\\nDEFENDING COUNTY CHAMPIONS MIDDLESEX ARE SIXTH IN THE DIVISION ONE TABLE, HAVING DRAWN ALL FOUR OF THEIR MATCHES THIS SEASON.\\nVOGES RETIRED FROM INTERNATIONAL CRICKET IN FEBRUARY WITH A TEST BATTING AVERAGE OF 61.87 FROM 31 INNINGS, SECOND ONLY TO AUSTRALIAN GREAT SIR DONALD BRADMAN'S CAREER AVERAGE OF 99.94 FROM 52 TESTS. \n", - "2 SEVEN PHOTOGRAPHS TAKEN IN THE NORFOLK COUNTRYSIDE BY PHOTOGRAPHER JOSH OLINS WILL APPEAR IN THE JUNE EDITION.\\nIN HER FIRST SITTING FOR A MAGAZINE, THE DUCHESS IS SEEN LOOKING RELAXED AND WEARING CASUAL CLOTHES.\\nTHE SHOOT WAS IN COLLABORATION WITH THE NATIONAL PORTRAIT GALLERY, WHERE TWO IMAGES ARE BEING DISPLAYED IN THE VOGUE 100: A CENTURY OF STYLE EXHIBITION.\\nTHE DUCHESS, WHO HAS A KEEN INTEREST IN PHOTOGRAPHY, HAS BEEN PATRON OF THE NATIONAL PORTRAIT GALLERY SINCE 2012.\\nNICHOLAS CULLINAN, DIRECTOR OF THE NATIONAL PORTRAIT GALLERY, SAID: \"JOSH HAS CAPTURED THE DUCHESS EXACTLY AS SHE IS - FULL OF LIFE, WITH A GREAT SENSE OF HUMOUR, THOUGHTFUL AND INTELLIGENT, AND IN FACT, VERY BEAUTIFUL.\"\\nHE SAID THE IMAGES ALSO ENCAPSULATED WHAT VOGUE HAD DONE OVER THE PAST 100 YEARS - \"TO PAIR THE BEST PHOTOGRAPHERS WITH THE GREAT PERSONALITIES OF THE DAY, IN ORDER TO REFLECT BROADER SHIFTS IN CULTURE AND SOCIETY\".\\nALEXANDRA SHULMAN, EDITOR-IN-CHIEF OF BRITISH VOGUE, SAID: \"TO BE ABLE TO PUBLISH A PHOTOGRAPHIC SHOOT WITH THE DUCHESS OF CAMBRIDGE HAS BEEN ONE OF MY GREATEST AMBITIONS FOR THE MAGAZINE.\"\\nTHE COLLABORATION FOR THE JUNE EDITION HAD RESULTED IN \"A TRUE CELEBRATION OF OUR CENTENARY AS WELL AS A FITTING TRIBUTE TO A YOUNG WOMAN WHOSE INTEREST IN BOTH PHOTOGRAPHY AND THE COUNTRYSIDE IS WELL KNOWN\", SHE SAID.\\nOTHER ROYAL PORTRAITS TO HAVE FEATURED IN THE FASHION MAGAZINE INCLUDE DIANA, PRINCESS OF WALES - WHO GRACED THE COVER FOUR TIMES - AND PRINCESS ANNE.\\nTHE DUCHESS IS TO VISIT THE EXHIBITION AT THE NATIONAL PORTRAIT GALLERY ON WEDNESDAY, KENSINGTON PALACE SAID. \n", - "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003.\\nIT HAS GONE ON TO BE CLOSELY ASSOCIATED WITH OFFENSIVE AND OFTEN ILLEGAL ACTIVITY, INCLUDING INSTANCES WHERE THE IMAGES OF CHILD ABUSE WERE SHARED.\\nIT WAS WIDELY CREDITED AS BEING THE FIRST PLACE WHERE LEAKED IMAGES OF NUDE CELEBRITIES WERE POSTED FOLLOWING 2014'S WELL-PUBLICISED SECURITY BREACH AFFECTING APPLE'S ICLOUD SERVICE. THAT INCIDENT PROMPTED A POLICY CHANGE ON THE SITE.\\nHOWEVER, 4CHAN HAS ALSO BEEN THE RALLYING POINT FOR MANY INSTANCES OF ONLINE ACTIVISM FROM THE LIKES OF ANONYMOUS, THE LOOSELY ORGANIZED HACKTIVISM GROUP.\\nMR POOLE SHARED NEWS OF HIS NEW POSITION ON BLOGGING SITE TUMBLR.\\n\"WHEN MEETING WITH CURRENT AND FORMER GOOGLERS, I CONTINUALLY FIND MYSELF DRAWN TO THEIR INTELLIGENCE, PASSION, AND ENTHUSIASM - AS WELL AS A UNIVERSAL DESIRE TO SHARE IT WITH OTHERS.\"\\n\"I'M ALSO IMPRESSED BY GOOGLE'S COMMITMENT TO ENABLING THESE SAME TALENTED PEOPLE TO TACKLE SOME OF THE WORLD'S MOST INTERESTING AND IMPORTANT PROBLEMS.\\nHE ADDED: \"I CAN'T WAIT TO CONTRIBUTE MY OWN EXPERIENCE FROM A DOZEN YEARS OF BUILDING ONLINE COMMUNITIES, AND TO BEGIN THE NEXT CHAPTER OF MY CAREER AT SUCH AN INCREDIBLE COMPANY.\"\\nMR POOLE STEPPED DOWN AS THE ADMINISTRATOR OF 4CHAN IN JANUARY 2015. NOW HE IS EXPECTED TO TURN HIS ATTENTIONS TO GOOGLE'S SOCIAL NETWORKING EFFORTS.\\nHIS ARRIVAL WAS WELCOMED BY BRADLEY HOROWITZ, THE HEAD OF \"STREAMS, PHOTOS AND SHARING\" AT THE SEARCH GIANT'S FLOUNDERING SOCIAL NETWORK, GOOGLE+.\\n\"I'M THRILLED HE'S JOINING OUR TEAM HERE AT GOOGLE,\" MR HOROWITZ SAID.\\n\"WELCOME CHRIS!Ï»¿\"\\nSEVERAL COMMENTATORS DESCRIBED THE APPOINTMENT AS \"UNEXPECTED\" BUT NOTED THAT MR POOLE'S EXPERTISE WITH SOCIAL MEDIA COULD PROVE USEFUL TO THE SEARCH FIRM.\\nFOLLOW DAVE LEE ON TWITTER @DAVELEEBBC AND ON FACEBOOK \n", - "4 FOUR POLICE OFFICERS WERE INJURED IN THE INCIDENT ON FRIDAY NIGHT.\\nA MAN, AGED 19, AND A BOY, AGED 16, HAVE BEEN CHARGED WITH SIX COUNTS OF AGGRAVATED VEHICLE TAKING.\\nTHEY ARE DUE TO APPEAR BEFORE BELFAST MAGISTRATES' COURT ON MONDAY.\\nTHE 19-YEAR-OLD MAN HAS ALSO BEEN CHARGED WITH DRIVING WHILE DISQUALIFIED AND USING A MOTOR VEHICLE WITHOUT INSURANCE. \n", - "5 THE INJURED PEDESTRIAN - A YOUNG MAN - IS THOUGHT TO HAVE BEEN WALKING WITH A GROUP OF PEOPLE FROM A GRADUATION CEREMONY AT THE CAIRD HALL.\\nTHE INCIDENT TOOK PLACE ON HIGH STREET AT ABOUT 18:00.\\nTHE MAN'S INJURIES ARE BELIEVED NOT TO BE LIFE-THREATENING. THE DRIVER OF THE TAXI IS THOUGHT TO BE UNINJURED. \n", - "6 BARCA WILL BE INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS.\\nTHE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES\\nROSELL, SPEAKING AT A NEWS CONFERENCE AFTER A BARCA BOARD MEETING, INSISTED HE HAD \"ACTED CORRECTLY\".\\nVICE-PRESIDENT JOSEP MARIA BARTOMEU NOW TAKES OVER FROM THE 49-YEAR-OLD ROSELL, WHO CAME TO POWER IN 2010.\\nROSELL'S FUTURE HAS BEEN A REAL SOURCE OF CONCERN EVER SINCE A SPANISH NATIONAL COURT JUDGE ACCEPTED A LAWSUIT THIS WEEK FROM BARCELONA CLUB MEMBER JORDI CASES, WHO ALLEGED THAT THE AMOUNT PAID FOR NEYMAR WAS MORE THAN THE REPORTED FEE.\\nROSELL MAINTAINS THE ACCUSATION IS \"UNFAIR AND RECKLESS\".\\nHE ADDED: \"FOR SOME TIME, MY FAMILY AND MYSELF HAVE SUFFERED THREATS AND ATTACKS IN SILENCE. THESE THREATS AND ATTACKS HAVE MADE ME WONDER IF BEING PRESIDENT MEANS HAVING TO JEOPARDISE MY FAMILY.\\n\"FROM THE BEGINNING, I HAVE SAID THE SIGNING OF NEYMAR HAS BEEN CORRECT AND HIS SIGNING HAS CAUSED DESPAIR AND ENVY IN SOME OF OUR ADVERSARIES.\"\\nROSELL SAID HE WAS RESIGNING TO SPARE THE CLUB'S BOARD OF DIRECTORS FROM \"UNFAIR ATTACKS\" THAT COULD \"NEGATIVELY AFFECT THEIR MANAGEMENT OR THE IMAGE OF THE CLUB\".\\nHE ADDED: \"IT HAS BEEN AN HONOUR TO SERVE THE BARCELONISTAS. IT'S BEEN A PRIVILEGE TO BE THE PRESIDENT OF FC BARCELONA.\"\\nROSELL WAS NAMED BARCA PRESIDENT IN THE SUMMER OF 2010 AFTER WINNING A LANDSLIDE VOTE TO SUCCEED JOAN LAPORTA, EARNING 61.34% OF A RECORD TURN-OUT OF 57,088 VOTERS.\\nHE HAD ALSO BEEN VICE-PRESIDENT UNDER LAPORTA BEFORE RESIGNING IN 2005. \n", - "7 THE THINK TANK SAID THE CITY'S 1,536 SCHOOLS NEEDED TO SAVE £360M IN THE FIRST YEAR IF THE GOVERNMENT'S NATIONAL FUNDING FORMULA (NFF) PLAN GOES AHEAD.\\nTHE AMOUNT IS THE EQUIVALENT OF 12,857 QUALIFIED TEACHERS, ON AN AVERAGE SALARY OF £28,000.\\nTHE GOVERNMENT SAID LONDON WAS THE HIGHEST FUNDED PART OF THE COUNTRY.\\nIT ADDED THAT UNDER THE PLANS, WHICH ARE UNDER CONSULTATION, INNER-CITY SCHOOLS WOULD BE ALLOCATED 30% MORE MONEY PER PUPIL THAN THE NATIONAL AVERAGE.\\nBUT LONDON COUNCILS, WHICH REPRESENTS THE CITY'S 32 BOROUGHS AND THE CITY, SAID NO SCHOOL WOULD GAIN ENOUGH FUNDING FROM THE NFF TO COMPENSATE FOR INCREASED COST PRESSURES FROM INFLATION, HIGHER PENSION CONTRIBUTIONS AND NATIONAL INSURANCE.\\nMINISTERS SAID THE NEW FORMULA WAS NEEDED TO TACKLE UNEVEN LEVELS OF FUNDING ACROSS ENGLAND, WITH THE BEST FUNDED AREAS GETTING MORE THAN £6,300 PER PUPIL PER YEAR, WHILE THE WORST-FUNDED AVERAGING £4,200.\\nIT SAID THE FUNDING CUT WAS ON TOP OF NATIONAL AUDIT OFFICE FIGURES WHICH SHOWED ENGLAND SCHOOLS FACED AN EIGHT PER CENT REAL-TERMS CUT PER PUPIL BY 2019-20 BECAUSE IT WIDER COST PRESSURES.\\nIN A STATEMENT, LONDON COUNCILS SAID: \"AT A TIME WHEN UK SCHOOLS ARE SEEN AS UNDERPERFORMING BY INTERNATIONAL STANDARDS, AND WHEN BUSINESSES BASED IN LONDON ARE FACING MASSIVE UNCERTAINTY ABOUT RECRUITING SKILLED STAFF, THERE IS AN URGENT NEED TO INVEST IN SCHOOLS IN LONDON AND ACROSS THE REST OF THE COUNTRY.\"\\nIT ADDED: \"WITHOUT THE RIGHT QUALIFICATIONS AND SKILLS, LONDON'S CHILDREN WILL BE UNABLE TO ACCESS JOBS AND CONTRIBUTE TO THE NATIONAL ECONOMY. OVER 60% OF JOBS IN INNER LONDON REQUIRE A DEGREE AND AROUND 45% OF JOBS IN THE REST OF THE CAPITAL REQUIRE A DEGREE.\" \n", - "8 the ex-reading defender denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for nigerian sport.\\nmr sodje, 37, is jointly charged with elder brothers efe, 44, bright, 50 and stephen, 42.\\nappearing at the old bailey earlier, all four denied the offence.\\nthe charge relates to offences which allegedly took place between 2008 and 2014.\\nsam, from kent, efe and bright, of greater manchester, and stephen, from bexley, are due to stand trial in july.\\nthey were all released on bail. \n", - "9 voges was forced to retire hurt on 86 after suffering the injury while batting during the county championship draw with somerset on 4 june.\\nmiddlesex hope to have the australian back for their t20 blast game against hampshire at lord's on 3 august.\\nthe 37-year-old has scored 230 runs in four first-class games this season at an average of 57.50.\\n\"losing adam is naturally a blow as he contributes significantly to everything we do,\" director of cricket angus fraser said.\\n\"his absence, however, does give opportunities to other players who are desperate to play in the first xi.\\n\"in the past we have coped well without an overseas player and i expect us to do so now.\"\\ndefending county champions middlesex are sixth in the division one table, having drawn all four of their matches this season.\\nvoges retired from international cricket in february with a test batting average of 61.87 from 31 innings, second only to australian great sir donald bradman's career average of 99.94 from 52 tests. \n", - "10 seven photographs taken in the norfolk countryside by photographer josh olins will appear in the june edition.\\nin her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes.\\nthe shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue 100: a century of style exhibition.\\nthe duchess, who has a keen interest in photography, has been patron of the national portrait gallery since 2012.\\nnicholas cullinan, director of the national portrait gallery, said: \"josh has captured the duchess exactly as she is - full of life, with a great sense of humour, thoughtful and intelligent, and in fact, very beautiful.\"\\nhe said the images also encapsulated what vogue had done over the past 100 years - \"to pair the best photographers with the great personalities of the day, in order to reflect broader shifts in culture and society\".\\nalexandra shulman, editor-in-chief of british vogue, said: \"to be able to publish a photographic shoot with the duchess of cambridge has been one of my greatest ambitions for the magazine.\"\\nthe collaboration for the june edition had resulted in \"a true celebration of our centenary as well as a fitting tribute to a young woman whose interest in both photography and the countryside is well known\", she said.\\nother royal portraits to have featured in the fashion magazine include diana, princess of wales - who graced the cover four times - and princess anne.\\nthe duchess is to visit the exhibition at the national portrait gallery on wednesday, kensington palace said. \n", - "11 chris poole - known as \"moot\" online - created the site in 2003.\\nit has gone on to be closely associated with offensive and often illegal activity, including instances where the images of child abuse were shared.\\nit was widely credited as being the first place where leaked images of nude celebrities were posted following 2014's well-publicised security breach affecting apple's icloud service. that incident prompted a policy change on the site.\\nhowever, 4chan has also been the rallying point for many instances of online activism from the likes of anonymous, the loosely organized hacktivism group.\\nmr poole shared news of his new position on blogging site tumblr.\\n\"when meeting with current and former googlers, i continually find myself drawn to their intelligence, passion, and enthusiasm - as well as a universal desire to share it with others.\"\\n\"i'm also impressed by google's commitment to enabling these same talented people to tackle some of the world's most interesting and important problems.\\nhe added: \"i can't wait to contribute my own experience from a dozen years of building online communities, and to begin the next chapter of my career at such an incredible company.\"\\nmr poole stepped down as the administrator of 4chan in january 2015. now he is expected to turn his attentions to google's social networking efforts.\\nhis arrival was welcomed by bradley horowitz, the head of \"streams, photos and sharing\" at the search giant's floundering social network, google+.\\n\"i'm thrilled he's joining our team here at google,\" mr horowitz said.\\n\"welcome chris!\"\\nseveral commentators described the appointment as \"unexpected\" but noted that mr poole's expertise with social media could prove useful to the search firm.\\nfollow dave lee on twitter @daveleebbc and on facebook \n", - "12 four police officers were injured in the incident on friday night.\\na man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking.\\nthey are due to appear before belfast magistrates' court on monday.\\nthe 19-year-old man has also been charged with driving while disqualified and using a motor vehicle without insurance. \n", - "13 the injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the caird hall.\\nthe incident took place on high street at about 18:00.\\nthe man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured. \n", - "14 barca will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos.\\nthe signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries\\nrosell, speaking at a news conference after a barca board meeting, insisted he had \"acted correctly\".\\nvice-president josep maria bartomeu now takes over from the 49-year-old rosell, who came to power in 2010.\\nrosell's future has been a real source of concern ever since a spanish national court judge accepted a lawsuit this week from barcelona club member jordi cases, who alleged that the amount paid for neymar was more than the reported fee.\\nrosell maintains the accusation is \"unfair and reckless\".\\nhe added: \"for some time, my family and myself have suffered threats and attacks in silence. these threats and attacks have made me wonder if being president means having to jeopardise my family.\\n\"from the beginning, i have said the signing of neymar has been correct and his signing has caused despair and envy in some of our adversaries.\"\\nrosell said he was resigning to spare the club's board of directors from \"unfair attacks\" that could \"negatively affect their management or the image of the club\".\\nhe added: \"it has been an honour to serve the barcelonistas. it's been a privilege to be the president of fc barcelona.\"\\nrosell was named barca president in the summer of 2010 after winning a landslide vote to succeed joan laporta, earning 61.34% of a record turn-out of 57,088 voters.\\nhe had also been vice-president under laporta before resigning in 2005. \n", - "15 the think tank said the city's 1,536 schools needed to save £360m in the first year if the government's national funding formula (nff) plan goes ahead.\\nthe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\\nthe government said london was the highest funded part of the country.\\nit added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\\nbut london councils, which represents the city's 32 boroughs and the city, said no school would gain enough funding from the nff to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\\nministers said the new formula was needed to tackle uneven levels of funding across england, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\\nit said the funding cut was on top of national audit office figures which showed england schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\\nin a statement, london councils said: \"at a time when uk schools are seen as underperforming by international standards, and when businesses based in london are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in london and across the rest of the country.\"\\nit added: \"without the right qualifications and skills, london's children will be unable to access jobs and contribute to the national economy. over 60% of jobs in inner london require a degree and around 45% of jobs in the rest of the capital require a degree.\" \n", - "\n", - " expected_result \\\n", - "0 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, all \n", - "1 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced. \n", - "2 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100 \n", - "3 The man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports. \n", - "4 Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday. \n", - "5 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought to \n", - "6 Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar. \n", - "7 The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government said \n", - "8 The ex-Reading defender denied fraudulent trading charges relating to the Sodje Sports Foundation - a charity to raise money for Nigerian sport. Mr Sodje, 37, is jointly charged with elder brothers Efe, 44, Bright, 50 and Stephen, 42. Appearing at the Old Bailey earlier, all \n", - "9 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season because of a stress fracture in his lower back, the county have announced. \n", - "10 Seven photographs taken in the Norfolk countryside by photographer Josh Olins will appear in the June edition of British Vogue. In her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. The shoot was in collaboration with the National Portrait Gallery, where two images are being displayed in the Vogue 100 \n", - "11 The man who created the notorious online forum 4chan has been appointed head of Google's social network, Google+, the search giant has announced. The BBC's Dave Lee reports. \n", - "12 Four police officers were injured in the incident on Friday night. A man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. They are due to appear before Belfast Magistrates' Court on Monday. \n", - "13 The injured pedestrian - a young man - is thought to have been walking with a group of people from a graduation ceremony at the Caird Hall. The incident took place on High Street at about 18:00. The man's injuries are believed not to be life-threatening. The driver of the taxi is thought to \n", - "14 Barcelona's president, Sandro Rosell, has resigned from his post following the alleged misappropriation of funds in the £48.6m (57m euros) deal for Neymar. \n", - "15 The London Councils think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead. The amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000. The government said \n", - "\n", - " actual_result \\\n", - "0 Former Liverpool defender Steven Caulker has pleaded guilty to one count of conspiracy to commit fraud and one count of conspiracy to commit money laundering in connection with the alleged manipulation of foreign exchange markets. \n", - "1 Former Australia batsman Adam Voges has been forced to retire from first-class cricket after injuring his ankle during Middlesex's draw with Somerset on 4 June. \n", - "2 The Duchess of Cambridge will appear on the cover of British Vogue for the first time in the magazine's 100th anniversary issue, it has been announced. \n", - "3 CHRIS POOLE - KNOWN AS \"MOOT\" ONLINE - CREATED THE SITE IN 2003. It has gone on to be CLOSELY ASSOCIATED with OFFENSIVE and OFTEN ILLEGAL ACTIVITY, including inSTANCES where the IMAGES of Child ABUSE were shared. It was \n", - "4 A 19-year-old man and a 16-year-old boy have been charged after four police officers were injured in a car crash in Belfast on Friday night. \n", - "5 A man in his 20s has been seriously injured after he was hit by a TAXI in central London. He is thought to have been walking with a group of people from a graduation ceremony at the CAIRD HALL. \n", - "6 Barcelona will be INVESTIGATED FOR ALLEGED MISAPPROPRIATION OF FUNDS IN THE £48.6M (57M EUROS) DEAL WITH SANTOS. \n", - "7 London's 1,536 schools need to save £360M in the first year if the government's national funding plan goes ahead, says think tank. \n", - "8 Ex-Reading defender Samuel Eto'o has denied fraudulent trading charges relating to the sodje sports foundation - a charity to raise money for Nigerian sport. \n", - "9 Former Australia batsman Adam Voges has been ruled out of Middlesex's first-class season after injuring his hamstring during the county championship draw with Somerset on June 4. \n", - "10 Seven photographs taken in the norfolk countryside by photographer josholins will appear in the june edition of British Vogue. in her first sitting for a magazine, the duchess is seen looking relaxed and wearing casual clothes. the shoot was in collaboration with the national portrait gallery, where two images are being displayed in the vogue \n", - "11 Google has appointed the former administrator of 4chan, one of the world's most notorious online forums, to its social media team. Dave Lee reports. \n", - "12 Four police officers were injured in the incident on Friday night. a man, aged 19, and a boy, aged 16, have been charged with six counts of aggravated vehicle taking. they are due to appear before Belfast magistrates' court on Monday. \n", - "13 A pedestrian is injured after being hit by a taxi in central London. The incident took place on high street at about 18:00. the man's injuries are believed not to be life-threatening. the driver of the taxi is thought to be uninjured. \n", - "14 Barcelona will be investigated for alleged misappropriation of funds in the £48.6m (57m euros) deal with santos, the club has announced. \n", - "15 London's 1,536 schools need to save £360m in the first year if the government's national funding formula (nff) plan goes ahead \n", - "\n", - " eval_score pass \n", - "0 0.000000 False \n", - "1 0.301887 False \n", - "2 0.121951 False \n", - "3 0.030769 False \n", - "4 0.294118 False \n", - "5 0.391304 False \n", - "6 0.487805 False \n", - "7 0.486486 False \n", - "8 0.558824 True \n", - "9 0.545455 True \n", - "10 0.945455 True \n", - "11 0.113208 False \n", - "12 1.000000 True \n", - "13 0.591837 True \n", - "14 0.444444 False \n", - "15 0.547945 True " + " category test_type original_context \\\n", + "0 robustness lowercase 20 euro note -- Until now there has been only ... \n", + "1 robustness lowercase 2018–19 UEFA Champions League -- The final wil... \n", + "2 robustness lowercase Bullsnake -- Bullsnakes are very powerful cons... \n", + "3 robustness add_typo 20 euro note -- Until now there has been only ... \n", + "4 robustness add_typo 2018–19 UEFA Champions League -- The final wil... \n", + "5 robustness add_typo Bullsnake -- Bullsnakes are very powerful cons... \n", + "\n", + " original_question \\\n", + "0 is the first series 20 euro note still legal t... \n", + "1 do the champions league winners get automatic ... \n", + "2 can a bull snake kill a small dog \n", + "3 is the first series 20 euro note still legal t... \n", + "4 do the champions league winners get automatic ... \n", + "5 can a bull snake kill a small dog \n", + "\n", + " perturbed_context \\\n", + "0 20 euro note -- until now there has been only ... \n", + "1 2018–19 uefa champions league -- the final wil... \n", + "2 bullsnake -- bullsnakes are very powerful cons... \n", + "3 20 euro note -- Until now tuere has been only ... \n", + "4 2018–19 UEFA Champions League -- The final wil... \n", + "5 Bullsnake -- Bullsnakes are very powerful cons... \n", + "\n", + " perturbed_question \\\n", + "0 is the first series 20 euro note still legal t... \n", + "1 do the champions league winners get automatic ... \n", + "2 can a bull snake kill a small dog \n", + "3 is the flrst series 20 euro note still legal t... \n", + "4 do the champions league winners gey automatic ... \n", + "5 can a bull snake kill a xmall dog \n", + "\n", + " expected_result \\\n", + "0 \\n\\nI'm not sure if this is the right place to... \n", + "1 \\n\\nAnswer:\\n\\nThe Champions League winners wi... \n", + "2 \\n\\nThe bot will be able to answer the questio... \n", + "3 \\n\\nI'm not sure if this is the right place to... \n", + "4 \\n\\nAnswer:\\n\\nThe Champions League winners wi... \n", + "5 \\n\\nThe bot will be able to answer the questio... \n", + "\n", + " actual_result pass \n", + "0 \\n\\nI'm not sure if this is the right place to... False \n", + "1 \\n\\nAnswer:\\n\\nThe 2018–19 uefa champions leag... False \n", + "2 \\n\\nI have a question about the bull snake. I ... False \n", + "3 \\n\\nI'm not sure if this is the right place to... False \n", + "4 \\n\\nThe answer to the question is True.\\n\\nThe... False \n", + "5 \\n\\nThe bot will be able to answer the questio... False " ] }, - "execution_count": 39, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -1455,30 +2752,50 @@ ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "TKB8Rsr2GZME" + }, "source": [ - "### Generate Report" + "This method returns the generated results in the form of a pandas dataframe, which provides a convenient and easy-to-use format for working with the test results. You can use this method to quickly identify the test cases that failed and to determine where fixes are needed." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PBSlpWnUU55G" + }, + "source": [ + "### Final Results" ] }, { - "attachments": {}, "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "RZPE4Llkf8TQ" + }, "source": [ "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag." ] }, { "cell_type": "code", - "execution_count": 40, - "metadata": {}, + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 112 + }, + "id": "gp57HcF9yxi7", + "outputId": "ca6a05b9-a9d8-4f58-b07a-35497521e661" + }, "outputs": [ { "data": { "text/html": [ - "
\n", + "\n", + "
\n", + "
\n", "\n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" ], "text/plain": [ " category test_type fail_count pass_count pass_rate minimum_pass_rate \\\n", - "0 robustness uppercase 8 0 0% 66% \n", - "1 robustness lowercase 2 6 75% 60% \n", + "0 robustness lowercase 3 0 0% 70% \n", + "1 robustness add_typo 3 0 0% 70% \n", "\n", " pass \n", "0 False \n", - "1 True " + "1 False " ] }, - "execution_count": 40, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1554,8 +3011,7 @@ "accelerator": "TPU", "colab": { "machine_shape": "hm", - "provenance": [], - "toc_visible": true + "provenance": [] }, "gpuClass": "standard", "kernelspec": { @@ -1572,7 +3028,2747 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.9.13" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "0341ee74e543493ca3765809e41611e3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "098411bb18d9448b9a9592294e1ce81b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9e8db2529d7452b946c3ca0151d6d55", + "placeholder": "​", + "style": "IPY_MODEL_f35ce4f6d998440e8f04a3cbc8394e11", + "value": " 441/441 [00:00<00:00, 37.7kB/s]" + } + }, + "0fd6422fbb7747aa9983aa0ad77390a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "11dd94d811424b41ab7c332074aecbfa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2304b5d9f4904ce4b133943a6770d276", + "placeholder": "​", + "style": "IPY_MODEL_ae8a33f996d64ab3a409a9d7d8599ba8", + "value": "Downloading (…)olve/main/merges.txt: 100%" + } + }, + "145895cb381e48a7beb2bccbb14f0795": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1a1f78619dd144baa168713c59ffc737": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1b8a289aeb51470a8c1a2950f8c57aa7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1a1f78619dd144baa168713c59ffc737", + "max": 653, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4a5677beefb242c583ce0bd3af5e2a95", + "value": 653 + } + }, + "1d28ccffbccf43e080aa39c6a6f53b1c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1d6b9a06881d451dbbc342917bb3bdfc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8e3f068f9ad441bda11ac18f216b1f0d", + "placeholder": "​", + "style": "IPY_MODEL_fccb920a50114e859e0d3b9c176f53d5", + "value": " 899k/899k [00:00<00:00, 3.39MB/s]" + } + }, + "1fa614546c0848dab5984d98ab2aff6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2304b5d9f4904ce4b133943a6770d276": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "294762c36d48446890ec7bca8a1f5591": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "29fbd43db74740e4b9af627253cc0bdb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d690e77373945ba8652dd260b760bcf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2eb51073cd02468b946163f433f3962a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "325256d7089640e694151e389d7a2df5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f7f320c30f4245c89544a1959ea323c1", + "max": 137, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_0fd6422fbb7747aa9983aa0ad77390a3", + "value": 137 + } + }, + "336a4708137f4fe4b9aa551ab127227d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_44086a4a5aaf4e4a912f4b26551d8f51", + "IPY_MODEL_4e0a3727b36d43f2ba34f1ca24df9be5", + "IPY_MODEL_d2078e9c1dfe4e7aa97332baf9473dd5" + ], + "layout": "IPY_MODEL_5e731174370041d4bbe536886f139948" + } + }, + "3e86321f68d54720803a357c49212d60": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "43580b4168f442139012428ba0e6c759": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4400c80adb044860a2607681da776437": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44086a4a5aaf4e4a912f4b26551d8f51": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ca1ba76b0dd04d4885a42091872d6a62", + "placeholder": "​", + "style": "IPY_MODEL_f594ac19e3a344d2b64aa55af953d115", + "value": "Downloading builder script: 100%" + } + }, + "44201622077d46b3beb20b665fa2e191": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4a5677beefb242c583ce0bd3af5e2a95": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4ac2635d7b954f1bb8658490ce4b102c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7b6b5d4199cc4f8785c8fabcae9a8036", + "placeholder": "​", + "style": "IPY_MODEL_cf93f2c9cb294637aef9766c794f7fb2", + "value": "Downloading (…)lve/main/config.json: 100%" + } + }, + "4bb819239c4748fb9de90f5c66aab70d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4c109f682d4f4d699f3f509248bbf804": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4d9d1da4041d4a529df11e08c9089142": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4e0a3727b36d43f2ba34f1ca24df9be5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b952c931792240e6b23bec6dd3ec49d5", + "max": 6270, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_796e43018c574e8b89e32ab7d9e29b84", + "value": 6270 + } + }, + "4e10f045a75f4665bac2cf9a539f7b0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1d28ccffbccf43e080aa39c6a6f53b1c", + "max": 2631639353, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8633a4a4ee2543caa02cfdbc4f21846d", + "value": 2631639353 + } + }, + "5b650dafed624813b692c84d24657cfe": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3e86321f68d54720803a357c49212d60", + "placeholder": "​", + "style": "IPY_MODEL_9cd8390472734d8f89ad8ea058127fd0", + "value": " 653/653 [00:00<00:00, 52.1kB/s]" + } + }, + "5c3c82880ed44efbb03cae396bc72918": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e731174370041d4bbe536886f139948": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5ea1c82925dc47cc9b672cdb92029f74": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ec0afe2bcb234063a15525bb4d2dbee2", + "max": 685, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b546d25c1f1e4f6ea18d33af8a8d4359", + "value": 685 + } + }, + "5ef205051e974687ae134cc575d8b41e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_fbfbdc0a14c04e2fb03a64ad467b95d5", + "IPY_MODEL_4e10f045a75f4665bac2cf9a539f7b0b", + "IPY_MODEL_923b976ddb594ddaa37578088f5ff87c" + ], + "layout": "IPY_MODEL_4400c80adb044860a2607681da776437" + } + }, + "6021523b6760479f89367d38fffe06f8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "674dc3d1f55841ea9cd682c863df6d9f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "68591dcca5d84e5fac075a2d0f4d0cdf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6c3ab47327624fc0877b9f8bb63bb167": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c8bca45bffcb407aa9e287423323930a", + "IPY_MODEL_9e7ef7a229c44ebb9903c2d2bc7a9b07", + "IPY_MODEL_098411bb18d9448b9a9592294e1ce81b" + ], + "layout": "IPY_MODEL_89082c41aee84ab99e411ab5db986302" + } + }, + "723ecc2c963541828a408f829b4b883c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bc61b35936984566821f40758b155069", + "max": 456318, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8cbad88f229949149c894e1a8a71f9de", + "value": 456318 + } + }, + "796e43018c574e8b89e32ab7d9e29b84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7b6b5d4199cc4f8785c8fabcae9a8036": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7cb46e8a3c974d24864314fa17b79d85": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_294762c36d48446890ec7bca8a1f5591", + "max": 898822, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_7cd725d017384a90be458e35799b1279", + "value": 898822 + } + }, + "7cd725d017384a90be458e35799b1279": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8633a4a4ee2543caa02cfdbc4f21846d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "86a1bf7601b949ecbc7fd8b53936da0c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "89082c41aee84ab99e411ab5db986302": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8b1ab800a4b6466b8f982c57653bb456": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8b3333afb5d54c65b6cfce98f97d16b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a0383864826b4d1db8e080510e1853c4", + "placeholder": "​", + "style": "IPY_MODEL_4d9d1da4041d4a529df11e08c9089142", + "value": " 685/685 [00:00<00:00, 60.8kB/s]" + } + }, + "8cbad88f229949149c894e1a8a71f9de": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8d3102653db94e5aaf576c6059c98960": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8e3f068f9ad441bda11ac18f216b1f0d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "923b976ddb594ddaa37578088f5ff87c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0341ee74e543493ca3765809e41611e3", + "placeholder": "​", + "style": "IPY_MODEL_c04797768f1149d7a97adb994b45c09e", + "value": " 2.63G/2.63G [02:50<00:00, 13.5MB/s]" + } + }, + "95cf3f8f98124519b9d9224b94743959": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9716d1570ef74ed0a67660f9379c278c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43580b4168f442139012428ba0e6c759", + "placeholder": "​", + "style": "IPY_MODEL_9ff53ca79e9a4379b51b61773571a159", + "value": "Downloading (…)olve/main/vocab.json: 100%" + } + }, + "98e76b406ed6487094f78d5559d2d764": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9cd8390472734d8f89ad8ea058127fd0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9e7ef7a229c44ebb9903c2d2bc7a9b07": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4c109f682d4f4d699f3f509248bbf804", + "max": 441, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2d690e77373945ba8652dd260b760bcf", + "value": 441 + } + }, + "9ff53ca79e9a4379b51b61773571a159": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a0383864826b4d1db8e080510e1853c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a3b4eb7bb33c4d39966c4ffdad654170": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5c3c82880ed44efbb03cae396bc72918", + "placeholder": "​", + "style": "IPY_MODEL_68591dcca5d84e5fac075a2d0f4d0cdf", + "value": "Downloading (…)neration_config.json: 100%" + } + }, + "a61f506ecd39454c9c108eb1f53d268a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae8a33f996d64ab3a409a9d7d8599ba8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "afb8a9e85c4a40dabf112268d7e9196c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b546d25c1f1e4f6ea18d33af8a8d4359": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b8a4848be055497da44a449e7f2c672b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a3b4eb7bb33c4d39966c4ffdad654170", + "IPY_MODEL_325256d7089640e694151e389d7a2df5", + "IPY_MODEL_c614a0ee97364305a835f3a902d6e939" + ], + "layout": "IPY_MODEL_a61f506ecd39454c9c108eb1f53d268a" + } + }, + "b952c931792240e6b23bec6dd3ec49d5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bc61b35936984566821f40758b155069": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c04797768f1149d7a97adb994b45c09e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c1987349992a40e7833027843ec52c79": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_11dd94d811424b41ab7c332074aecbfa", + "IPY_MODEL_723ecc2c963541828a408f829b4b883c", + "IPY_MODEL_cde87e4029ce469a9d048742ea06cea9" + ], + "layout": "IPY_MODEL_86a1bf7601b949ecbc7fd8b53936da0c" + } + }, + "c614a0ee97364305a835f3a902d6e939": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8b1ab800a4b6466b8f982c57653bb456", + "placeholder": "​", + "style": "IPY_MODEL_44201622077d46b3beb20b665fa2e191", + "value": " 137/137 [00:00<00:00, 11.8kB/s]" + } + }, + "c8bca45bffcb407aa9e287423323930a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_145895cb381e48a7beb2bccbb14f0795", + "placeholder": "​", + "style": "IPY_MODEL_4bb819239c4748fb9de90f5c66aab70d", + "value": "Downloading (…)cial_tokens_map.json: 100%" + } + }, + "ca1ba76b0dd04d4885a42091872d6a62": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cde87e4029ce469a9d048742ea06cea9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2eb51073cd02468b946163f433f3962a", + "placeholder": "​", + "style": "IPY_MODEL_95cf3f8f98124519b9d9224b94743959", + "value": " 456k/456k [00:00<00:00, 32.3MB/s]" + } + }, + "cf93f2c9cb294637aef9766c794f7fb2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d2078e9c1dfe4e7aa97332baf9473dd5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_daa0487fb8e24f76afc361bb1202479e", + "placeholder": "​", + "style": "IPY_MODEL_8d3102653db94e5aaf576c6059c98960", + "value": " 6.27k/6.27k [00:00<00:00, 388kB/s]" + } + }, + "daa0487fb8e24f76afc361bb1202479e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e00a999af46a4d898f3f9ca1083b68d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1fa614546c0848dab5984d98ab2aff6e", + "placeholder": "​", + "style": "IPY_MODEL_6021523b6760479f89367d38fffe06f8", + "value": "Downloading (…)okenizer_config.json: 100%" + } + }, + "e3772ec851be44cf838067837104f36d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e00a999af46a4d898f3f9ca1083b68d1", + "IPY_MODEL_5ea1c82925dc47cc9b672cdb92029f74", + "IPY_MODEL_8b3333afb5d54c65b6cfce98f97d16b6" + ], + "layout": "IPY_MODEL_98e76b406ed6487094f78d5559d2d764" + } + }, + "e92f799378dd4be697cb05ad64e070ab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9716d1570ef74ed0a67660f9379c278c", + "IPY_MODEL_7cb46e8a3c974d24864314fa17b79d85", + "IPY_MODEL_1d6b9a06881d451dbbc342917bb3bdfc" + ], + "layout": "IPY_MODEL_f7f9a53cdbee4497955bec5d2441e56c" + } + }, + "e9e8db2529d7452b946c3ca0151d6d55": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ec0afe2bcb234063a15525bb4d2dbee2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f16fa357e41946eeab0a00930b898379": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4ac2635d7b954f1bb8658490ce4b102c", + "IPY_MODEL_1b8a289aeb51470a8c1a2950f8c57aa7", + "IPY_MODEL_5b650dafed624813b692c84d24657cfe" + ], + "layout": "IPY_MODEL_29fbd43db74740e4b9af627253cc0bdb" + } + }, + "f35ce4f6d998440e8f04a3cbc8394e11": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f594ac19e3a344d2b64aa55af953d115": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f7f320c30f4245c89544a1959ea323c1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f7f9a53cdbee4497955bec5d2441e56c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fbfbdc0a14c04e2fb03a64ad467b95d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_674dc3d1f55841ea9cd682c863df6d9f", + "placeholder": "​", + "style": "IPY_MODEL_afb8a9e85c4a40dabf112268d7e9196c", + "value": "Downloading pytorch_model.bin: 100%" + } + }, + "fccb920a50114e859e0d3b9c176f53d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } } }, "nbformat": 4, diff --git a/docs/pages/tutorials/tutorials.md b/docs/pages/tutorials/tutorials.md index f85632076..025bccc8d 100644 --- a/docs/pages/tutorials/tutorials.md +++ b/docs/pages/tutorials/tutorials.md @@ -23,45 +23,46 @@ The following table gives an overview of the different tutorial notebooks. We ha {:.table2} -|Tutorial Description|Library|Task|Open In Colab| -|-| -|Accuracy Tests |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Accuracy_Demo.ipynb)| -|Bias Tests |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Bias_Demo.ipynb)| -|Fairness Tests |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Fairness_Demo.ipynb)| -|Representation Tests |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Representation_Demo.ipynb)| -|Robustness Tests |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Robustness_DEMO.ipynb)| -|Toxicity Test |OpenAI |Toxicity|[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Toxicity_NB.ipynb)| -|Add Custom Data |Spacy/Hugging Face |NER/Text-Classification|[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Add_Custom_Data_Demo.ipynb)| -|End-to-End Workflow |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Notebook.ipynb)| -|End-to-End Custom Pipeline Workflow |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Custom_Pipeline_Notebook.ipynb)| -|End-to-End Workflow |Spacy |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/Spacy_Real_World_Notebook.ipynb)| -|End-to-End Workflow |Hugging Face |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/HuggingFace_Real_World_Notebook.ipynb)| -|End-to-End Workflow |OpenAI |Question-Answering/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/OpenAI_QA_Summarization_Testing_Notebook.ipynb)| -|End-to-End Workflow |AI21 |Question-Answering/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/AI21_QA_Summarization_Testing_Notebook.ipynb)| -|End-to-End Workflow |Cohere |Question-Answering/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/Cohere_QA_Summarization_Testing_Notebook.ipynb)| -|End-to-End Workflow |Hugging Face Inference API |Question-Answering/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb)| -|End-to-End Workflow |Azure-OpenAI |Question-Answering/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/Azure_OpenAI_QA_Summarization_Testing_Notebook.ipynb)| -|Translation |Hugging Face/John Snow Labs |Translation |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/task-specific-notebooks/Translation_Notebook.ipynb)| -|OpenbookQA |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/OpenbookQA_dataset.ipynb)| -|Quac |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/quac_dataset.ipynb)| -|MMLU |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/mmlu_dataset.ipynb)| -|TruthfulQA |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/TruthfulQA_dataset.ipynb)| -|NarrativeQA |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/NarrativeQA_Question_Answering.ipynb)| -|HellaSWag |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/HellaSwag_Question_Answering.ipynb)| -|BBQ |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/BBQ_dataset.ipynb)| -|NQ open |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/NQ_open_dataset.ipynb)| -|BoolQ |OpenAI |Question-Answering |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/BoolQ_dataset.ipynb)| -|XSum |OpenAI |Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/XSum_dataset.ipynb)| -|HuggingFaceDataset-Support |Hugging Face/OpenAI |Text-Classification/Summarization |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/HuggingFace_Dataset_Notebook.ipynb)| -|Augmentation-Control |/John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Augmentation_Control_Notebook.ipynb)| -|Comparing Models |Hugging Face/John Snow Labs/Spacy |NER/Text-Classification |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Comparing_Models_Notebook.ipynb)| -|Runtime Test |Hugging Face/John Snow Labs/Spacy |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/RuntimeTest_Notebook.ipynb)| -|Multiple Variations |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_Variations_Notebook.ipynb)| -|Editing Testcases |Hugging Face |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Editing_TestCases_Notebook.ipynb)| -|Different Report Formats |Spacy |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Different_Report_formats.ipynb)| -|Templatic-Augmentation |John Snow Labs |NER |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Templatic_Augmentation_Notebook.ipynb)| -|Clinical-Tests-Notebook |OpenAI |Clinical-Tests |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Clinical_Tests.ipynb)| -|Prompt-Injection-Notebook |OpenAI |Security |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Prompt_Injections_Tests.ipynb)| +| Tutorial Description | Library | Task | Open In Colab | +| ----------------------------------- | +| Accuracy Tests | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Accuracy_Demo.ipynb) | +| Bias Tests | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Bias_Demo.ipynb) | +| Fairness Tests | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Fairness_Demo.ipynb) | +| Representation Tests | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Representation_Demo.ipynb) | +| Robustness Tests | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Robustness_DEMO.ipynb) | +| Toxicity Test | OpenAI | Toxicity | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Toxicity_NB.ipynb) | +| Add Custom Data | Spacy/Hugging Face | NER/Text-Classification | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/test-specific-notebooks/Add_Custom_Data_Demo.ipynb) | +| End-to-End Workflow | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Notebook.ipynb) | +| End-to-End Custom Pipeline Workflow | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Custom_Pipeline_Notebook.ipynb) | +| End-to-End Workflow | Spacy | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/Spacy_Real_World_Notebook.ipynb) | +| End-to-End Workflow | Hugging Face | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/end-to-end-notebooks/HuggingFace_Real_World_Notebook.ipynb) | +| End-to-End Workflow | OpenAI | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/OpenAI_QA_Summarization_Testing_Notebook.ipynb) | +| End-to-End Workflow | AI21 | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/AI21_QA_Summarization_Testing_Notebook.ipynb) | +| End-to-End Workflow | Cohere | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Cohere_QA_Summarization_Testing_Notebook.ipynb) | +| End-to-End Workflow | Hugging Face Inference API | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/HuggingFaceAPI_QA_Summarization_Testing_Notebook.ipynb) | +| End-to-End Workflow | Hugging Face Hub | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/HuggingFaceHub_QA_Summarization_Testing_Notebook.ipynb) | +| End-to-End Workflow | Azure-OpenAI | Question-Answering/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Azure_OpenAI_QA_Summarization_Testing_Notebook.ipynb) | +| Translation | Hugging Face/John Snow Labs | Translation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/task-specific-notebooks/Translation_Notebook.ipynb) | +| OpenbookQA | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/OpenbookQA_dataset.ipynb) | +| Quac | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/quac_dataset.ipynb) | +| MMLU | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/mmlu_dataset.ipynb) | +| TruthfulQA | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/TruthfulQA_dataset.ipynb) | +| NarrativeQA | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/NarrativeQA_Question_Answering.ipynb) | +| HellaSWag | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/HellaSwag_Question_Answering.ipynb) | +| BBQ | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/BBQ_dataset.ipynb) | +| NQ open | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/NQ_open_dataset.ipynb) | +| BoolQ | OpenAI | Question-Answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/BoolQ_dataset.ipynb) | +| XSum | OpenAI | Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/XSum_dataset.ipynb) | +| HuggingFaceDataset-Support | Hugging Face/OpenAI | Text-Classification/Summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/HuggingFace_Dataset_Notebook.ipynb) | +| Augmentation-Control | /John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Augmentation_Control_Notebook.ipynb) | +| Comparing Models | Hugging Face/John Snow Labs/Spacy | NER/Text-Classification | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Comparing_Models_Notebook.ipynb) | +| Runtime Test | Hugging Face/John Snow Labs/Spacy | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/RuntimeTest_Notebook.ipynb) | +| Multiple Variations | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_Variations_Notebook.ipynb) | +| Editing Testcases | Hugging Face | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Editing_TestCases_Notebook.ipynb) | +| Different Report Formats | Spacy | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Different_Report_formats.ipynb) | +| Templatic-Augmentation | John Snow Labs | NER | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Templatic_Augmentation_Notebook.ipynb) | +| Clinical-Tests-Notebook | OpenAI | Clinical-Tests | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Clinical_Tests.ipynb) | +| Prompt-Injection-Notebook | OpenAI | Security | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Prompt_Injections_Tests.ipynb) |