"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "harness.report()"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "machine_shape": "hm",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "04fad307273b4f54b5b15646efebb157": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "059f8125a73f484cb0b2d4f8a2026624": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_30396d8addf64e62b9aee6fd458b6147",
+ "max": 213450,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_af51a3baa3e94847b557e9f994886a0e",
+ "value": 213450
+ }
+ },
+ "0767a85207994fd1bf8c60e97b42cecc": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "07b117e164a44f79bc582fdda270076d": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "1001db8a1bee424385929d7dd5113352": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_17aa55bf55c7451dbc2a5a8ce5442411",
+ "placeholder": "",
+ "style": "IPY_MODEL_e13ed70114e2470e97814679ca3c143b",
+ "value": " 2.00/2.00 [00:00<00:00, 193B/s]"
+ }
+ },
+ "109fd6ccac294c3e8c690d075bd612e4": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "11843b0f61824383ba8f1477837b372d": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_55496e94dacd473f842c3a061021246d",
+ "placeholder": "",
+ "style": "IPY_MODEL_6cb3964ce93a41d0a691eb26eaf260d6",
+ "value": "tokenizer_config.json: 100%"
+ }
+ },
+ "174d07b3bcb245f38fd50216c7b78a1d": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "17aa55bf55c7451dbc2a5a8ce5442411": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "1967e05f8bd44132919b9856617d1dda": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "22c62124e1f24bb092e575890497b3a4": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "2dfb0cd0b71e4523971ef87c2978ead4": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_9e11e578ef824c5a833e1993e4c37d65",
+ "IPY_MODEL_5ca9c99b0a2f4298851061725876731b",
+ "IPY_MODEL_67ef12076e9e49a2bef4bc630f3b4280"
+ ],
+ "layout": "IPY_MODEL_b82fc8ba2a3c43d89228c6ea299ef0d2"
+ }
+ },
+ "30396d8addf64e62b9aee6fd458b6147": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "30849f0661544814870e640f197bc422": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "3225b9c982b4486dadbcfda73517ea94": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_22c62124e1f24bb092e575890497b3a4",
+ "placeholder": "",
+ "style": "IPY_MODEL_954f6183d22a44df87f121077c4c8626",
+ "value": "special_tokens_map.json: 100%"
+ }
+ },
+ "3b36a4c564954a4db40f0e755af4227a": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "3b9f0b58affa4afd87cc58ee9c65a078": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "499a9cfd951f48a9b93692cb97260dd1": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_f48624c6aa0246228b2aa65fccdf0d51",
+ "max": 112,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_f2a586957ad14110ae3394d50e1b0efd",
+ "value": 112
+ }
+ },
+ "4e6e857f002344ff9a6b342a689f243a": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "500cebec6e4d46a2ba09e3e0ccdf575c": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_07b117e164a44f79bc582fdda270076d",
+ "placeholder": "",
+ "style": "IPY_MODEL_9bc44d3e346542daafdf6b708d17b2d4",
+ "value": " 213k/213k [00:00<00:00, 532kB/s]"
+ }
+ },
+ "5022a84ccefa4c888e7b7283f40ad1f8": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_6a0378e4bdef468ea9633a41f187c100",
+ "placeholder": "",
+ "style": "IPY_MODEL_982e805a22224e7ca21119d6dfe2e661",
+ "value": " 433M/433M [00:05<00:00, 70.3MB/s]"
+ }
+ },
+ "51b19ae99c7f47d38b0cc7460b2fb8e1": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "526a57ea6def48e3bf241c41b8179ddf": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "52b13a75e2bc4291a6039f96dbccbcd3": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_4e6e857f002344ff9a6b342a689f243a",
+ "placeholder": "",
+ "style": "IPY_MODEL_1967e05f8bd44132919b9856617d1dda",
+ "value": " 112/112 [00:00<00:00, 10.3kB/s]"
+ }
+ },
+ "54e485ca393a4c0cad4e06d80287b4e3": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "55496e94dacd473f842c3a061021246d": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "5ba83daef26c4e34b386d974986bcc5a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "5ca9c99b0a2f4298851061725876731b": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_109fd6ccac294c3e8c690d075bd612e4",
+ "max": 829,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_a0a78418c15b4607854d1da5924d501c",
+ "value": 829
+ }
+ },
+ "5e496da2c3d34eea89b16f0e243ef0da": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "65cb9cefe2934ee7a50ca6d4d70bf8ee": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_51b19ae99c7f47d38b0cc7460b2fb8e1",
+ "max": 2,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_7731f14c246043d8a76ff9ea44d0b17a",
+ "value": 2
+ }
+ },
+ "675cd83e139749a4b1641e21cabcafee": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_3b9f0b58affa4afd87cc58ee9c65a078",
+ "placeholder": "",
+ "style": "IPY_MODEL_174d07b3bcb245f38fd50216c7b78a1d",
+ "value": "vocab.txt: 100%"
+ }
+ },
+ "6761482d010040ee8584d40770c0e7b9": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_dcc1386faf57485584383aeda8880d77",
+ "max": 433292294,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_b8cde32f0b0c44d4a3492211ffcda060",
+ "value": 433292294
+ }
+ },
+ "67ef12076e9e49a2bef4bc630f3b4280": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_7426c97a2b9a48ce888df6aa07a18b92",
+ "placeholder": "",
+ "style": "IPY_MODEL_5e496da2c3d34eea89b16f0e243ef0da",
+ "value": " 829/829 [00:00<00:00, 68.9kB/s]"
+ }
+ },
+ "683f3df353e1479e8ae5483df5225dbd": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_d279c6275158449e9ec5f58b391b0069",
+ "IPY_MODEL_65cb9cefe2934ee7a50ca6d4d70bf8ee",
+ "IPY_MODEL_1001db8a1bee424385929d7dd5113352"
+ ],
+ "layout": "IPY_MODEL_de722c2bd03f4e638a877882932cf9eb"
+ }
+ },
+ "6a0378e4bdef468ea9633a41f187c100": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "6b1c659ec6a6418eb446bed941361fc6": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_de8eba29e71e47e5b7f4ec1dfeea28e2",
+ "placeholder": "",
+ "style": "IPY_MODEL_93fbd5ae29424a4ba2f46700d9ece4fb",
+ "value": " 59.0/59.0 [00:00<00:00, 5.05kB/s]"
+ }
+ },
+ "6b3b952b5d4e4d3b8d9f64092273016c": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "6cb3964ce93a41d0a691eb26eaf260d6": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "7216ca2a83d04b389fa9f6b11d6e00d9": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_675cd83e139749a4b1641e21cabcafee",
+ "IPY_MODEL_059f8125a73f484cb0b2d4f8a2026624",
+ "IPY_MODEL_500cebec6e4d46a2ba09e3e0ccdf575c"
+ ],
+ "layout": "IPY_MODEL_7e4121ebd9de4f55a9e8c3dd432a9e83"
+ }
+ },
+ "7426c97a2b9a48ce888df6aa07a18b92": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "7731f14c246043d8a76ff9ea44d0b17a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "7e4121ebd9de4f55a9e8c3dd432a9e83": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "83694568504a4a26ab4d44b2e50f25a4": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "8843bebcd357479a8225e3956586ce34": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "93fbd5ae29424a4ba2f46700d9ece4fb": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "954f6183d22a44df87f121077c4c8626": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "982e805a22224e7ca21119d6dfe2e661": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "9bc44d3e346542daafdf6b708d17b2d4": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "9e11e578ef824c5a833e1993e4c37d65": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_ec53df8dbac94e5d90b131473d01a232",
+ "placeholder": "",
+ "style": "IPY_MODEL_5ba83daef26c4e34b386d974986bcc5a",
+ "value": "config.json: 100%"
+ }
+ },
+ "a0a78418c15b4607854d1da5924d501c": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "af51a3baa3e94847b557e9f994886a0e": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "b82fc8ba2a3c43d89228c6ea299ef0d2": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "b8cde32f0b0c44d4a3492211ffcda060": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "c996405fead84c07aefb48c4e0ed8b58": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_3225b9c982b4486dadbcfda73517ea94",
+ "IPY_MODEL_499a9cfd951f48a9b93692cb97260dd1",
+ "IPY_MODEL_52b13a75e2bc4291a6039f96dbccbcd3"
+ ],
+ "layout": "IPY_MODEL_83694568504a4a26ab4d44b2e50f25a4"
+ }
+ },
+ "cad2ce042df647f181fb192eb3612bca": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_54e485ca393a4c0cad4e06d80287b4e3",
+ "placeholder": "",
+ "style": "IPY_MODEL_6b3b952b5d4e4d3b8d9f64092273016c",
+ "value": "model.safetensors: 100%"
+ }
+ },
+ "d279c6275158449e9ec5f58b391b0069": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_30849f0661544814870e640f197bc422",
+ "placeholder": "",
+ "style": "IPY_MODEL_04fad307273b4f54b5b15646efebb157",
+ "value": "added_tokens.json: 100%"
+ }
+ },
+ "d852ffbc8eab49d7bf805d130a9e21e9": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_cad2ce042df647f181fb192eb3612bca",
+ "IPY_MODEL_6761482d010040ee8584d40770c0e7b9",
+ "IPY_MODEL_5022a84ccefa4c888e7b7283f40ad1f8"
+ ],
+ "layout": "IPY_MODEL_8843bebcd357479a8225e3956586ce34"
+ }
+ },
+ "dcc1386faf57485584383aeda8880d77": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "de722c2bd03f4e638a877882932cf9eb": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "de8eba29e71e47e5b7f4ec1dfeea28e2": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "e13ed70114e2470e97814679ca3c143b": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "e1a46736d7a145e485c8ebfb6e145e65": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_11843b0f61824383ba8f1477837b372d",
+ "IPY_MODEL_e5c31b70aa7b437bb6370d6bf8522cb8",
+ "IPY_MODEL_6b1c659ec6a6418eb446bed941361fc6"
+ ],
+ "layout": "IPY_MODEL_526a57ea6def48e3bf241c41b8179ddf"
+ }
+ },
+ "e5c31b70aa7b437bb6370d6bf8522cb8": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_3b36a4c564954a4db40f0e755af4227a",
+ "max": 59,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_0767a85207994fd1bf8c60e97b42cecc",
+ "value": 59
+ }
+ },
+ "ec53df8dbac94e5d90b131473d01a232": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "f2a586957ad14110ae3394d50e1b0efd": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "f48624c6aa0246228b2aa65fccdf0d51": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ }
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/docs/_includes/docs-langtest-pagination.html b/docs/_includes/docs-langtest-pagination.html
index e01638fcb..0eda22c7f 100644
--- a/docs/_includes/docs-langtest-pagination.html
+++ b/docs/_includes/docs-langtest-pagination.html
@@ -1,12 +1,15 @@
+ 1.7.0
+ 1.6.0
+ 1.5.0
+ 1.4.0
+ 1.3.0
+ 1.2.0
+
+
diff --git a/docs/pages/docs/langtest_versions/latest_release.md b/docs/pages/docs/langtest_versions/latest_release.md
index 0615df8a6..7ee2e2f91 100644
--- a/docs/pages/docs/langtest_versions/latest_release.md
+++ b/docs/pages/docs/langtest_versions/latest_release.md
@@ -10,285 +10,355 @@ modify_date: 2024-04-02
-## 2.1.0
+## 2.2.0
------------------
## 📢 Highlights
-John Snow Labs is thrilled to announce the release of LangTest 2.1.0! This update brings exciting new features and improvements designed to streamline your language model testing workflows and provide deeper insights.
+John Snow Labs is excited to announce the release of LangTest 2.2.0! This update introduces powerful new features and enhancements to elevate your language model testing experience and deliver even greater insights.
-- **🔗 Enhanced API-based LLM Integration:** LangTest now supports testing API-based Large Language Models (LLMs). This allows you to seamlessly integrate diverse LLM models with LangTest and conduct performance evaluations across various datasets.
+- 🏆 **Model Ranking & Leaderboard**: LangTest introduces a comprehensive model ranking system. Use harness.get_leaderboard() to rank models based on various test metrics and retain previous rankings for historical comparison.
-- **📂 Expanded File Format Support:** LangTest 2.1.0 introduces support for additional file formats, further increasing its flexibility in handling different data structures used in LLM testing.
+- 🔍 **Few-Shot Model Evaluation:** Optimize and evaluate your models using few-shot prompt techniques. This feature enables you to assess model performance with minimal data, providing valuable insights into model capabilities with limited examples.
-- **📊 Improved Multi-Dataset Handling:** We've made significant improvements in how LangTest manages multiple datasets. This simplifies workflows and allows for more efficient testing across a wider range of data sources.
+- 📊 **Evaluating NER in LLMs:** This release extends support for Named Entity Recognition (NER) tasks specifically for Large Language Models (LLMs). Evaluate and benchmark LLMs on their NER performance with ease.
-- **🖥️ New Benchmarking Commands**: LangTest now boasts a set of new commands specifically designed for benchmarking language models. These commands provide a structured approach to evaluating model performance and comparing results across different models and datasets.
+- 🚀 **Enhanced Data Augmentation:** The new DataAugmenter module allows for streamlined and harness-free data augmentation, making it simpler to enhance your datasets and improve model robustness.
+
+- 🎯 **Multi-Dataset Prompts:** LangTest now offers optimized prompt handling for multiple datasets, allowing users to add custom prompts for each dataset, enabling seamless integration and efficient testing.
## 🔥 Key Enhancements:
-### **🔗 Streamlined Integration and Enhanced Functionality for API-Based Large Language Models:**
-[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Generic_API-Based_Model_Testing_Demo.ipynb)
-
-This feature empowers you to seamlessly integrate virtually any language model hosted on an external API platform. Whether you prefer OpenAI, Hugging Face, or even custom vLLM solutions, LangTest now adapts to your workflow. `input_processor` and `output_parser` functions are not required for openai api compatible server.
-
-#### Key Features:
-
-- **Effortless API Integration:** Connect to any API system by specifying the API URL, parameters, and a custom function for parsing the returned results. This intuitive approach allows you to leverage your preferred language models with minimal configuration.
+### **🏆 Comprehensive Model Ranking & Leaderboard**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Benchmarking_with_Harness.ipynb)
+The new Model Ranking & Leaderboard system offers a comprehensive way to evaluate and compare model performance based on various metrics across different datasets. This feature allows users to rank models, retain historical rankings, and analyze performance trends.
-- **Customizable Parameters:** Define the URL, parameters specific to your chosen API, and a parsing function tailored to extract the desired output. This level of control ensures compatibility with diverse API structures.
+**Key Features:**
+- **Comprehensive Ranking**: Rank models based on various performance metrics across multiple datasets.
+- **Historical Comparison**: Retain and compare previous rankings for consistent performance tracking.
+- **Dataset-Specific Insights**: Evaluate model performance on different datasets to gain deeper insights.
-- **Unparalleled Flexibility:** Generic API Support removes platform limitations. Now, you can seamlessly integrate language models from various sources, including OpenAI, Hugging Face, and even custom vLLM solutions hosted on private platforms.
-
-#### How it Works:
-
-**Parameters:**
-Define the `input_processer` function for creating a payload and the `output_parser` function is used to extract the output from the response.
-
-```python
-GOOGLE_API_KEY = ""
-model_url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent?key={GOOGLE_API_KEY}"
+**How It Works:**
-# headers
-headers = {
- "Content-Type": "application/json",
-}
+The following are steps to do model ranking and visualize the leaderboard for `google/flan-t5-base` and `google/flan-t5-large` models.
+**1.** Setup and configuration of the Harness are as follows:
-# function to create a payload
-def input_processor(content):
- return {"contents": [
- {
- "role": "user",
- "parts": [
- {
- "text": content
- }
- ]
- }
- ]}
-
-# function to extract output from model response
-def output_parser(response):
- try:
- return response['candidates'][0]['content']['parts'][0]['text']
- except:
- return ""
+```yaml
+# config.yaml
+model_parameters:
+ max_tokens: 64
+ device: 0
+ task: text2text-generation
+tests:
+ defaults:
+ min_pass_rate: 0.65
+ robustness:
+ add_typo:
+ min_pass_rate: 0.7
+ lowercase:
+ min_pass_rate: 0.7
```
-
-To take advantage of this feature, users can utilize the following setup code:
-
```python
from langtest import Harness
-# Initialize Harness with API parameters
harness = Harness(
task="question-answering",
model={
- "model": {
- "url": url,
- "headers": headers,
- "input_processor": input_processor,
- "output_parser": output_parser,
- },
- "hub": "web",
+ "model": "google/flan-t5-base",
+ "hub": "huggingface"
},
- data={
- "data_source": "OpenBookQA",
- "split": "test-tiny",
+ data=[
+ {
+ "data_source": "MedMCQA"
+ },
+ {
+ "data_source": "PubMedQA"
+ },
+ {
+ "data_source": "MMLU"
+ },
+ {
+ "data_source": "MedQA"
+ }
+ ],
+ config="config.yml",
+ benchmarking={
+ "save_dir":"~/.langtest/leaderboard/" # required for benchmarking
}
)
-# Generate, Run and get Report
-harness.generate().run().report()
```
-
+**2**. generate the test cases, run on the model, and get the report as follows:
+```python
+harness.generate().run().report()
+```
+
-### 📂 Streamlined Data Handling and Evaluation
+**3**. Similarly, do the same steps for the `google/flan-t5-large` model with the same `save_dir` path for benchmarking and the same `config.yaml`
-This feature streamlines your testing workflows by enabling LangTest to process a wider range of file formats directly.
+**4**. Finally, the leaderboard can show the model rank by calling the below code.
+```python
+harness.get_leaderboard()
+```
+
-#### Key Features:
+**Conclusion:**
+The Model Ranking & Leaderboard system provides a robust and structured method for evaluating and comparing models across multiple datasets, enabling users to make data-driven decisions and continuously improve model performance.
-- **Effortless File Format Handling:** LangTest now seamlessly ingests data from various file formats, including pickles (.pkl) in addition to previously supported formats. Simply provide the data source path in your harness configuration, and LangTest takes care of the rest.
-- **Simplified Data Source Management**: LangTest intelligently recognizes the file extension and automatically selects the appropriate processing method. This eliminates the need for manual configuration, saving you time and effort.
+### **🔍 Efficient Few-Shot Model Evaluation**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Fewshot_QA_Notebook.ipynb)
+Few-Shot Model Evaluation optimizes and evaluates model performance using minimal data. This feature provides rapid insights into model capabilities, enabling efficient assessment and optimization with limited examples.
-- **Enhanced Maintainability**: The underlying code structure is optimized for flexibility. Adding support for new file formats in the future requires minimal effort, ensuring LangTest stays compatible with evolving data storage practices.
+**Key Features:**
+- **Few-Shot Techniques**: Evaluate models with minimal data to gauge performance quickly.
+- **Optimized Performance**: Improve model outputs using targeted few-shot prompts.
+- **Efficient Evaluation**: Streamlined process for rapid and effective model assessment.
-#### How it works:
+**How It Works:**
+**1.** Set up few-shot prompts tailored to specific evaluation needs.
+```yaml
+# config.yaml
+prompt_config:
+ "BoolQ":
+ instructions: >
+ You are an intelligent bot and it is your responsibility to make sure
+ to give a concise answer. Answer should be `true` or `false`.
+ prompt_type: "instruct" # instruct for completion and chat for conversation(chat models)
+ examples:
+ - user:
+ context: >
+ The Good Fight -- A second 13-episode season premiered on March 4, 2018.
+ On May 2, 2018, the series was renewed for a third season.
+ question: "is there a third series of the good fight?"
+ ai:
+ answer: "True"
+ - user:
+ context: >
+ Lost in Space -- The fate of the castaways is never resolved,
+ as the series was unexpectedly canceled at the end of season 3.
+ question: "did the robinsons ever get back to earth"
+ ai:
+ answer: "True"
+ "NQ-open":
+ instructions: >
+ You are an intelligent bot and it is your responsibility to make sure
+ to give a short concise answer.
+ prompt_type: "instruct" # completion
+ examples:
+ - user:
+ question: "where does the electron come from in beta decay?"
+ ai:
+ answer: "an atomic nucleus"
+ - user:
+ question: "who wrote you're a grand ol flag?"
+ ai:
+ answer: "George M. Cohan"
+
+tests:
+ defaults:
+ min_pass_rate: 0.8
+ robustness:
+ uppercase:
+ min_pass_rate: 0.8
+ add_typo:
+ min_pass_rate: 0.8
+```
+**2.** Initialize the Harness with `config.yaml` file as below code
+```python
+harness = Harness(
+ task="question-answering",
+ model={"model": "gpt-3.5-turbo-instruct","hub":"openai"},
+ data=[{"data_source" :"BoolQ",
+ "split":"test-tiny"},
+ {"data_source" :"NQ-open",
+ "split":"test-tiny"}],
+ config="config.yaml"
+ )
+```
+**3.** Generate the test cases, run them on the model, and then generate the report.
```python
-from langtest import Harness
+harness.generate().run().report()
+```
+
-harness = Harness(
- task="question-answering",
- model={
- "model": "http://localhost:1234/v1/chat/completions",
- "hub": "lm-studio",
- },
- data={
- "data_source": "path/to/file.pkl", #
- },
-)
-# generate, run and report
+**Conclusion:**
+Few-Shot Model Evaluation provides valuable insights into model capabilities with minimal data, allowing for rapid and effective performance optimization. This feature ensures that models can be assessed and improved efficiently, even with limited examples.
+
+
+### **📊 Evaluating NER in LLMs**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/NER%20Casual%20LLM.ipynb)
+Evaluating NER in LLMs enables precise extraction and evaluation of entities using Large Language Models (LLMs). This feature enhances the capability to assess LLM performance on Named Entity Recognition tasks.
+
+**Key Features:**
+- **LLM-Specific Support**: Tailored for evaluating NER tasks using LLMs.
+- **Accurate Entity Extraction**: Improved techniques for precise entity extraction.
+- **Comprehensive Evaluation**: Detailed assessment of entity extraction performance.
+
+**How It Works:**
+**1.** Set up NER tasks for specific LLM evaluation.
+```python
+# Create a Harness object
+harness = Harness(task="ner",
+ model={
+ "model": "gpt-3.5-turbo-instruct",
+ "hub": "openai", },
+ data={
+ "data_source": 'path/to/conll03.conll'
+ },
+ config={
+ "model_parameters": {
+ "temperature": 0,
+ },
+ "tests": {
+ "defaults": {
+ "min_pass_rate": 1.0
+ },
+ "robustness": {
+ "lowercase": {
+ "min_pass_rate": 0.7
+ }
+ },
+ "accuracy": {
+ "min_f1_score": {
+ "min_score": 0.7,
+ },
+ }
+ }
+ }
+ )
+```
+**2.** Generate the test cases based on the configuration in the Harness, run them on the model, and get the report.
+```python
harness.generate().run().report()
```
-### 📊 Multi-Dataset Handling and Evaluation
-[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_dataset.ipynb)
+
+
+Examples:
+
+
+**Conclusion:**
+Evaluating NER in LLMs allows for accurate entity extraction and performance assessment using LangTest's comprehensive evaluation methods. This feature ensures thorough and reliable evaluation of LLMs on Named Entity Recognition tasks.
-This feature empowers you to efficiently benchmark your language models across a wider range of datasets.
-#### Key Features:
+### **🚀 Enhanced Data Augmentation**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Data_Augmenter_Notebook.ipynb)
+Enhanced Data Augmentation introduces a new `DataAugmenter` class, enabling streamlined and harness-free data augmentation. This feature simplifies the process of enriching datasets to improve model robustness and performance.
-- **Effortless Multi-Dataset Testing:** LangTest now seamlessly integrates and executes tests on multiple datasets within a single harness configuration. This streamlined approach eliminates the need for repetitive setups, saving you time and resources.
+**Key Features:**
+- **Harness-Free Augmentation**: Perform data augmentation without the need for harness testing.
+- **Improved Workflow**: Simplified processes for enhancing datasets efficiently.
+- **Robust Models**: Increase model robustness through effective data augmentation techniques.
-- **Enhanced Fairness Evaluation**: By testing models across diverse datasets, LangTest helps identify and mitigate potential biases. This ensures your models perform fairly and accurately on a broader spectrum of data, promoting ethical and responsible AI development.
+**How It Works:**
+The following are steps to import the `DataAugmenter` class from LangTest.
+**1.** Create a config.yaml for the data augmentation.
+```yaml
+# config.yaml
+parameters:
+ type: proportion
+ style: new
+tests:
+ robustness:
+ uppercase:
+ max_proportion: 0.2
+ lowercase:
+ max_proportion: 0.2
+
+```
+**2.** Initialize the `DataAugmenter` class and apply various tests for augmentation to your datasets.
+```python
+from langtest.augmentation import DataAugmenter
+from langtest.tasks.task import TaskManager
+
+data_augmenter = DataAugmenter(
+ task=TaskManager("ner"), # use the ner, text-classification, question-answering...
+ config="config.yaml",
+)
+```
+**3.** Provide the training dataset to `data_augmenter`.
+```python
+data_augmenter.augment(data={
+ 'data_source': 'path/to/conll03.conll'
+})
+```
+**4.** Then, save the augmented dataset.
+```
+data_augmenter.save("augmented.conll")
+```
+**Conclusion:**
+Enhanced Data Augmentation capabilities in LangTest ensure that your models are more robust and capable of handling diverse data scenarios. This feature simplifies the augmentation process, leading to improved model performance and reliability.
-- **Robust Accuracy Assessment:** Multi-dataset support empowers you to conduct more rigorous accuracy testing. By evaluating models on various datasets, you gain a deeper understanding of their strengths and weaknesses across different data distributions. This comprehensive analysis strengthens your confidence in the model's real-world performance.
-#### How it works:
+### **🎯Multi-Dataset Prompts**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/MultiPrompt_MultiDataset.ipynb)
+Multi-Dataset Prompts streamline the process of integrating and testing various data sources by allowing users to define custom prompts for each dataset. This enhancement ensures efficient prompt handling across multiple datasets, enabling comprehensive performance evaluations.
-Initiate the Harness class
+**Key Features:**
+
+- **Custom Prompts:** Add tailored prompts for each dataset to enhance testing accuracy.
+- **Seamless Integration:** Easily incorporate multiple datasets into your testing environment.
+- **Improved Efficiency:** Simplified workflows for handling diverse data sources.
+
+**How It Works:**
+**1.** Initiate the Harness with `BoolQ` and `NQ-open` datasets.
```python
+# Import Harness from the LangTest library
+from langtest import Harness
+
harness = Harness(
task="question-answering",
model={"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
data=[
- {"data_source": "NQ-open", "split": "test-tiny",},
- {"data_source": "MedQA", "split": "test-tiny"},
- {"data_source": "LogiQA", "split": "test-tiny"},
+ {"data_source": "BoolQ", "split": "dev-tiny"},
+ {"data_source": "NQ-open", "split": "test-tiny"}
],
)
```
-Configure the accuracy tests in Harness class
+**2.** Configure prompts specific to each dataset, allowing tailored evaluations.
```python
harness.configure(
{
+ "model_parameters": {
+ "user_prompt": {
+ "BoolQ": "Answer the following question with a True or False. {context}\nQuestion {question}",
+ "NQ-open": "Answer the following question. Question {question}",
+ }
+ },
"tests": {
"defaults": {"min_pass_rate": 0.65},
-
- "accuracy": {
- "llm_eval": {"min_score": 0.60},
- "min_rouge1_score": {"min_score": 0.60},
- "min_rouge2_score": {"min_score": 0.60},
- "min_rougeL_score": {"min_score": 0.60},
- "min_rougeLsum_score": {"min_score": 0.60},
+ "robustness": {
+ "uppercase": {"min_pass_rate": 0.66},
+ "dyslexia_word_swap": {"min_pass_rate": 0.60},
+ "add_abbreviation": {"min_pass_rate": 0.60},
+ "add_slangs": {"min_pass_rate": 0.60},
+ "add_speech_to_text_typo": {"min_pass_rate": 0.60},
},
}
}
)
```
-harness.generate() generates testcases, .run() executes them, and .report() compiles results.
+**3.** Generate the test cases, run them on the model, and get the report.
```python
harness.generate().run().report()
```
-
-
-### 🖥️ Streamlined Evaluation Workflows with Enhanced CLI Commands
-[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Langtest_Cli_Eval_Command.ipynb)
-
-LangTest's evaluation capabilities, focusing on report management and leaderboards. These enhancements empower you to:
-
-- **Streamlined Reporting and Tracking:** Effortlessly save and load detailed evaluation reports directly from the command line using `langtest eval`, enabling efficient performance tracking and comparative analysis over time, with manual file review options in the `~/.langtest` or `./.langtest` folder.
-
-- **Enhanced Leaderboards:** Gain valuable insights with the new langtest show-leaderboard command. This command displays existing leaderboards, providing a centralized view of ranked model performance across evaluations.
-
-- **Average Model Ranking:** Leaderboard now include the average ranking for each evaluated model. This metric provides a comprehensive understanding of model performance across various datasets and tests.
-
-### How it works:
-
-First, create the `parameter.json` or `parameter.yaml` in the working directory
-
-**JSON Format**
-```json
-{
- "task": "question-answering",
- "model": {
- "model": "google/flan-t5-base",
- "hub": "huggingface"
- },
- "data": [
- {
- "data_source": "MedMCQA"
- },
- {
- "data_source": "PubMedQA"
- },
- {
- "data_source": "MMLU"
- },
- {
- "data_source": "MedQA"
- }
- ],
- "config": {
- "model_parameters": {
- "max_tokens": 64,
- "device": 0,
- "task": "text2text-generation"
- },
- "tests": {
- "defaults": {
- "min_pass_rate": 0.70
- },
- "robustness": {
- "add_typo": {
- "min_pass_rate": 0.70
- }
- }
- }
- }
-}
-```
-**Yaml Format**
-```yaml
-task: question-answering
-model:
- model: google/flan-t5-base
- hub: huggingface
-data:
-- data_source: MedMCQA
-- data_source: PubMedQA
-- data_source: MMLU
-- data_source: MedQA
-config:
- model_parameters:
- max_tokens: 64
- device: 0
- task: text2text-generation
- tests:
- defaults:
- min_pass_rate: 0.70
- robustness:
- add_typo:
- min_pass_rate: 0.7
-
-```
-And open the terminal or cmd in your system
-```bash
-langtest eval --model \
- --hub \
- -c < your configuration file like parameter.json or parameter.yaml>
-```
-Finally, we can know the leaderboard and rank of the model.
-
+
-----
-
-To visualize the leaderboard anytime using the CLI command
-```bash
-langtest show-leaderboard
-```
-
+**Conclusion:**
+Multi-dataset prompts in LangTest empower users to efficiently manage and test multiple data sources, resulting in more effective and comprehensive language model evaluations.
## 📒 New Notebooks
+{:.table2}
| Notebooks | Colab Link |
|--------------------|-------------|
-| Generic API-based Model Testing | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Generic_API-Based_Model_Testing_Demo.ipynb)|
-| Multi-Dataset | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_dataset.ipynb) |
-| Langtest Eval Cli Command | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Langtest_Cli_Eval_Command.ipynb) |
-----------------
+| Model Ranking & Leaderboard | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Benchmarking_with_Harness.ipynb)|
+| Fewshot Model Evaluation | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Fewshot_QA_Notebook.ipynb) |
+| Evaluating NER in LLMs | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/NER%20Casual%20LLM.ipynb) |
+| Data Augmenter | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Data_Augmenter_Notebook.ipynb) |
+| Multi-Dataset Prompts | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/MultiPrompt_MultiDataset.ipynb) |
+
{%- include docs-langtest-pagination.html -%}
diff --git a/docs/pages/docs/langtest_versions/release_notes_2_1_0.md b/docs/pages/docs/langtest_versions/release_notes_2_1_0.md
new file mode 100644
index 000000000..926aab701
--- /dev/null
+++ b/docs/pages/docs/langtest_versions/release_notes_2_1_0.md
@@ -0,0 +1,327 @@
+---
+layout: docs
+header: true
+seotitle: LangTest - Deliver Safe and Effective Language Models | John Snow Labs
+title: LangTest Release Notes
+permalink: /docs/pages/docs/langtest_versions/release_notes_2_1_0
+key: docs-release-notes
+modify_date: 2024-04-02
+---
+
+
+
+## 2.1.0
+
+## 📢 Highlights
+
+John Snow Labs is thrilled to announce the release of LangTest 2.1.0! This update brings exciting new features and improvements designed to streamline your language model testing workflows and provide deeper insights.
+
+- **🔗 Enhanced API-based LLM Integration:** LangTest now supports testing API-based Large Language Models (LLMs). This allows you to seamlessly integrate diverse LLM models with LangTest and conduct performance evaluations across various datasets.
+
+- **📂 Expanded File Format Support:** LangTest 2.1.0 introduces support for additional file formats, further increasing its flexibility in handling different data structures used in LLM testing.
+
+- **📊 Improved Multi-Dataset Handling:** We've made significant improvements in how LangTest manages multiple datasets. This simplifies workflows and allows for more efficient testing across a wider range of data sources.
+
+- **🖥️ New Benchmarking Commands**: LangTest now boasts a set of new commands specifically designed for benchmarking language models. These commands provide a structured approach to evaluating model performance and comparing results across different models and datasets.
+
+
+
+## 🔥 Key Enhancements:
+
+### **🔗 Streamlined Integration and Enhanced Functionality for API-Based Large Language Models:**
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Generic_API-Based_Model_Testing_Demo.ipynb)
+
+This feature empowers you to seamlessly integrate virtually any language model hosted on an external API platform. Whether you prefer OpenAI, Hugging Face, or even custom vLLM solutions, LangTest now adapts to your workflow. `input_processor` and `output_parser` functions are not required for openai api compatible server.
+
+#### Key Features:
+
+- **Effortless API Integration:** Connect to any API system by specifying the API URL, parameters, and a custom function for parsing the returned results. This intuitive approach allows you to leverage your preferred language models with minimal configuration.
+
+- **Customizable Parameters:** Define the URL, parameters specific to your chosen API, and a parsing function tailored to extract the desired output. This level of control ensures compatibility with diverse API structures.
+
+- **Unparalleled Flexibility:** Generic API Support removes platform limitations. Now, you can seamlessly integrate language models from various sources, including OpenAI, Hugging Face, and even custom vLLM solutions hosted on private platforms.
+
+#### How it Works:
+
+**Parameters:**
+Define the `input_processer` function for creating a payload and the `output_parser` function is used to extract the output from the response.
+
+```python
+GOOGLE_API_KEY = ""
+model_url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent?key={GOOGLE_API_KEY}"
+
+# headers
+headers = {
+ "Content-Type": "application/json",
+}
+
+# function to create a payload
+def input_processor(content):
+ return {"contents": [
+ {
+ "role": "user",
+ "parts": [
+ {
+ "text": content
+ }
+ ]
+ }
+ ]}
+
+# function to extract output from model response
+def output_parser(response):
+ try:
+ return response['candidates'][0]['content']['parts'][0]['text']
+ except:
+ return ""
+```
+
+To take advantage of this feature, users can utilize the following setup code:
+
+```python
+from langtest import Harness
+
+# Initialize Harness with API parameters
+harness = Harness(
+ task="question-answering",
+ model={
+ "model": {
+ "url": url,
+ "headers": headers,
+ "input_processor": input_processor,
+ "output_parser": output_parser,
+ },
+ "hub": "web",
+ },
+ data={
+ "data_source": "OpenBookQA",
+ "split": "test-tiny",
+ }
+)
+# Generate, Run and get Report
+harness.generate().run().report()
+```
+
+
+
+### 📂 Streamlined Data Handling and Evaluation
+
+This feature streamlines your testing workflows by enabling LangTest to process a wider range of file formats directly.
+
+#### Key Features:
+
+- **Effortless File Format Handling:** LangTest now seamlessly ingests data from various file formats, including pickles (.pkl) in addition to previously supported formats. Simply provide the data source path in your harness configuration, and LangTest takes care of the rest.
+
+- **Simplified Data Source Management**: LangTest intelligently recognizes the file extension and automatically selects the appropriate processing method. This eliminates the need for manual configuration, saving you time and effort.
+
+- **Enhanced Maintainability**: The underlying code structure is optimized for flexibility. Adding support for new file formats in the future requires minimal effort, ensuring LangTest stays compatible with evolving data storage practices.
+
+#### How it works:
+
+```python
+from langtest import Harness
+
+harness = Harness(
+ task="question-answering",
+ model={
+ "model": "http://localhost:1234/v1/chat/completions",
+ "hub": "lm-studio",
+ },
+ data={
+ "data_source": "path/to/file.pkl", #
+ },
+)
+# generate, run and report
+harness.generate().run().report()
+```
+### 📊 Multi-Dataset Handling and Evaluation
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_dataset.ipynb)
+
+This feature empowers you to efficiently benchmark your language models across a wider range of datasets.
+
+#### Key Features:
+
+- **Effortless Multi-Dataset Testing:** LangTest now seamlessly integrates and executes tests on multiple datasets within a single harness configuration. This streamlined approach eliminates the need for repetitive setups, saving you time and resources.
+
+- **Enhanced Fairness Evaluation**: By testing models across diverse datasets, LangTest helps identify and mitigate potential biases. This ensures your models perform fairly and accurately on a broader spectrum of data, promoting ethical and responsible AI development.
+
+- **Robust Accuracy Assessment:** Multi-dataset support empowers you to conduct more rigorous accuracy testing. By evaluating models on various datasets, you gain a deeper understanding of their strengths and weaknesses across different data distributions. This comprehensive analysis strengthens your confidence in the model's real-world performance.
+
+#### How it works:
+
+Initiate the Harness class
+```python
+harness = Harness(
+ task="question-answering",
+ model={"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ data=[
+ {"data_source": "NQ-open", "split": "test-tiny",},
+ {"data_source": "MedQA", "split": "test-tiny"},
+ {"data_source": "LogiQA", "split": "test-tiny"},
+ ],
+)
+```
+Configure the accuracy tests in Harness class
+```python
+harness.configure(
+ {
+ "tests": {
+ "defaults": {"min_pass_rate": 0.65},
+
+ "accuracy": {
+ "llm_eval": {"min_score": 0.60},
+ "min_rouge1_score": {"min_score": 0.60},
+ "min_rouge2_score": {"min_score": 0.60},
+ "min_rougeL_score": {"min_score": 0.60},
+ "min_rougeLsum_score": {"min_score": 0.60},
+ },
+ }
+ }
+)
+```
+harness.generate() generates testcases, .run() executes them, and .report() compiles results.
+```python
+harness.generate().run().report()
+```
+
+
+### 🖥️ Streamlined Evaluation Workflows with Enhanced CLI Commands
+[](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Langtest_Cli_Eval_Command.ipynb)
+
+LangTest's evaluation capabilities, focusing on report management and leaderboards. These enhancements empower you to:
+
+- **Streamlined Reporting and Tracking:** Effortlessly save and load detailed evaluation reports directly from the command line using `langtest eval`, enabling efficient performance tracking and comparative analysis over time, with manual file review options in the `~/.langtest` or `./.langtest` folder.
+
+- **Enhanced Leaderboards:** Gain valuable insights with the new langtest show-leaderboard command. This command displays existing leaderboards, providing a centralized view of ranked model performance across evaluations.
+
+- **Average Model Ranking:** Leaderboard now include the average ranking for each evaluated model. This metric provides a comprehensive understanding of model performance across various datasets and tests.
+
+### How it works:
+
+First, create the `parameter.json` or `parameter.yaml` in the working directory
+
+**JSON Format**
+```json
+{
+ "task": "question-answering",
+ "model": {
+ "model": "google/flan-t5-base",
+ "hub": "huggingface"
+ },
+ "data": [
+ {
+ "data_source": "MedMCQA"
+ },
+ {
+ "data_source": "PubMedQA"
+ },
+ {
+ "data_source": "MMLU"
+ },
+ {
+ "data_source": "MedQA"
+ }
+ ],
+ "config": {
+ "model_parameters": {
+ "max_tokens": 64,
+ "device": 0,
+ "task": "text2text-generation"
+ },
+ "tests": {
+ "defaults": {
+ "min_pass_rate": 0.70
+ },
+ "robustness": {
+ "add_typo": {
+ "min_pass_rate": 0.70
+ }
+ }
+ }
+ }
+}
+```
+**Yaml Format**
+```yaml
+task: question-answering
+model:
+ model: google/flan-t5-base
+ hub: huggingface
+data:
+- data_source: MedMCQA
+- data_source: PubMedQA
+- data_source: MMLU
+- data_source: MedQA
+config:
+ model_parameters:
+ max_tokens: 64
+ device: 0
+ task: text2text-generation
+ tests:
+ defaults:
+ min_pass_rate: 0.70
+ robustness:
+ add_typo:
+ min_pass_rate: 0.7
+
+```
+And open the terminal or cmd in your system
+```bash
+langtest eval --model \
+ --hub \
+ -c < your configuration file like parameter.json or parameter.yaml>
+```
+Finally, we can know the leaderboard and rank of the model.
+
+
+----
+
+To visualize the leaderboard anytime using the CLI command
+```bash
+langtest show-leaderboard
+```
+
+
+## 📒 New Notebooks
+
+{:.table2}
+| Notebooks | Colab Link |
+|--------------------|-------------|
+| Generic API-based Model Testing | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Generic_API-Based_Model_Testing_Demo.ipynb)|
+| Multi-Dataset | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_dataset.ipynb) |
+| Langtest Eval Cli Command | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Langtest_Cli_Eval_Command.ipynb) |
+
+
+## 🐛 Fixes
+
+- Fixed multi-dataset support for accuracy task [#998]
+- Fixed bugs in langtest package [#1003][#1004]
+
+
+## ⚡ Enhancements
+- Improved the error handling in Harness run method [#990]
+- Websites Updates [#1001]
+- Updated new version for dependencies [#992]
+- Improved the data augmentation for Question-Answering task [#991]
+
+## What's Changed
+
+* Feautre/integration with web api by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/986
+* Refactor TestFactory class to handle exceptions in async tests by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/990
+* data augmentation support for question-answering task by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/991
+* Updated dependencies by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/992
+* Fix/implement the multiple dataset support for accuracy tests by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/998
+* Feature/add support for other file formats by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/993
+* Bug Fix: Generated results are none by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/1000
+* Feature/implement load & save for benchmark reports by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/999
+* Fix/bug fixes langtest 2 1 0 rc1 by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/1003
+* website updates by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/1001
+* Fix/bug fixes langtest 2 1 0 rc1 by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/1004
+* Release/2.0.1 by @chakravarthik27 in https://github.com/JohnSnowLabs/langtest/pull/1005
+
+
+**Full Changelog**: https://github.com/JohnSnowLabs/langtest/compare/2.0.0...2.1.0
+
+## ⚒️ Previous Versions
+
+{%- include docs-langtest-pagination.html -%}
diff --git a/docs/pages/tutorials/LLM_testing_Notebooks/llm_testing_notebooks.md b/docs/pages/tutorials/LLM_testing_Notebooks/llm_testing_notebooks.md
index aa0b67247..dcb763825 100644
--- a/docs/pages/tutorials/LLM_testing_Notebooks/llm_testing_notebooks.md
+++ b/docs/pages/tutorials/LLM_testing_Notebooks/llm_testing_notebooks.md
@@ -39,4 +39,6 @@ The following table gives an overview of the different tutorial notebooks to tes
| [**Sycophancy**](sycophancy) : It is an undesirable behavior in which models tailor their responses to align with a human user's view, even when that view is not objectively correct. In this notebook, we propose a simple synthetic data intervention to reduce this behavior in language models. | OpenAI | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Sycophancy.ipynb) |
| [**Stereotype**](stereotype): In this tutorial, we assess the model on gender occupational stereotype statements. | OpenAI/AI21 | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Wino_Bias_LLM.ipynb) |
| [**LM Studio**](lm_studio): Running Hugging Face quantized models through LM-Studio and testing these models for a Question Answering task. | LM Studio | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/LM-Studio-Demo.ipynb) |
-| [**Question Answering Benchmarking**](question_answering_benchmarking): This notebook provides a demo on benchmarking Language Models (LLMs) for Question-Answering tasks. | Hugging Face Inference API | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Question-Answering.ipynb) |
\ No newline at end of file
+| [**Question Answering Benchmarking**](question_answering_benchmarking): This notebook provides a demo on benchmarking Language Models (LLMs) for Question-Answering tasks. | Hugging Face Inference API | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/benchmarks/Question-Answering.ipynb) |
+| **Fewshot Model Evaluation**: This notebook provides a demo on Optimize and evaluate your models using few-shot prompt techniques | OpenAI | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Fewshot_QA_Notebook.ipynb) |
+| **Evaluating NER in LLMs**:In this tutorial, we assess the support for Named Entity Recognition (NER) tasks specifically for Large Language Models (LLMs) | OpenAI | Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/NER%20Casual%20LLM.ipynb) |
\ No newline at end of file
diff --git a/docs/pages/tutorials/miscellaneous_notebooks/miscellaneous_notebooks.md b/docs/pages/tutorials/miscellaneous_notebooks/miscellaneous_notebooks.md
index 13c26c50e..530ffd633 100644
--- a/docs/pages/tutorials/miscellaneous_notebooks/miscellaneous_notebooks.md
+++ b/docs/pages/tutorials/miscellaneous_notebooks/miscellaneous_notebooks.md
@@ -39,3 +39,5 @@ The following table gives an overview of the different tutorial notebooks. In th
| **LangTestCallback**: In this section, we discussed how to utilize the LangTestCallback funtion while training an Text Classification transformers model. | Hugging Face | Text-Classification | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/HF_Callback_Text_Classification.ipynb) |
| **Multiple_dataset**: In this section, we discussed how to evaluate multiple datasets for a particular model. | OpenAI |Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Multiple_dataset.ipynb) |
| **Generic API-Based Model**: In this section, we discussed how to test API-based models hosted using Ollama, vLLM, and other tools. | Web |Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/Generic_API-Based_Model_Testing_Demo.ipynb) |
+| **Data Augmenter**: In this Notebook, we can allows for streamlined and harness-free data augmentation, making it simpler to enhance your datasets and improve model robustness. | - |NER | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/Data_Augmenter_Notebook.ipynb) |
+| **Multi-Dataset Prompt Configs**: In this Notebook, we discussed about optimized prompt handling for multiple datasets, allowing users to add custom prompts for each dataset, enabling seamless integration and efficient testing. | OpenAI |Question-Answering | [](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/misc/MultiPrompt_MultiDataset.ipynb) |
diff --git a/langtest/augmentation/__init__.py b/langtest/augmentation/__init__.py
index 26ae43e89..e1c70a2a3 100644
--- a/langtest/augmentation/__init__.py
+++ b/langtest/augmentation/__init__.py
@@ -1,616 +1,4 @@
-import os
-import random
-import re
-import string
-from abc import ABC, abstractmethod
-from collections import defaultdict
-from copy import deepcopy as copy
-from typing import Any, Dict, List, Optional, Union
+from .base import BaseAugmentaion, AugmentRobustness, TemplaticAugment
+from .augmenter import DataAugmenter
-import pandas as pd
-import yaml
-
-from langtest.datahandler.datasource import DataFactory, HuggingFaceDataset
-from langtest.transform import TestFactory
-from langtest.transform.utils import create_terminology
-from langtest.utils.custom_types import Sample
-from langtest.utils.custom_types.output import NEROutput
-from langtest.utils.custom_types.predictions import NERPrediction, SequenceLabel
-from langtest.utils.custom_types.sample import NERSample
-from langtest.tasks import TaskManager
-from ..utils.lib_manager import try_import_lib
-from ..errors import Errors
-
-
-class BaseAugmentaion(ABC):
- """Abstract base class for data augmentation techniques.
-
- Methods:
- fix: Abstract method that should be implemented by child classes.
- This method should perform the data augmentation operation.
- """
-
- @abstractmethod
- def fix(self, *args, **kwargs):
- """Abstract method that should be implemented by child classes.
-
- This method should perform the data augmentation operation.
-
- Returns:
- NotImplementedError: Raised if the method is not implemented by child classes.
- """
- return NotImplementedError
-
-
-class AugmentRobustness(BaseAugmentaion):
- """A class for performing a specified task with historical results.
-
- Attributes:
- task (str): A string indicating the task being performed.
- config (dict): A dictionary containing configuration parameters for the task.
- h_report (pandas.DataFrame): A DataFrame containing a report of historical results for the task.
- max_prop (float): The maximum proportion of improvement that can be suggested by the class methods.
- Defaults to 0.5.
-
- Methods:
- __init__(self, task, h_report, config, max_prop=0.5) -> None:
- Initializes an instance of MyClass with the specified parameters.
-
- fix(self) -> List[Sample]:
- .
-
- suggestions(self, prop) -> pandas.DataFrame:
- Calculates suggestions for improving test performance based on a given report.
- """
-
- def __init__(
- self,
- task: TaskManager,
- h_report: "pd.DataFrame",
- config: Dict,
- custom_proportions: Union[Dict, List] = None,
- max_prop=0.5,
- ) -> None:
- """Initializes an instance of MyClass with the specified parameters.
-
- Args:
- task (str): A string indicating the task being performed.
- h_report (pandas.DataFrame): A DataFrame containing a report of historical results for the task.
- config (dict): A dictionary containing configuration parameters for the task.
- custom_proportions
- max_prop (float): The maximum proportion of improvement that can be suggested by the class methods.
- Defaults to 0.5.
-
- Returns:
- None
-
- """
- super().__init__()
- self.task = task
- self.config = config
- self.h_report = h_report
- self.max_prop = max_prop
- self.custom_proportions = custom_proportions
-
- if isinstance(self.config, str):
- with open(self.config) as fread:
- self.config = yaml.safe_load(fread)
-
- def fix(
- self,
- training_data: dict,
- output_path: str,
- export_mode: str = "add",
- ):
- """Applies perturbations to the input data based on the recommendations from harness reports.
-
- Args:
- training_data (dict): A dictionary containing the input data for augmentation.
- output_path (str): The path to save the augmented data file.
- export_mode (str, optional): Determines how the samples are modified or exported.
- - 'inplace': Modifies the list of samples in place.
- - 'add': Adds new samples to the input data.
- - 'transformed': Exports only the transformed data, excluding untransformed samples.
- Defaults to 'add'.
-
- Returns:
- List[Dict[str, Any]]: A list of augmented data samples.
- """
-
- # if "source" in training_data and training_data["source"] == "huggingface":
- # self.df = HuggingFaceDataset(training_data, self.task)
- # data = self.df.load_data(
- # feature_column=training_data.get("feature_column", "text"),
- # target_column=training_data.get("target_column", "label"),
- # split=training_data.get("split", "test"),
- # subset=training_data.get("subset", None),
- # )
- # else:
- self.df = DataFactory(training_data, self.task)
- data = self.df.load()
- TestFactory.is_augment = True
- supported_tests = TestFactory.test_scenarios()
- suggest: pd.DataFrame = self.suggestions(self.h_report)
- sum_propotion = suggest["proportion_increase"].sum()
- if suggest.shape[0] <= 0 or suggest.empty:
- print("All tests have passed. Augmentation will not be applied in this case.")
- return None
-
- self.config = self._parameters_overrides(self.config, data)
-
- final_aug_data = []
- hash_map = {k: v for k, v in enumerate(data)}
- transformed_data = []
- for proportion in suggest.iterrows():
- cat = proportion[-1]["category"].lower()
- if cat not in ["robustness", "bias"]:
- continue
- test = proportion[-1]["test_type"].lower()
- test_type = {cat: {test: self.config.get("tests").get(cat).get(test)}}
- if proportion[-1]["test_type"] in supported_tests[cat]:
- sample_length = (
- len(data)
- * self.max_prop
- * (proportion[-1]["proportion_increase"] / sum_propotion)
- )
- if export_mode in ("inplace"):
- sample_indices = random.sample(
- range(0, len(data)), int(sample_length)
- )
- for each in sample_indices:
- if test == "swap_entities":
- test_type["robustness"]["swap_entities"]["parameters"][
- "labels"
- ] = [self.label[each]]
- res = TestFactory.transform(
- self.task, [hash_map[each]], test_type
- )
- if len(res) == 0:
- continue
- hash_map[each] = res[0]
- else:
- if test == "swap_entities":
- sample_data = data[: int(sample_length)]
- test_type["robustness"]["swap_entities"]["parameters"][
- "labels"
- ] = test_type["robustness"]["swap_entities"]["parameters"][
- "labels"
- ][
- : int(sample_length)
- ]
- else:
- sample_data = random.choices(data, k=int(sample_length))
- aug_data = TestFactory.transform(self.task, sample_data, test_type)
- final_aug_data.extend(aug_data)
-
- if export_mode == "transformed":
- transformed_data.extend(aug_data)
- if "." not in training_data["data_source"]:
- if export_mode == "inplace":
- final_aug_data = list(hash_map.values())
- self.df.export(final_aug_data, output_path)
- elif export_mode == "transformed":
- self.df.export(transformed_data, output_path)
- else:
- data.extend(final_aug_data)
- self.df.export(data, output_path)
-
- TestFactory.is_augment = False
- return final_aug_data
-
- else:
- if export_mode == "inplace":
- final_aug_data = list(hash_map.values())
- self.df.export(final_aug_data, output_path)
- elif export_mode == "transformed":
- self.df.export(transformed_data, output_path)
- else:
- data.extend(final_aug_data)
- self.df.export(data, output_path)
-
- TestFactory.is_augment = False
- return final_aug_data
-
- def suggestions(self, report: "pd.DataFrame") -> "pd.DataFrame":
- """Calculates suggestions for improving test performance based on a given report.
-
- Args:
- report (pandas.DataFrame): A DataFrame containing test results by category and test type,
- including pass rates and minimum pass rates.
-
- Returns:
- pandas.DataFrame: A DataFrame containing the following columns for each suggestion:
- - category: the test category
- - test_type: the type of test
- - ratio: the pass rate divided by the minimum pass rate for the test
- - proportion_increase: a proportion indicating how much the pass rate
- should increase to reach the minimum pass rate
- """
- report["ratio"] = report["pass_rate"] / report["minimum_pass_rate"]
-
- if self.custom_proportions and isinstance(self.custom_proportions, dict):
- report["proportion_increase"] = report["test_type"].map(
- self.custom_proportions
- )
- elif self.custom_proportions and isinstance(self.custom_proportions, list):
- report["proportion_increase"] = report["ratio"].apply(self._proportion_values)
- report = report[report["test_type"].isin(self.custom_proportions)]
- else:
- report["proportion_increase"] = report["ratio"].apply(self._proportion_values)
-
- report = report.dropna(subset=["proportion_increase"])[
- ["category", "test_type", "ratio", "proportion_increase"]
- ]
- return report
-
- @staticmethod
- def _proportion_values(x: float) -> Optional[float]:
- """Calculates a proportion indicating how much a pass rate should increase to reach a minimum pass rate.
-
- Args:
- x (float): The ratio of the pass rate to the minimum pass rate for a given test.
-
- Returns:
- float: A proportion indicating how much the pass rate should increase to reach the minimum pass rate.
- If the pass rate is greater than or equal to the minimum pass rate, returns None.
- If the pass rate is between 0.9 and 1.0 times the minimum pass rate, returns 0.05.
- If the pass rate is between 0.8 and 0.9 times the minimum pass rate, returns 0.1.
- If the pass rate is between 0.7 and 0.8 times the minimum pass rate, returns 0.2.
- If the pass rate is less than 0.7 times the minimum pass rate, returns 0.3.
-
- """
- if x >= 1:
- return None
- elif x > 0.9:
- return 0.05
- elif x > 0.8:
- return 0.1
- elif x > 0.7:
- return 0.2
- else:
- return 0.3
-
- def _parameters_overrides(self, config: dict, data_handler: List[Sample]) -> dict:
- tests = config.get("tests", {}).get("robustness", {})
- if "swap_entities" in config.get("tests", {}).get("robustness", {}):
- df = pd.DataFrame(
- {
- "text": [sample.original for sample in data_handler],
- "label": [
- [i.entity for i in sample.expected_results.predictions]
- for sample in data_handler
- ],
- }
- )
- params = tests["swap_entities"]
- params["parameters"] = {}
- params["parameters"]["terminology"] = create_terminology(df)
- params["parameters"]["labels"] = df.label.tolist()
- self.label = (
- self.config.get("tests")
- .get("robustness")
- .get("swap_entities")
- .get("parameters")
- .get("labels")
- )
- return config
-
-
-class TemplaticAugment(BaseAugmentaion):
- """This class is used for templatic augmentation. It is a subclass of the BaseAugmentation class.
-
- Attributes:
- __templates:
- A string or a list of strings or samples that represents the templates for the augmentation.
- __task:
- The task for which the augmentation is being performed.
- __generate_templates:
- if set to True, generates sample templates from the given ones.
- __show_templates:
- if set to True, displays the used templates.
-
-
- Methods:
- __init__(self, templates: Union[str, List[str]], task: str):
- Initializes the TemplaticAugment class.
- fix(self, training_data: str, output_path: str, *args, **kwargs):
- Performs the templatic augmentation and exports the results to a specified path.
- """
-
- def __init__(
- self,
- templates: Union[str, List[str]],
- task: TaskManager,
- generate_templates=False,
- show_templates=False,
- ) -> None:
- """This constructor for the TemplaticAugment class.
-
- Args:
- templates (Union[str, List[str]]): The templates to be used for the augmentation.
- task (str): The task for which the augmentation is being performed.
- generate_templates (bool, optional): if set to True, generates sample templates from the given ones.
- show_templates (bool, optional): if set to True, displays the used templates.
- """
- self.__templates: Union[str, List[str], List[Sample]] = templates
- self.__task = task
-
- if generate_templates:
- if try_import_lib("openai"):
- import openai
-
- given_template = self.__templates[:]
- for template in given_template:
- prompt = f"""Based on the template provided, create 10 new and unique templates that are variations on this theme. Present these as a Python list, with each template as a quoted string. The list should contain only the templates without any additional text or explanation.
-
- Template:
- "{template}"
-
- Expected Python List Output:
- ['Template 1', 'Template 2', 'Template 3', ...] # Replace with actual generated templates
- """
-
- response = openai.Completion.create(
- engine="gpt-3.5-turbo-instruct",
- prompt=prompt,
- max_tokens=500,
- temperature=0,
- )
-
- generated_response = response.choices[0].text.strip()
- # Process the generated response
- if generated_response:
- # Assuming the response format is a Python-like list in a string
- templates_list = generated_response.strip("[]").split('",')
- templates_list = [
- template.strip().strip('"')
- for template in templates_list
- if template.strip()
- ]
-
- # Extend the existing templates list
- self.__templates.extend(templates_list)
- else:
- print("No response or unexpected format.")
-
- else:
- raise RuntimeError(Errors.E084)
-
- if show_templates:
- [print(template) for template in self.__templates]
-
- if isinstance(self.__templates, str) and os.path.exists(self.__templates):
- self.__templates = DataFactory(self.__templates, self.__task).load()
- elif isinstance(self.__templates, str):
- self.__templates = [self.str_to_sample(self.__templates)]
- elif isinstance(self.__templates, list) and isinstance(self.__templates[0], str):
- self.__templates = [self.str_to_sample(i) for i in self.__templates]
-
- def fix(
- self,
- training_data: Dict[str, Any],
- output_path: str,
- max_num: int = None,
- append_original: bool = False,
- *args,
- **kwargs,
- ) -> bool:
- """This method is used to perform the templatic augmentation.
-
- It takes the input data, performs the augmentation and then saves the augmented data to the output path.
-
- Args:
- training_data (dict): A dictionary containing the input data for augmentation.
- output_path (str): The path where the augmented data will be saved.
- max_num (int): Maximum number of new samples to generate
- append_original (bool, optional): If set to True, appends the original data to the augmented data. Defaults to False.
- *args: Variable length argument list.
- **kwargs: Arbitrary keyword arguments.
-
- Returns:
- bool: Returns True upon successful completion of the method.
- """
- df = DataFactory(training_data, self.__task)
- data = df.load()
- new_data = (
- data.copy()
- if isinstance(data, (pd.DataFrame, pd.Series))
- else copy.deepcopy(data)
- if append_original
- else []
- )
- self.__search_results = self.search_sample_results(data)
-
- if not max_num:
- max_num = max(len(i) for i in self.__search_results.values())
-
- for template in self.__templates:
- for _ in range(max_num):
- new_sample = self.new_sample(template)
- if new_sample:
- new_data.append(new_sample)
-
- df.export(new_data, output_path)
- return True
-
- @staticmethod
- def search_sample_results(
- samples: List[Sample],
- ) -> Dict[str, List[Union[NERPrediction, SequenceLabel]]]:
- """This method is used to search the results of the samples for the entities in the templates.
-
- Args:
- samples (List[Sample]): The samples for which the results are to be searched.
-
- Returns:
- Dict[str, List[Union[NERPrediction, SequenceLabel]]]: A dictionary containing the search results.
- """
- results_dict = defaultdict(list)
- for sample in samples:
- chunk = []
- ent_name = ""
- for result in sample.expected_results.predictions:
- ent = result.entity.split("-")[-1]
- if ent != "O" and ent_name == "":
- ent_name = ent
- if result.entity.endswith(ent_name) and ent != "O":
- result.doc_id = 0
- result.doc_name = ""
- chunk.append(result)
- elif len(chunk) > 0:
- results_dict[ent_name].append(tuple(chunk))
- ent_name = ""
- chunk = []
-
- if chunk:
- results_dict[ent_name].append(tuple(chunk))
- return results_dict
-
- @staticmethod
- def extract_variable_names(f_string: str) -> List[str]:
- """This method is used to extract the variable names from the templates.
-
- Args:
- f_string (str): The template string.
-
- Returns:
- List[str]: A list of variable names.
- """
- pattern = r"{([^{}]*)}"
- matches = re.findall(pattern, f_string)
- variable_names = [match.strip() for match in matches]
- return variable_names
-
- def new_sample(self, template: Sample):
- """This method is used to generate a new sample from a template.
-
- Args:
- template (Sample): The template from which the new sample is to be generated.
-
- Returns:
- Sample: The new sample generated from the template.
- """
- template = copy(template)
- matches = re.finditer(r"{([^{}]*)}", template.original)
- cursor = 0
- other_predictions = []
- if matches:
- for match in matches:
- entity = match.group(1)
- if entity in self.__search_results:
- prediction = random.choice(self.__search_results[entity])
- word = " ".join(
- i.span.word for i in prediction if isinstance(i, NERPrediction)
- )
-
- template.original = template.original.replace(
- "{" + entity + "}", word, 1
- )
- for result in template.expected_results.predictions[cursor:]:
- if prediction[0].entity.endswith(result.entity):
- for each_prediction in prediction:
- if isinstance(each_prediction, NERPrediction):
- each_prediction.chunk_tag = "-X-"
- each_prediction.pos_tag = "-X-"
- other_predictions.extend(prediction)
- cursor += 1
- break
- else:
- if "{" in result.span.word and "}" in result.span.word:
- continue
- other_predictions.append(result)
- cursor += 1
- else:
- continue
- template.expected_results.predictions = (
- other_predictions + template.expected_results.predictions[cursor:]
- )
- return template
- else:
- return None
-
- def str_to_sample(self, template: str):
- """This method is used to convert a template string to a Sample object.
-
- Args:
- template (str): The template string to be converted.
-
- Returns:
- Sample: The Sample object generated from the template string.
- """
- if self.__task == "ner":
- template = self.add_spaces_around_punctuation(template)
- sample = NERSample()
- sample.original = template
- words = template.split()
- predictions = []
- cursor = 0
- for word in words:
- if "{" in word and "}" in word:
- entity = word.replace("{", "").replace("}", "")
- else:
- entity = "O"
- predictions.append(
- NERPrediction.from_span(
- entity,
- word,
- cursor,
- cursor + len(word),
- pos_tag="-X-",
- chunk_tag="-X-",
- doc_id=0,
- doc_name="",
- )
- )
- cursor += len(word) + 1
- sample.expected_results = NEROutput(predictions=predictions)
-
- elif self.__task == "text-classification":
- raise NotImplementedError
-
- return sample
-
- @property
- def templates(self):
- """Templates getter"""
- return self.__templates
-
- @templates.setter
- def templates(self, templates: Union[str, List[str]]):
- self.__init__(templates, self.__task)
-
- @property
- def task(self):
- """Task getter"""
- return self.__task
-
- @task.setter
- def task(self, task: str):
- self.__task = task
-
- @staticmethod
- def add_spaces_around_punctuation(text: str):
- """This method is used to add spaces around punctuation in a string.
-
- Args:
- text (str): The string to which spaces are to be added.
-
- Returns:
- str: The string with spaces added around punctuation.
- """
- for punct in string.punctuation:
- if punct not in ["{", "}", "_"]:
- if punct == ".":
- # To prevent spaces being added around decimal points
- text = re.sub(r"(\d)\.(\d)", r"\1[DOT]\2", text)
-
- text = text.replace(punct, f" {punct} ")
-
- if punct == ".":
- # Putting back the decimal points to original state
- text = text.replace("[DOT]", ".")
-
- # Removing extra spaces
- text = re.sub(r"\s+", " ", text).strip()
-
- return text
+__all__ = ["BaseAugmentaion", "AugmentRobustness", "TemplaticAugment", "DataAugmenter"]
diff --git a/langtest/augmentation/augmenter.py b/langtest/augmentation/augmenter.py
new file mode 100644
index 000000000..96b5125e1
--- /dev/null
+++ b/langtest/augmentation/augmenter.py
@@ -0,0 +1,184 @@
+import random
+import yaml
+
+from typing import Any, Dict, Iterable, Union
+from langtest.datahandler.datasource import DataFactory
+from langtest.transform import TestFactory
+from langtest.tasks.task import TaskManager
+
+
+class DataAugmenter:
+ def __init__(self, task: Union[str, TaskManager], config: Union[str, dict]) -> None:
+ """
+ Initialize the DataAugmenter.
+
+ Args:
+ config (Union[str, dict]): Configuration file or dictionary.
+ task (Union[str, TaskManager]): Task Manager.
+ columns_info ([type], optional): Columns information. Defaults to None.
+
+ """
+
+ self.__config = config
+ if isinstance(config, str):
+ self.__config = self.load_config(config)
+
+ self.__tests: dict = self.__config.get("tests", [])
+ if isinstance(task, str):
+ if task in ["ner", "text-classification", "question-answering"]:
+ task = TaskManager(task)
+ else:
+ raise ValueError(
+ "check the task name. It should be one of the following: ner, text-classification, question-answering"
+ )
+ self.__task = task
+
+ # Test Factory and Data Factory
+ self.__testfactory = TestFactory
+ self.__datafactory = DataFactory
+
+ self.__testfactory.is_augment = True
+
+ # parameters
+ self.__max_proportion = self.__tests.get("defaults", {}).get(
+ "max_proportion", 0.6
+ )
+ # self.__ntests = len(v for k, v in self.__tests.items()) - 1
+ self.__type = self.__config.get("parameters", {}).get("type", "proportion")
+ self.__style = self.__config.get("parameters", {}).get("style", "extend")
+
+ self.__df_config = self.__config_df()
+
+ def load_config(self, config: str) -> dict:
+ """
+ Load the configuration file.
+ """
+ with open(config, "r") as f:
+ return yaml.safe_load(f)
+
+ def augment(self, data: Union[str, Iterable]) -> str:
+ """
+ Augment the content.
+ """
+ # load the data
+ if isinstance(data, dict):
+ self.__datafactory = self.__datafactory(file_path=data, task=self.__task)
+ data = self.__datafactory.load()
+
+ # check the style of augmentation to be applied. Default is extend
+ if self.__style == "extend":
+ self.extend(data)
+ elif self.__style == "inplace":
+ self.inplace(data)
+ elif self.__style == "new":
+ self.new_data(data)
+ else:
+ raise ValueError("Invalid style")
+
+ return self
+
+ def extend(self, data: Iterable) -> "DataAugmenter":
+ """
+ Extend the content.
+ """
+ # calculate the number of rows to be added
+ n = len(data)
+
+ data_cut = random.sample(data, int(n * self.__max_proportion))
+
+ test_cases: list = self.__testfactory.transform(
+ self.__task, data_cut, self.__tests
+ )
+
+ self.__augmented_data = [*data, *test_cases] if isinstance(data, list) else data
+
+ return self
+
+ def inplace(self, data: Iterable) -> "DataAugmenter":
+ """
+ Inplace augmentation.
+ """
+ # calculate the number of rows to be added
+ size = int(len(data) * self.__max_proportion)
+
+ # create a dictionary with index as key and data as value
+ data_dict = self.prepare_hash_map(data)
+
+ # select random rows based on the size with its index
+ selected = random.sample(data_dict.keys(), int(size))
+
+ for idx in selected:
+ test_cases = self.__testfactory.transform(
+ self.__task, [data_dict[idx]], self.__tests
+ )
+ data_dict[idx] = test_cases[0] if test_cases else data_dict[idx]
+
+ self.__augmented_data = data_dict.values()
+
+ return self
+
+ def new_data(self, data: Iterable) -> "DataAugmenter":
+ """
+ Create new data.
+ """
+ # calculate the number of rows to be added
+ size = int(len(data) * self.__max_proportion)
+
+ data_cut = random.sample(data, size)
+
+ test_cases = self.__testfactory.transform(self.__task, data_cut, self.__tests)
+
+ self.__augmented_data = test_cases
+
+ return self
+
+ def size(self, category: str, test_name: str) -> int:
+ return (
+ self.__max_proportion
+ * self.__tests.get(category, {}).get(test_name, {}).get("max_proportion", 0.6)
+ ) / self.__df_config.shape[0]
+
+ def prepare_hash_map(self, data: Union[str, Iterable]) -> Dict[str, Any]:
+ hashmap = {index: sample for index, sample in enumerate(data)}
+
+ return hashmap
+
+ def save(self, file_path: str):
+ """
+ Save the augmented data.
+ """
+ self.__datafactory.export(data=self.__augmented_data, output_path=file_path)
+
+ def __or__(self, other: Iterable):
+ results = self.augment(other)
+ return results
+
+ def __ror__(self, other: Iterable):
+ results = self.augment(other)
+ return results
+
+ def __config_df(self):
+ """
+ Configure the data frame.
+ """
+
+ import pandas as pd
+
+ df = pd.DataFrame(columns=["category", "test_name", "proportion"])
+
+ # read the configuration
+ for category, tests in self.__tests.items():
+ if category not in ["robustness", "bias"]:
+ continue
+ for test_name, test in tests.items():
+ proportion = test.get("max_proportion", 0.6)
+ temp = pd.DataFrame(
+ {
+ "category": [category],
+ "test_name": [test_name],
+ "proportion": [proportion],
+ },
+ )
+ df = pd.concat([df, temp], ignore_index=True)
+
+ return df
diff --git a/langtest/augmentation/base.py b/langtest/augmentation/base.py
new file mode 100644
index 000000000..5b1036c4b
--- /dev/null
+++ b/langtest/augmentation/base.py
@@ -0,0 +1,616 @@
+import os
+import random
+import re
+import string
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from copy import deepcopy as copy
+from typing import Any, Dict, List, Optional, Union
+
+import pandas as pd
+import yaml
+
+from langtest.datahandler.datasource import DataFactory
+from langtest.transform import TestFactory
+from langtest.transform.utils import create_terminology
+from langtest.utils.custom_types import Sample
+from langtest.utils.custom_types.output import NEROutput
+from langtest.utils.custom_types.predictions import NERPrediction, SequenceLabel
+from langtest.utils.custom_types.sample import NERSample
+from langtest.tasks import TaskManager
+from ..utils.lib_manager import try_import_lib
+from ..errors import Errors
+
+
+class BaseAugmentaion(ABC):
+ """Abstract base class for data augmentation techniques.
+
+ Methods:
+ fix: Abstract method that should be implemented by child classes.
+ This method should perform the data augmentation operation.
+ """
+
+ @abstractmethod
+ def fix(self, *args, **kwargs):
+ """Abstract method that should be implemented by child classes.
+
+ This method should perform the data augmentation operation.
+
+ Returns:
+ NotImplementedError: Raised if the method is not implemented by child classes.
+ """
+ return NotImplementedError
+
+
+class AugmentRobustness(BaseAugmentaion):
+ """A class for performing a specified task with historical results.
+
+ Attributes:
+ task (str): A string indicating the task being performed.
+ config (dict): A dictionary containing configuration parameters for the task.
+ h_report (pandas.DataFrame): A DataFrame containing a report of historical results for the task.
+ max_prop (float): The maximum proportion of improvement that can be suggested by the class methods.
+ Defaults to 0.5.
+
+ Methods:
+ __init__(self, task, h_report, config, max_prop=0.5) -> None:
+ Initializes an instance of MyClass with the specified parameters.
+
+ fix(self) -> List[Sample]:
+ .
+
+ suggestions(self, prop) -> pandas.DataFrame:
+ Calculates suggestions for improving test performance based on a given report.
+ """
+
+ def __init__(
+ self,
+ task: TaskManager,
+ h_report: "pd.DataFrame",
+ config: Dict,
+ custom_proportions: Union[Dict, List] = None,
+ max_prop=0.5,
+ ) -> None:
+ """Initializes an instance of MyClass with the specified parameters.
+
+ Args:
+ task (str): A string indicating the task being performed.
+ h_report (pandas.DataFrame): A DataFrame containing a report of historical results for the task.
+ config (dict): A dictionary containing configuration parameters for the task.
+ custom_proportions
+ max_prop (float): The maximum proportion of improvement that can be suggested by the class methods.
+ Defaults to 0.5.
+
+ Returns:
+ None
+
+ """
+ super().__init__()
+ self.task = task
+ self.config = config
+ self.h_report = h_report
+ self.max_prop = max_prop
+ self.custom_proportions = custom_proportions
+
+ if isinstance(self.config, str):
+ with open(self.config) as fread:
+ self.config = yaml.safe_load(fread)
+
+ def fix(
+ self,
+ training_data: dict,
+ output_path: str,
+ export_mode: str = "add",
+ ):
+ """Applies perturbations to the input data based on the recommendations from harness reports.
+
+ Args:
+ training_data (dict): A dictionary containing the input data for augmentation.
+ output_path (str): The path to save the augmented data file.
+ export_mode (str, optional): Determines how the samples are modified or exported.
+ - 'inplace': Modifies the list of samples in place.
+ - 'add': Adds new samples to the input data.
+ - 'transformed': Exports only the transformed data, excluding untransformed samples.
+ Defaults to 'add'.
+
+ Returns:
+ List[Dict[str, Any]]: A list of augmented data samples.
+ """
+
+ # if "source" in training_data and training_data["source"] == "huggingface":
+ # self.df = HuggingFaceDataset(training_data, self.task)
+ # data = self.df.load_data(
+ # feature_column=training_data.get("feature_column", "text"),
+ # target_column=training_data.get("target_column", "label"),
+ # split=training_data.get("split", "test"),
+ # subset=training_data.get("subset", None),
+ # )
+ # else:
+ self.df = DataFactory(training_data, self.task)
+ data = self.df.load()
+ TestFactory.is_augment = True
+ supported_tests = TestFactory.test_scenarios()
+ suggest: pd.DataFrame = self.suggestions(self.h_report)
+ sum_propotion = suggest["proportion_increase"].sum()
+ if suggest.shape[0] <= 0 or suggest.empty:
+ print("All tests have passed. Augmentation will not be applied in this case.")
+ return None
+
+ self.config = self._parameters_overrides(self.config, data)
+
+ final_aug_data = []
+ hash_map = {k: v for k, v in enumerate(data)}
+ transformed_data = []
+ for proportion in suggest.iterrows():
+ cat = proportion[-1]["category"].lower()
+ if cat not in ["robustness", "bias"]:
+ continue
+ test = proportion[-1]["test_type"].lower()
+ test_type = {cat: {test: self.config.get("tests").get(cat).get(test)}}
+ if proportion[-1]["test_type"] in supported_tests[cat]:
+ sample_length = (
+ len(data)
+ * self.max_prop
+ * (proportion[-1]["proportion_increase"] / sum_propotion)
+ )
+ if export_mode in ("inplace"):
+ sample_indices = random.sample(
+ range(0, len(data)), int(sample_length)
+ )
+ for each in sample_indices:
+ if test == "swap_entities":
+ test_type["robustness"]["swap_entities"]["parameters"][
+ "labels"
+ ] = [self.label[each]]
+ res = TestFactory.transform(
+ self.task, [hash_map[each]], test_type
+ )
+ if len(res) == 0:
+ continue
+ hash_map[each] = res[0]
+ else:
+ if test == "swap_entities":
+ sample_data = data[: int(sample_length)]
+ test_type["robustness"]["swap_entities"]["parameters"][
+ "labels"
+ ] = test_type["robustness"]["swap_entities"]["parameters"][
+ "labels"
+ ][
+ : int(sample_length)
+ ]
+ else:
+ sample_data = random.choices(data, k=int(sample_length))
+ aug_data = TestFactory.transform(self.task, sample_data, test_type)
+ final_aug_data.extend(aug_data)
+
+ if export_mode == "transformed":
+ transformed_data.extend(aug_data)
+ if "." not in training_data["data_source"]:
+ if export_mode == "inplace":
+ final_aug_data = list(hash_map.values())
+ self.df.export(final_aug_data, output_path)
+ elif export_mode == "transformed":
+ self.df.export(transformed_data, output_path)
+ else:
+ data.extend(final_aug_data)
+ self.df.export(data, output_path)
+
+ TestFactory.is_augment = False
+ return final_aug_data
+
+ else:
+ if export_mode == "inplace":
+ final_aug_data = list(hash_map.values())
+ self.df.export(final_aug_data, output_path)
+ elif export_mode == "transformed":
+ self.df.export(transformed_data, output_path)
+ else:
+ data.extend(final_aug_data)
+ self.df.export(data, output_path)
+
+ TestFactory.is_augment = False
+ return final_aug_data
+
+ def suggestions(self, report: "pd.DataFrame") -> "pd.DataFrame":
+ """Calculates suggestions for improving test performance based on a given report.
+
+ Args:
+ report (pandas.DataFrame): A DataFrame containing test results by category and test type,
+ including pass rates and minimum pass rates.
+
+ Returns:
+ pandas.DataFrame: A DataFrame containing the following columns for each suggestion:
+ - category: the test category
+ - test_type: the type of test
+ - ratio: the pass rate divided by the minimum pass rate for the test
+ - proportion_increase: a proportion indicating how much the pass rate
+ should increase to reach the minimum pass rate
+ """
+ report["ratio"] = report["pass_rate"] / report["minimum_pass_rate"]
+
+ if self.custom_proportions and isinstance(self.custom_proportions, dict):
+ report["proportion_increase"] = report["test_type"].map(
+ self.custom_proportions
+ )
+ elif self.custom_proportions and isinstance(self.custom_proportions, list):
+ report["proportion_increase"] = report["ratio"].apply(self._proportion_values)
+ report = report[report["test_type"].isin(self.custom_proportions)]
+ else:
+ report["proportion_increase"] = report["ratio"].apply(self._proportion_values)
+
+ report = report.dropna(subset=["proportion_increase"])[
+ ["category", "test_type", "ratio", "proportion_increase"]
+ ]
+ return report
+
+ @staticmethod
+ def _proportion_values(x: float) -> Optional[float]:
+ """Calculates a proportion indicating how much a pass rate should increase to reach a minimum pass rate.
+
+ Args:
+ x (float): The ratio of the pass rate to the minimum pass rate for a given test.
+
+ Returns:
+ float: A proportion indicating how much the pass rate should increase to reach the minimum pass rate.
+ If the pass rate is greater than or equal to the minimum pass rate, returns None.
+ If the pass rate is between 0.9 and 1.0 times the minimum pass rate, returns 0.05.
+ If the pass rate is between 0.8 and 0.9 times the minimum pass rate, returns 0.1.
+ If the pass rate is between 0.7 and 0.8 times the minimum pass rate, returns 0.2.
+ If the pass rate is less than 0.7 times the minimum pass rate, returns 0.3.
+
+ """
+ if x >= 1:
+ return None
+ elif x > 0.9:
+ return 0.05
+ elif x > 0.8:
+ return 0.1
+ elif x > 0.7:
+ return 0.2
+ else:
+ return 0.3
+
+ def _parameters_overrides(self, config: dict, data_handler: List[Sample]) -> dict:
+ tests = config.get("tests", {}).get("robustness", {})
+ if "swap_entities" in config.get("tests", {}).get("robustness", {}):
+ df = pd.DataFrame(
+ {
+ "text": [sample.original for sample in data_handler],
+ "label": [
+ [i.entity for i in sample.expected_results.predictions]
+ for sample in data_handler
+ ],
+ }
+ )
+ params = tests["swap_entities"]
+ params["parameters"] = {}
+ params["parameters"]["terminology"] = create_terminology(df)
+ params["parameters"]["labels"] = df.label.tolist()
+ self.label = (
+ self.config.get("tests")
+ .get("robustness")
+ .get("swap_entities")
+ .get("parameters")
+ .get("labels")
+ )
+ return config
+
+
+class TemplaticAugment(BaseAugmentaion):
+ """This class is used for templatic augmentation. It is a subclass of the BaseAugmentation class.
+
+ Attributes:
+ __templates:
+ A string or a list of strings or samples that represents the templates for the augmentation.
+ __task:
+ The task for which the augmentation is being performed.
+ __generate_templates:
+ if set to True, generates sample templates from the given ones.
+ __show_templates:
+ if set to True, displays the used templates.
+
+
+ Methods:
+ __init__(self, templates: Union[str, List[str]], task: str):
+ Initializes the TemplaticAugment class.
+ fix(self, training_data: str, output_path: str, *args, **kwargs):
+ Performs the templatic augmentation and exports the results to a specified path.
+ """
+
+ def __init__(
+ self,
+ templates: Union[str, List[str]],
+ task: TaskManager,
+ generate_templates=False,
+ show_templates=False,
+ ) -> None:
+ """This constructor for the TemplaticAugment class.
+
+ Args:
+ templates (Union[str, List[str]]): The templates to be used for the augmentation.
+ task (str): The task for which the augmentation is being performed.
+ generate_templates (bool, optional): if set to True, generates sample templates from the given ones.
+ show_templates (bool, optional): if set to True, displays the used templates.
+ """
+ self.__templates: Union[str, List[str], List[Sample]] = templates
+ self.__task = task
+
+ if generate_templates:
+ if try_import_lib("openai"):
+ import openai
+
+ given_template = self.__templates[:]
+ for template in given_template:
+ prompt = f"""Based on the template provided, create 10 new and unique templates that are variations on this theme. Present these as a Python list, with each template as a quoted string. The list should contain only the templates without any additional text or explanation.
+
+ Template:
+ "{template}"
+
+ Expected Python List Output:
+ ['Template 1', 'Template 2', 'Template 3', ...] # Replace with actual generated templates
+ """
+
+ response = openai.Completion.create(
+ engine="gpt-3.5-turbo-instruct",
+ prompt=prompt,
+ max_tokens=500,
+ temperature=0,
+ )
+
+ generated_response = response.choices[0].text.strip()
+ # Process the generated response
+ if generated_response:
+ # Assuming the response format is a Python-like list in a string
+ templates_list = generated_response.strip("[]").split('",')
+ templates_list = [
+ template.strip().strip('"')
+ for template in templates_list
+ if template.strip()
+ ]
+
+ # Extend the existing templates list
+ self.__templates.extend(templates_list)
+ else:
+ print("No response or unexpected format.")
+
+ else:
+ raise RuntimeError(Errors.E084)
+
+ if show_templates:
+ [print(template) for template in self.__templates]
+
+ if isinstance(self.__templates, str) and os.path.exists(self.__templates):
+ self.__templates = DataFactory(self.__templates, self.__task).load()
+ elif isinstance(self.__templates, str):
+ self.__templates = [self.str_to_sample(self.__templates)]
+ elif isinstance(self.__templates, list) and isinstance(self.__templates[0], str):
+ self.__templates = [self.str_to_sample(i) for i in self.__templates]
+
+ def fix(
+ self,
+ training_data: Dict[str, Any],
+ output_path: str,
+ max_num: int = None,
+ append_original: bool = False,
+ *args,
+ **kwargs,
+ ) -> bool:
+ """This method is used to perform the templatic augmentation.
+
+ It takes the input data, performs the augmentation and then saves the augmented data to the output path.
+
+ Args:
+ training_data (dict): A dictionary containing the input data for augmentation.
+ output_path (str): The path where the augmented data will be saved.
+ max_num (int): Maximum number of new samples to generate
+ append_original (bool, optional): If set to True, appends the original data to the augmented data. Defaults to False.
+ *args: Variable length argument list.
+ **kwargs: Arbitrary keyword arguments.
+
+ Returns:
+ bool: Returns True upon successful completion of the method.
+ """
+ df = DataFactory(training_data, self.__task)
+ data = df.load()
+ new_data = (
+ data.copy()
+ if isinstance(data, (pd.DataFrame, pd.Series))
+ else copy.deepcopy(data)
+ if append_original
+ else []
+ )
+ self.__search_results = self.search_sample_results(data)
+
+ if not max_num:
+ max_num = max(len(i) for i in self.__search_results.values())
+
+ for template in self.__templates:
+ for _ in range(max_num):
+ new_sample = self.new_sample(template)
+ if new_sample:
+ new_data.append(new_sample)
+
+ df.export(new_data, output_path)
+ return True
+
+ @staticmethod
+ def search_sample_results(
+ samples: List[Sample],
+ ) -> Dict[str, List[Union[NERPrediction, SequenceLabel]]]:
+ """This method is used to search the results of the samples for the entities in the templates.
+
+ Args:
+ samples (List[Sample]): The samples for which the results are to be searched.
+
+ Returns:
+ Dict[str, List[Union[NERPrediction, SequenceLabel]]]: A dictionary containing the search results.
+ """
+ results_dict = defaultdict(list)
+ for sample in samples:
+ chunk = []
+ ent_name = ""
+ for result in sample.expected_results.predictions:
+ ent = result.entity.split("-")[-1]
+ if ent != "O" and ent_name == "":
+ ent_name = ent
+ if result.entity.endswith(ent_name) and ent != "O":
+ result.doc_id = 0
+ result.doc_name = ""
+ chunk.append(result)
+ elif len(chunk) > 0:
+ results_dict[ent_name].append(tuple(chunk))
+ ent_name = ""
+ chunk = []
+
+ if chunk:
+ results_dict[ent_name].append(tuple(chunk))
+ return results_dict
+
+ @staticmethod
+ def extract_variable_names(f_string: str) -> List[str]:
+ """This method is used to extract the variable names from the templates.
+
+ Args:
+ f_string (str): The template string.
+
+ Returns:
+ List[str]: A list of variable names.
+ """
+ pattern = r"{([^{}]*)}"
+ matches = re.findall(pattern, f_string)
+ variable_names = [match.strip() for match in matches]
+ return variable_names
+
+ def new_sample(self, template: Sample):
+ """This method is used to generate a new sample from a template.
+
+ Args:
+ template (Sample): The template from which the new sample is to be generated.
+
+ Returns:
+ Sample: The new sample generated from the template.
+ """
+ template = copy(template)
+ matches = re.finditer(r"{([^{}]*)}", template.original)
+ cursor = 0
+ other_predictions = []
+ if matches:
+ for match in matches:
+ entity = match.group(1)
+ if entity in self.__search_results:
+ prediction = random.choice(self.__search_results[entity])
+ word = " ".join(
+ i.span.word for i in prediction if isinstance(i, NERPrediction)
+ )
+
+ template.original = template.original.replace(
+ "{" + entity + "}", word, 1
+ )
+ for result in template.expected_results.predictions[cursor:]:
+ if prediction[0].entity.endswith(result.entity):
+ for each_prediction in prediction:
+ if isinstance(each_prediction, NERPrediction):
+ each_prediction.chunk_tag = "-X-"
+ each_prediction.pos_tag = "-X-"
+ other_predictions.extend(prediction)
+ cursor += 1
+ break
+ else:
+ if "{" in result.span.word and "}" in result.span.word:
+ continue
+ other_predictions.append(result)
+ cursor += 1
+ else:
+ continue
+ template.expected_results.predictions = (
+ other_predictions + template.expected_results.predictions[cursor:]
+ )
+ return template
+ else:
+ return None
+
+ def str_to_sample(self, template: str):
+ """This method is used to convert a template string to a Sample object.
+
+ Args:
+ template (str): The template string to be converted.
+
+ Returns:
+ Sample: The Sample object generated from the template string.
+ """
+ if self.__task == "ner":
+ template = self.add_spaces_around_punctuation(template)
+ sample = NERSample()
+ sample.original = template
+ words = template.split()
+ predictions = []
+ cursor = 0
+ for word in words:
+ if "{" in word and "}" in word:
+ entity = word.replace("{", "").replace("}", "")
+ else:
+ entity = "O"
+ predictions.append(
+ NERPrediction.from_span(
+ entity,
+ word,
+ cursor,
+ cursor + len(word),
+ pos_tag="-X-",
+ chunk_tag="-X-",
+ doc_id=0,
+ doc_name="",
+ )
+ )
+ cursor += len(word) + 1
+ sample.expected_results = NEROutput(predictions=predictions)
+
+ elif self.__task == "text-classification":
+ raise NotImplementedError
+
+ return sample
+
+ @property
+ def templates(self):
+ """Templates getter"""
+ return self.__templates
+
+ @templates.setter
+ def templates(self, templates: Union[str, List[str]]):
+ self.__init__(templates, self.__task)
+
+ @property
+ def task(self):
+ """Task getter"""
+ return self.__task
+
+ @task.setter
+ def task(self, task: str):
+ self.__task = task
+
+ @staticmethod
+ def add_spaces_around_punctuation(text: str):
+ """This method is used to add spaces around punctuation in a string.
+
+ Args:
+ text (str): The string to which spaces are to be added.
+
+ Returns:
+ str: The string with spaces added around punctuation.
+ """
+ for punct in string.punctuation:
+ if punct not in ["{", "}", "_"]:
+ if punct == ".":
+ # To prevent spaces being added around decimal points
+ text = re.sub(r"(\d)\.(\d)", r"\1[DOT]\2", text)
+
+ text = text.replace(punct, f" {punct} ")
+
+ if punct == ".":
+ # Putting back the decimal points to original state
+ text = text.replace("[DOT]", ".")
+
+ # Removing extra spaces
+ text = re.sub(r"\s+", " ", text).strip()
+
+ return text
diff --git a/langtest/datahandler/datasource.py b/langtest/datahandler/datasource.py
index 61ed9dd21..674af8664 100644
--- a/langtest/datahandler/datasource.py
+++ b/langtest/datahandler/datasource.py
@@ -923,9 +923,11 @@ def _import_data(self, file_name, **kwargs) -> List[Sample]:
temp_data = data.groupby("dataset_name")
samples = {}
for name, df in temp_data:
+ temp_samples = []
for i in df.to_dict(orient="records"):
sample = self.task.get_sample_class(**i)
- samples[name] = sample
+ temp_samples.append(sample)
+ samples[name] = temp_samples
return samples
for i in data.to_dict(orient="records"):
diff --git a/langtest/langtest.py b/langtest/langtest.py
index b3f5b614e..f3e1f51d6 100644
--- a/langtest/langtest.py
+++ b/langtest/langtest.py
@@ -20,11 +20,12 @@
from .transform import TestFactory
from .utils import report_utils as report, config_utils
-
from .transform.utils import RepresentationOperation
+from langtest.utils.benchmark_utils import Leaderboard, Summary
from langtest.utils.lib_manager import try_import_lib
from langtest.utils.custom_types.helpers import TestResultManager
from langtest.utils.checkpoints import divide_into_batches, CheckpointManager
+from langtest.prompts import PromptManager
from .errors import Warnings, Errors
EVAL_MODEL = None
@@ -92,6 +93,7 @@ def __init__(
model: Optional[Union[list, dict]] = None,
data: Optional[Union[list, dict]] = None,
config: Optional[Union[str, dict]] = None,
+ benchmarking: dict = None,
):
"""Initialize the Harness object.
@@ -111,6 +113,25 @@ def __init__(
self.is_default = False
self.__data_dict = data
self.__is_multi_model = False
+ self.__model_info = model
+ self.__benchmarking = benchmarking
+
+ # check the list of strings in the data
+ if isinstance(data, list) and all(isinstance(i, str) for i in data):
+ temp_data = []
+ for dataset in data:
+ if isinstance(task, dict):
+ temp_task = task["category"]
+ else:
+ temp_task = task
+ temp_data.append(
+ config_utils.BenchmarkDatasets.get_dataset_dict(
+ dataset_name=dataset, task=temp_task
+ )
+ )
+
+ data = temp_data
+ self.__data_dict = data
# reset classes to default state
self.__reset_defaults()
@@ -181,6 +202,11 @@ def __init__(
resource_filename("langtest", "data/config.yml")
)
+ # prompt config
+ self.__prompt_config = self._config.get("prompt_config", None)
+ if self.__prompt_config:
+ self.prompt_manager = PromptManager.from_prompt_configs(self.__prompt_config)
+
# model section
if isinstance(model, list):
model_dict = {}
@@ -446,6 +472,11 @@ def report(
pd.DataFrame:
DataFrame containing the results of the tests.
"""
+
+ # benchmarking true
+ if self.__benchmarking:
+ self.__tracking()
+
if self._generated_results is None:
raise RuntimeError(Errors.E011)
@@ -991,6 +1022,22 @@ def import_edited_testcases(self, input_path: str, **kwargs):
Args:
input_path (str): location of the file to load
"""
+ if isinstance(self.data, dict) and not self.__is_multi_model:
+ self._testcases = {}
+ elif isinstance(self.data, list):
+ self._testcases = []
+
+ # check the category of the testcases and count the categories
+ categories_count = list(self._config.get("tests", {}))
+
+ # Remove the "robustness" and "bias" categories from categories_count
+ categories_count = len(
+ [
+ category
+ for category in categories_count
+ if category not in ["robustness", "bias", "defaults"]
+ ]
+ )
# multi dataset case is handled separately
if isinstance(self._testcases, dict) and not self.__is_multi_model:
@@ -1003,15 +1050,31 @@ def import_edited_testcases(self, input_path: str, **kwargs):
for k, v in self._testcases.items()
}
+ if categories_count > 0:
+ testcases = self.__temp_generate()._testcases
+ for k, v in testcases.items():
+ if k in temp_testcases:
+ temp_testcases[k].extend(v)
+ else:
+ temp_testcases[k] = v
+
imported_testcases = DataFactory(
{"data_source": input_path}, task=self.task, is_import=True
).load()
+ # merge the testcases with the imported ones to the temp_testcases
for name, list_samples in imported_testcases.items():
if name not in temp_testcases:
temp_testcases[name] = list_samples
temp_testcases[name].extend(list_samples)
+ # update the testcases in the harness
+ for k, v in temp_testcases.items():
+ if k in self._testcases:
+ self._testcases[k].extend(v)
+ else:
+ self._testcases[k] = v
+
# single dataset case
elif isinstance(self._testcases, list):
temp_testcases = [
@@ -1020,6 +1083,10 @@ def import_edited_testcases(self, input_path: str, **kwargs):
if sample.category not in ["robustness", "bias"]
]
+ if categories_count > 0:
+ testcases = self.__temp_generate()._testcases
+ temp_testcases.extend(testcases)
+
self._testcases = DataFactory(
{"data_source": input_path}, task=self.task, is_import=True
).load()
@@ -1342,7 +1409,11 @@ def __single_dataset_generate(self, dataset: list):
return testcases
elif str(self.task) in ("question-answering", "summarization"):
- if "bias" in tests.keys() and "bias" == self.__data_dict.get("split"):
+ if (
+ "bias" in tests.keys()
+ and isinstance(self.__data_dict, dict)
+ and "bias" == self.__data_dict.get("split")
+ ):
if self.__data_dict["data_source"] in ("BoolQ", "XSum"):
tests_to_filter = tests["bias"].keys()
testcases = DataFactory.filter_curated_bias(tests_to_filter, dataset)
@@ -1572,8 +1643,21 @@ def __multi_datasets_run(
):
generated_results = {}
+ # temp_store_prompt
+ temp_store_prompt = self._config.get("model_parameters", {}).get(
+ "user_prompt", None
+ )
+
# Run the testcases for each dataset
for dataset_name, samples in testcases.items():
+ # set prompt in prompt manager
+ if hasattr(self, "prompt_manager") and self.prompt_manager is not None:
+ self.prompt_manager.default_state = dataset_name
+ # update user prompt for each dataset
+ if temp_store_prompt and isinstance(temp_store_prompt, dict):
+ self._config.get("model_parameters", {}).update(
+ {"user_prompt": temp_store_prompt.get(dataset_name)}
+ )
# Get the raw data for the dataset
if isinstance(self.data, dict):
raw_data = self.data.get(dataset_name)
@@ -1597,6 +1681,12 @@ def __multi_datasets_run(
print(f"{'':-^80}\n")
+ # resore user prompt
+ if temp_store_prompt:
+ self._config.get("model_parameters", {}).update(
+ {"user_prompt": temp_store_prompt}
+ )
+
if (
self.is_multi_dataset
and self._generated_results is None
@@ -1611,3 +1701,113 @@ def __reset_defaults(self):
"""Reset the default values."""
model_response = TestResultManager()
model_response.clear_data()
+
+ # Reset the PromptManager
+ prompt_manager = PromptManager()
+ prompt_manager.reset()
+
+ def __tracking(self, *args, **kwargs):
+ """Track the progress of the testcases."""
+ if self.__benchmarking:
+ df = self.generated_results()
+
+ path = self.__benchmarking.get(
+ os.path.expanduser("save_dir"),
+ os.path.expanduser("~/.langtest/leaderboard/"),
+ )
+ summary = Summary(path)
+
+ # temp dict
+ temp_dict = {}
+ if isinstance(self.__data_dict, dict):
+ temp_dict[self.__data_dict.get("data_source")] = self.__data_dict
+ else:
+ for i in self.__data_dict:
+ temp_dict[i.get("data_source")] = i
+
+ # add the dataset_name column if the data is single dataset
+ if isinstance(self.__data_dict, dict) and (not self.is_multi_dataset):
+ df["dataset_name"] = self.__data_dict.get("data_source", "-")
+
+ df["split"] = df["dataset_name"].apply(
+ lambda x: temp_dict[x].get("split", "-")
+ )
+ df["subset"] = df["dataset_name"].apply(
+ lambda x: temp_dict[x].get("subset", "-")
+ )
+
+ df["hub"] = self.__model_info.get("hub", "-")
+ if self.__model_info.get("hub", "-") == "lm-studio":
+ import requests as req
+
+ response = req.get(
+ "http://localhost:1234/v1/models",
+ ).json()
+
+ model_name = response["data"][0]["id"]
+ df["model"] = model_name
+ else:
+ df["model"] = self.__model_info.get("model", "-")
+ df["task"] = str(self.task)
+ summary.add_report(df)
+
+ def get_leaderboard(
+ self,
+ indices=[],
+ columns=[],
+ category=False,
+ split_wise=False,
+ test_wise=False,
+ rank_by: Union[str, list] = "Avg",
+ *args,
+ **kwargs,
+ ):
+ """Get the rank of the model on the leaderboard."""
+
+ if os.path.exists(os.path.expanduser(self.__benchmarking.get("save_dir"))):
+ path = os.path.expanduser(self.__benchmarking.get("save_dir"))
+ elif os.path.exists(os.path.expanduser("~/.langtest/leaderboard/")):
+ path = os.path.expanduser("./.langtest/leaderboard/")
+ else:
+ raise FileNotFoundError(f"Summary.csv File is not exists in {path}")
+
+ leaderboard = Leaderboard(path)
+
+ # print(leaderboard.default().to_markdown())
+ if indices or columns:
+ return leaderboard.custom_wise(indices, columns)
+ if category:
+ return leaderboard.category_wise(rank_by=rank_by)
+
+ if test_wise:
+ return leaderboard.test_wise(rank_by=rank_by)
+
+ if split_wise:
+ return leaderboard.split_wise(rank_by=rank_by)
+
+ return leaderboard.default(rank_by=rank_by)
+
+ def __temp_generate(self, *args, **kwargs):
+ """Temporary function to generate the testcases."""
+
+ # temp config other than robustness and bias
+ temp_config = {
+ "tests": {
+ k: v
+ for k, v in self._config.get("tests", {}).items()
+ if k not in ["robustness", "bias"]
+ }
+ }
+ print(
+ f"{'':=^80}\n{'Adding Test Cases Other than Robustness and Bias Categories ':^80}\n{'':=^80}"
+ )
+ temp_harness = self.__class__(
+ task=str(self.task),
+ model=self.__model_info,
+ data=self.__data_dict,
+ config=temp_config,
+ )
+ temp_harness.generate()
+ print(f"{'':-^80}\n")
+
+ return temp_harness
diff --git a/langtest/modelhandler/llm_modelhandler.py b/langtest/modelhandler/llm_modelhandler.py
index 4a9081f33..884be8797 100644
--- a/langtest/modelhandler/llm_modelhandler.py
+++ b/langtest/modelhandler/llm_modelhandler.py
@@ -1,9 +1,13 @@
import inspect
-from typing import Any, Union
+from typing import Any, List, Union
import langchain.llms as lc
-import langchain.chat_models as cm
-from langchain import LLMChain, PromptTemplate
-from pydantic import ValidationError
+from langchain.chains import LLMChain
+from langchain_core.prompts import PromptTemplate
+from langchain_core.exceptions import OutputParserException
+from pydantic import Field, ValidationError
+
+from langtest.utils.custom_types.output import NEROutput
+from langtest.utils.custom_types.predictions import NERPrediction
from ..modelhandler.modelhandler import ModelAPI, LANGCHAIN_HUBS
from ..errors import Errors, Warnings
import logging
@@ -76,7 +80,9 @@ def load_model(cls, hub: str, path: str, *args, **kwargs) -> "PretrainedModelFor
try:
cls._update_model_parameters(hub, filtered_kwargs)
if path in ("gpt-4", "gpt-3.5-turbo", "gpt-4-1106-preview"):
- model = cm.ChatOpenAI(model=path, *args, **filtered_kwargs)
+ from langchain_openai.chat_models import ChatOpenAI
+
+ model = ChatOpenAI(model=path, *args, **filtered_kwargs)
return cls(hub, model, *args, **filtered_kwargs)
else:
model = getattr(lc, LANGCHAIN_HUBS[hub])
@@ -131,10 +137,19 @@ def predict(self, text: Union[str, dict], prompt: dict, *args, **kwargs):
The prediction result.
"""
try:
- prompt_template = PromptTemplate(**prompt)
+ # loading a prompt manager
+ from langtest.prompts import PromptManager
+
+ prompt_manager = PromptManager()
+
+ prompt_template = prompt_manager.get_prompt()
+
+ if prompt_template is None:
+ prompt_template = PromptTemplate(**prompt)
+
llmchain = LLMChain(prompt=prompt_template, llm=self.model)
- output = llmchain.run(**text)
- return output
+ output = llmchain.invoke(text)
+ return output.get(llmchain.output_key, "")
except Exception as e:
raise ValueError(Errors.E089.format(error_message=e))
@@ -197,6 +212,93 @@ def __str__(self):
return self.message
+class PretrainedModelForNER(PretrainedModelForQA, ModelAPI):
+ """A class representing a pretrained model for named entity recognition.
+
+ Inherits:
+ PretrainedModelForQA: The base class for pretrained models.
+ """
+
+ @lru_cache(maxsize=102400)
+ def predict(self, text: Union[str, dict], *args, **kwargs) -> NEROutput:
+ """Perform prediction using the pretrained model.
+
+ Args:
+ text (Union[str, dict]): The input text or dictionary.
+ *args: Additional positional arguments.
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ dict: A dictionary containing the prediction result.
+ - 'result': The prediction result.
+ """
+ try:
+ prompt = {
+ "input_variables": ["text"],
+ "template": "Extract the named entities from the text. \n{format_instructions}\n {text}",
+ "partial_variables": {
+ "format_instructions": self.__output_parser().get_format_instructions()
+ },
+ }
+ prompt_template = PromptTemplate(**prompt)
+ llmchain = LLMChain(
+ prompt=prompt_template,
+ llm=self.model,
+ output_parser=self.__output_parser(),
+ )
+ result = llmchain.invoke({"text": text})
+ result: dict = result.get(llmchain.output_key, {"entities": []})
+
+ try:
+ predictions = []
+ for entity in result.get("entities", []):
+ try:
+ entity = NERPrediction.from_span(**entity)
+ predictions.append(entity)
+ except Exception:
+ pass
+
+ return NEROutput(predictions=predictions)
+ except Exception:
+ return NEROutput(predictions=[])
+
+ except OutputParserException:
+ return NEROutput(predictions=[])
+
+ except Exception as e:
+ raise ValueError(Errors.E089.format(error_message=e))
+
+ def __call__(self, text: str, *args, **kwargs):
+ return self.predict(text, *args, **kwargs)
+
+ def __output_parser(self):
+ from langchain_core.output_parsers import JsonOutputParser
+ from pydantic import BaseModel
+
+ class Word(BaseModel):
+ """Single word in a named entity recognition prediction"""
+
+ word: str = Field(description="Word in the text")
+ start: int = Field(
+ description="Start index of the character in the word from the text"
+ )
+ end: int = Field(
+ description="End index of the character in the word from the text"
+ )
+ entity: str = Field(description="Named entity type")
+ score: float = Field(description="Confidence score of the prediction")
+ pos_tag: str = Field(description="Part of speech tag")
+ chunk_tag: str = Field(description="Chunk tag")
+
+ class NERParser(BaseModel):
+ """Named entity recognition prediction parser"""
+
+ entities: List[Word] = Field(description="List of named entities in the text")
+
+ parser = JsonOutputParser(pydantic_object=NERParser)
+ return parser
+
+
class PretrainedModelForSummarization(PretrainedModelForQA, ModelAPI):
"""A class representing a pretrained model for summarization.
diff --git a/langtest/modelhandler/lmstudio_modelhandler.py b/langtest/modelhandler/lmstudio_modelhandler.py
index 33b652477..fbcc59f4f 100644
--- a/langtest/modelhandler/lmstudio_modelhandler.py
+++ b/langtest/modelhandler/lmstudio_modelhandler.py
@@ -1,4 +1,7 @@
from typing import Any, Callable, Union
+
+from langtest.prompts import PromptManager
+
from .modelhandler import ModelAPI
from abc import ABC
from functools import lru_cache
@@ -6,6 +9,8 @@
from ..errors import Errors
from langtest.utils.lib_manager import try_import_lib
from ..utils.custom_types.helpers import SimplePromptTemplate
+from langtest.utils.custom_types.output import NEROutput
+from langtest.utils.custom_types.predictions import NERPrediction
from langtest.utils.custom_types.helpers import HashableDict
@@ -42,10 +47,20 @@ def chat_completion_api(text: str, url: str, server_prompt: str, **kwargs):
input_data_func = kwargs.get("data")
data = input_data_func(text)
else:
- server_prompt = {"role": "assistant", "content": server_prompt}
+ examples = []
user_text = {"role": "user", "content": text}
+
+ if kwargs.get("examples", None) and isinstance(kwargs.get("examples"), list):
+ examples.extend(kwargs.get("examples", []))
+
+ if isinstance(server_prompt, str):
+ server_prompt = {"role": "system", "content": server_prompt}
+ # user_text = {"role": "user", "content": text}
+ messages = [*examples, user_text]
+ # if server_prompt:
+ # messages.insert(0, server_prompt)
data = {
- "messages": [server_prompt, user_text],
+ "messages": messages,
"temperature": kwargs.get("temperature", 0.2),
"max_tokens": kwargs.get("max_tokens", -1),
"stream": kwargs.get("stream", False),
@@ -146,12 +161,21 @@ def predict(
str: The predicted output.
"""
try:
- prompt_template = SimplePromptTemplate(**prompt)
- p = prompt_template.format(**text)
+ prompt_examples = PromptManager()
+ examples = prompt_examples.get_prompt(hub="lm-studio")
+
+ if examples:
+ prompt["template"] = "".join(f"{k.title()}: {{{k}}}" for k in text.keys())
+ prompt_template = SimplePromptTemplate(**prompt)
+ p = prompt_template.format(**text)
+ else:
+ prompt_template = SimplePromptTemplate(**prompt)
+ p = prompt_template.format(**text)
op = chat_completion_api(
text=p,
url=self.model,
server_prompt=server_prompt,
+ examples=examples,
*args,
**self.kwargs,
)
@@ -206,6 +230,96 @@ def __call__(
)
+class PretrainedModelForNER(PretrainedModel, ModelAPI):
+ """A class representing a pretrained model for named entity recognition.
+
+ Inherits:
+ PretrainedModel: The base class for pretrained models.
+
+ Methods
+ -------
+ predict(text: str, *args, **kwargs)
+ Predicts the answer to a given question based on the pre-trained model.
+
+ Raises
+ ------
+ Exception
+ If an error occurs during prediction.
+ """
+
+ @lru_cache(maxsize=102400)
+ def predict(self, text: str, *args, **kwargs) -> NEROutput:
+ """
+ Perform prediction using the pretrained model."""
+
+ # prompt configuration
+ prompt = HashableDict(
+ {
+ "input_variables": ["text"],
+ "template": "Given the text: {text}, identify the named entities.",
+ }
+ )
+
+ # server prompt configuration
+ server_prompt = self.kwargs.get(
+ "system_prompt", "Please identify the named entities in the given text."
+ )
+
+ # check if server prompt is a list and convert it to a tuple
+ if isinstance(server_prompt, list):
+ server_prompt = tuple(HashableDict(item) for item in server_prompt)
+
+ # convert text to a dictionary
+ text = HashableDict({"text": text})
+
+ # result
+ predictions = None
+
+ try:
+ prompt_template = SimplePromptTemplate(**prompt)
+ p = prompt_template.format(**text)
+ op = chat_completion_api(
+ text=p,
+ url=self.model,
+ server_prompt=server_prompt,
+ *args,
+ **self.kwargs,
+ )
+ if self.output_parser:
+ predictions = self.output_parser(op)
+ else:
+ predictions = op["choices"][0]["message"]["content"]
+ except Exception as e:
+ raise ValueError(Errors.E089.format(error_message=e))
+
+ if isinstance(predictions, str):
+ try:
+ predictions = eval(predictions)
+ if not isinstance(predictions, list):
+ predictions = []
+ except SyntaxError:
+ predictions = []
+
+ return NEROutput(
+ predictions=[
+ NERPrediction.from_span(
+ entity=prediction.get("entity_group", prediction.get("entity", None)),
+ word=prediction.get("word", ""),
+ start=prediction.get("start", -1),
+ end=prediction.get("end", -1),
+ )
+ for prediction in predictions
+ ]
+ )
+
+ def __call__(self, text: str, *args, **kwargs) -> NEROutput:
+ """
+ Calls the predict method for the given input text.
+ """
+
+ return self.predict(text, *args, **kwargs)
+
+
class PretrainedModelForQA(PretrainedModel, ModelAPI):
"""
A class for handling a pre-trained model for question answering.
diff --git a/langtest/modelhandler/transformers_modelhandler.py b/langtest/modelhandler/transformers_modelhandler.py
index eff7f77f8..5ab16b5c5 100644
--- a/langtest/modelhandler/transformers_modelhandler.py
+++ b/langtest/modelhandler/transformers_modelhandler.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
import logging
import numpy as np
from functools import lru_cache
@@ -11,7 +11,11 @@
TranslationOutput,
)
from ..errors import Errors, Warnings
-from ..utils.custom_types.helpers import SimplePromptTemplate, HashableDict
+from ..utils.custom_types.helpers import (
+ SimplePromptTemplate,
+ HashableDict,
+ default_llm_chat_prompt,
+)
from ..utils.hf_utils import HuggingFacePipeline
@@ -22,7 +26,7 @@ class PretrainedModelForNER(ModelAPI):
model (transformers.pipeline.Pipeline): Pretrained HuggingFace NER pipeline for predictions.
"""
- def __init__(self, model):
+ def __init__(self, model, **kwargs):
"""Constructor method
Args:
@@ -33,6 +37,7 @@ def __init__(self, model):
)
self.model = model
+ self.kwargs = kwargs
self.predict.cache_clear()
@staticmethod
@@ -136,7 +141,7 @@ def group_entities(self, entities: List[Dict]) -> List[Dict]:
return entity_groups
@classmethod
- def load_model(cls, path: str) -> "Pipeline":
+ def load_model(cls, path: Union[str, Any], **kwargs) -> "Pipeline":
"""Load the NER model into the `model` attribute.
Args:
@@ -146,9 +151,27 @@ def load_model(cls, path: str) -> "Pipeline":
Returns:
'Pipeline':
"""
- if isinstance(path, str):
- return cls(pipeline(model=path, task="ner", ignore_labels=[]))
- return cls(path)
+ # cls.kwargs = kwargs
+ task = kwargs.get("task", "ner")
+ device = kwargs.get("device", -1)
+
+ if isinstance(path, str) and task == "ner":
+ return cls(
+ pipeline(model=path, task=task, device=device, ignore_labels=[]), **kwargs
+ )
+ elif isinstance(path, str):
+ import torch
+
+ return cls(
+ pipeline(
+ model=path,
+ task="text-generation",
+ device=device,
+ torch_dtype=torch.bfloat16,
+ ),
+ **kwargs,
+ )
+ return cls(path, **kwargs)
@lru_cache(maxsize=102400)
def predict(self, text: str, **kwargs) -> NEROutput:
@@ -164,17 +187,37 @@ def predict(self, text: str, **kwargs) -> NEROutput:
Returns:
NEROutput: A list of named entities recognized in the input text.
"""
- predictions = self.model(text, **kwargs)
- aggregated_words = self._aggregate_words(predictions)
- aggregated_predictions = self.group_entities(aggregated_words)
+
+ if self.model.model.__class__.__name__.endswith("ForCausalLM"):
+ predictions, _ = self.__predict_causalLm(text, **kwargs)
+ if predictions:
+ if not isinstance(predictions, list):
+ predictions = [predictions]
+ else:
+ predictions = [
+ {
+ "entity": "",
+ "score": -1,
+ "index": -1,
+ "word": "",
+ "start": -1,
+ "end": -1,
+ }
+ ]
+
+ aggregated_predictions = predictions
+ else:
+ predictions = self.model(text, **kwargs)
+ aggregated_words = self._aggregate_words(predictions)
+ aggregated_predictions = self.group_entities(aggregated_words)
return NEROutput(
predictions=[
NERPrediction.from_span(
entity=prediction.get("entity_group", prediction.get("entity", None)),
- word=prediction["word"],
- start=prediction["start"],
- end=prediction["end"],
+ word=prediction.get("word", ""),
+ start=prediction.get("start", -1),
+ end=prediction.get("end", -1),
)
for prediction in aggregated_predictions
]
@@ -203,6 +246,81 @@ def __call__(self, text: str, *args, **kwargs) -> NEROutput:
"""Alias of the 'predict' method"""
return self.predict(text=text, **kwargs)
+ def __predict_causalLm(self, text: str, **kwargs):
+ """Perform predictions on the input text using a causal language model."""
+
+ # override default tokenizer parameters with the user-defined parameters
+ tokenizer_params = {
+ "tokenize": False,
+ "add_generation_prompt": True,
+ "return_dict": True,
+ **self.kwargs.get("tokenizer_kwargs", {}),
+ }
+
+ # override default model parameters with the user-defined parameters
+ model_params = {
+ "max_new_tokens": 512,
+ "do_sample": True,
+ "temperature": 0.2,
+ "top_k": 10,
+ "top_p": 0.95,
+ "batch_size": 1,
+ **self.kwargs.get("model_kwargs", {}),
+ }
+
+ # override the default parameters for the tokenizer
+ tokenizer_params = {
+ **tokenizer_params,
+ "add_generation_prompt": True,
+ }
+ # override the default parameters for the model
+ model_params = {
+ **model_params,
+ "return_full_text": False,
+ "handle_long_generation": "hole",
+ }
+
+ # process the input text
+ messages = self.input_process(text)
+
+ # preprocess the messages and generate the prompt for the model
+ prompt = self.model.tokenizer.apply_chat_template(messages, **tokenizer_params)
+
+ # generate the response from the model
+ outputs = self.model(
+ prompt,
+ **model_params,
+ )
+
+ # Extracting the structured responses from the generated text
+ predictions = []
+ if isinstance(outputs[0]["generated_text"], str):
+ try:
+ predictions = eval(outputs[0]["generated_text"])
+ if not isinstance(predictions, list):
+ predictions = []
+ except SyntaxError:
+ predictions = []
+
+ return predictions, outputs
+
+ def input_process(self, input_sen):
+ """
+ Process the input text to be used in the model
+ """
+
+ # extract the system prompt from the user-defined parameters
+ system_prompt = self.kwargs.get("system_prompt", default_llm_chat_prompt)
+
+ if system_prompt:
+ if isinstance(system_prompt, str):
+ return f"{system_prompt}\n\n {input_sen}"
+ elif isinstance(system_prompt, list):
+ input_sen = {"role": "user", "content": input_sen}
+ return [*system_prompt, input_sen]
+
+ return default_llm_chat_prompt
+
class PretrainedModelForTextClassification(ModelAPI):
"""Transformers pretrained model for text classification tasks
diff --git a/langtest/prompts.py b/langtest/prompts.py
new file mode 100644
index 000000000..b9df1d28e
--- /dev/null
+++ b/langtest/prompts.py
@@ -0,0 +1,262 @@
+from collections import defaultdict
+from typing import Dict, List, Union
+
+from pydantic import BaseModel, Extra, validator
+
+
+class MessageType(BaseModel):
+ __field_order: List[str] = [
+ "content",
+ "context",
+ "question",
+ "original",
+ "testcase",
+ "options",
+ "answer",
+ ]
+
+ class Config:
+ extra = (
+ Extra.allow
+ ) # Allow any additional fields that are not explicitly declared
+
+ @validator("*", pre=True, allow_reuse=True)
+ def add_field(cls, v, values, field, **kwargs):
+ if "fields" not in values:
+ values["fields"] = []
+ values["fields"].append(field)
+ return v
+
+ @property
+ def get_template(self):
+ """Generate a template string based on the dynamic fields of the instance."""
+
+ temp = []
+ order_less = []
+ for field in self.__dict__:
+ if field in self.__field_order:
+ temp.append(f"{field.title()}: {{{field}}}")
+ else:
+ order_less.append(f"{field.title()}: {{{field}}}")
+ if order_less:
+ temp.extend(order_less)
+ return "\n" + "\n".join(temp)
+
+ @property
+ def get_example(self):
+ """Generate an example string based on the dynamic fields of the instance."""
+ # return {k: v for k, v in self.__dict__.items() if k != 'fields'}
+ temp = {}
+ for field in self.__field_order:
+ if field in self.__dict__:
+ temp[field] = self.__dict__[field]
+ return temp
+
+ @property
+ def input_variables(self):
+ temp = []
+ for field in self.__field_order:
+ if field in self.__dict__:
+ temp.append(field)
+ return temp
+
+
+class Conversion(BaseModel):
+ """Conversion model for the conversion of the input and output of the model."""
+
+ user: MessageType
+ ai: MessageType
+
+ class Config:
+ extra = (
+ Extra.allow
+ ) # Allow any additional fields that are not explicitly declared
+
+ @validator("*", pre=True, allow_reuse=True)
+ def add_field(cls, v, values, field, **kwargs):
+ if "fields" not in values:
+ values["fields"] = []
+ values["fields"].append(field)
+ return v
+
+ @property
+ def get_examples(self):
+ """Generate a list of examples based on the dynamic fields of the instance."""
+ return {**self.user.get_example, **self.ai.get_example}
+
+ @property
+ def get_suffix_user(self):
+ if self.user.get_template:
+ return self.user.get_template
+
+
+class PromptConfig(BaseModel):
+ instructions: str
+ prompt_type: str
+ examples: Union[Conversion, List[Conversion]] = None
+
+ @property
+ def get_examples(self) -> List[dict]:
+ """Generate a list of examples based on the dynamic fields of the instance."""
+ if isinstance(self.examples, Conversion):
+ return [self.examples.get_examples]
+ elif isinstance(self.examples, list):
+ return [example.get_examples for example in self.examples]
+
+ @property
+ def get_template(self):
+ """Generate a template string based on the dynamic fields of the instance."""
+ if isinstance(self.examples, Conversion):
+ return self.examples.user.get_template
+ elif isinstance(self.examples, list):
+ return [
+ ("human", self.examples[0].user.get_template),
+ ("ai", self.examples[0].ai.get_template),
+ ]
+ # return self.examples.get_template
+
+ @property
+ def get_input_variables(self):
+ """Generate a list of input variables based on the dynamic fields of the instance."""
+ if isinstance(self.examples, Conversion):
+ return self.examples.user.input_variables
+ elif isinstance(self.examples, list):
+ return self.examples[0].user.input_variables
+
+ def prompt_style(self):
+ """Generate a prompt based on the prompt type."""
+ if self.prompt_type == "chat":
+ from langchain.prompts import (
+ ChatPromptTemplate,
+ FewShotChatMessagePromptTemplate,
+ )
+
+ example_prompt = ChatPromptTemplate.from_messages(self.get_template)
+
+ few_shot_prompt = FewShotChatMessagePromptTemplate(
+ examples=self.get_examples,
+ example_prompt=example_prompt,
+ )
+
+ final_prompt = ChatPromptTemplate.from_messages(
+ [
+ ("system", self.instructions),
+ few_shot_prompt,
+ # ('human', conf),
+ self.get_template[0],
+ ]
+ )
+ return final_prompt
+
+ elif self.prompt_type == "instruct":
+ from langchain.prompts import FewShotPromptTemplate, PromptTemplate
+
+ template = "".join(v for _, v in self.get_template)
+ template = f"{template.replace('Answer:', '')}"
+ examples = [v.get_examples for v in self.examples]
+ suffix = self.examples[0].get_suffix_user
+
+ example = PromptTemplate.from_template(template)
+
+ final_prompt = FewShotPromptTemplate(
+ examples=examples,
+ example_prompt=example,
+ input_variables=self.get_input_variables,
+ suffix=suffix,
+ prefix=self.instructions,
+ )
+
+ return final_prompt
+
+ def get_prompt(self, hub=None):
+ if hub == "lm-studio":
+ return self.lm_studio_prompt()
+ return self.prompt_style()
+
+ def get_shot_prompt(self):
+ print(self.get_examples)
+ return f"{len(self.get_examples)}-shot prompt"
+
+ def lm_studio_prompt(self):
+ messages = [
+ {"role": "system", "content": self.instructions},
+ ]
+
+ for example in self.examples:
+ temp_user = {}
+ temp_ai = {}
+
+ # user role
+ temp_user["role"] = "user"
+ temp_user["content"] = example.user.get_template.format(
+ **example.user.get_example
+ )
+
+ # assistant role
+ temp_ai["role"] = "assistant"
+ temp_ai["content"] = example.ai.get_template.format(**example.ai.get_example)
+
+ messages.append(temp_user)
+ messages.append(temp_ai)
+ # return messages
+ return messages
+
+
+class PromptManager:
+ _instance = None
+ prompt_configs: Dict[str, PromptConfig] = defaultdict(PromptConfig)
+ _default_state = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance.prompt_configs = defaultdict(PromptConfig)
+ return cls._instance
+
+ @classmethod
+ def from_prompt_configs(cls, prompt_configs: dict):
+ """Create a prompt manager from a dictionary of prompt configurations."""
+ prompt_manager = cls()
+ if set(["instructions", "prompt_type", "examples"]).issubset(
+ set(prompt_configs.keys())
+ ):
+ prompt_manager.add_prompt("default", prompt_configs)
+ return prompt_manager
+ for name, prompt_config in prompt_configs.items():
+ prompt_manager.add_prompt(name, prompt_config)
+
+ if len(prompt_manager.prompt_configs) == 1:
+ prompt_manager.default_state = list(prompt_manager.prompt_configs.keys())[0]
+ return prompt_manager
+
+ def add_prompt(self, name: str, prompt_config: dict):
+ """Add a prompt template to the prompt manager."""
+ prompt_config_o = PromptConfig(**prompt_config)
+ self.prompt_configs[name] = prompt_config_o
+
+ def get_prompt(self, name: str = None, hub: str = None):
+ """Get a prompt template based on the name."""
+ if name is None and self.default_state is None:
+ return None
+ if name is None:
+ name = self.default_state
+ prompt_template = self.prompt_configs[name].get_prompt(hub)
+ return prompt_template
+
+ @property
+ def default_state(self):
+ return self._default_state
+
+ @default_state.setter
+ def default_state(self, name: str):
+ self._default_state = name
+
+ @property
+ def get_prompt_shot(self):
+ return self.get_prompt().get_shot_prompt()
+
+ def reset(self):
+ """Reset the prompt manager to its initial state."""
+ self.prompt_configs = defaultdict(PromptConfig)
+ self._instance = None
+ return self
diff --git a/langtest/transform/robustness.py b/langtest/transform/robustness.py
index 990ece8ca..7c6671ecb 100644
--- a/langtest/transform/robustness.py
+++ b/langtest/transform/robustness.py
@@ -1680,12 +1680,12 @@ def randomize_ages(text):
for i in range(count):
if isinstance(sample, str):
s, _ = randomize_ages(sample)
- perturbed_samples.append(s)
else:
s = deepcopy(sample)
s.test_case, transformations = randomize_ages(s.original)
if s.task in ("ner", "text-classification"):
s.transformations = transformations
s.category = "robustness"
+ perturbed_samples.append(s)
return perturbed_samples
diff --git a/langtest/utils/benchmark_utils.py b/langtest/utils/benchmark_utils.py
new file mode 100644
index 000000000..f3e08f839
--- /dev/null
+++ b/langtest/utils/benchmark_utils.py
@@ -0,0 +1,376 @@
+import os
+from typing import TypeVar, Generic, Union
+import pandas as pd
+
+
+class Leaderboard(Generic[TypeVar("T", bound="Leaderboard")]):
+
+ """
+ Leaderboard class to manage the ranking of the models
+
+ Args:
+ path (str): The path to the summary file
+
+
+ """
+
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ """
+ Singleton pattern to ensure only one instance of the class is created
+ """
+ if not cls._instance:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(
+ self,
+ path: str = os.path.expanduser("~/.langtest/leaderboard/summary.csv"),
+ *args,
+ **kwargs,
+ ) -> None:
+ """
+ Initialize the Leaderboard class with the summary file
+ """
+ self.summary = Summary(path, *args, **kwargs)
+
+ def default(self, rank_by: Union[str, list] = "Avg"):
+ """
+ Get the score board for the models
+ """
+ # check if the rank_by is a string
+ if isinstance(rank_by, str):
+ rank_by = [rank_by]
+
+ ascending = [False] * len(rank_by)
+
+ df = self.summary.summary_df
+ df = self.__drop_duplicates(df)
+ pvt_table = df.pivot_table(
+ index=["model"], columns="dataset_name", values="score"
+ )
+
+ # mean column
+ pvt_table.insert(0, "Avg", pvt_table.mean(axis=1))
+ pvt_table = pvt_table.sort_values(by=rank_by, ascending=ascending)
+
+ # reset the index and fill the NaN values
+ pvt_table = pvt_table.rename_axis(None, axis=1).reset_index()
+ pvt_table = pvt_table.fillna("-")
+
+ return pvt_table
+
+ def split_wise(self, rank_by: Union[str, list] = "Avg"):
+ """
+ Get the score board for the models by test type
+ """
+
+ # check if the rank_by is a string
+ if isinstance(rank_by, str):
+ rank_by = [rank_by]
+
+ ascending = [False] * len(rank_by)
+
+ df = self.summary.summary_df
+ df = self.__drop_duplicates(df)
+ pvt_table = df.pivot_table(
+ index=["model", "split"],
+ columns=["dataset_name"],
+ values="score",
+ )
+
+ # mean column
+ pvt_table.insert(0, "Avg", pvt_table.mean(axis=1))
+ pvt_table = pvt_table.sort_values(by=rank_by, ascending=ascending)
+
+ pvt_table = pvt_table.fillna("-")
+
+ return pvt_table
+
+ def test_wise(self, rank_by: Union[str, list] = "Avg"):
+ """
+ Get the score board for the models by test type
+ """
+
+ # check if the rank_by is a string
+ if isinstance(rank_by, str):
+ rank_by = [rank_by]
+
+ # check if the test_type in the rank_by
+ if "test_type" not in rank_by:
+ rank_by.insert(0, "test_type")
+ rank_by.insert(0, "category")
+
+ ascending = [True, True] + [False] * (len(rank_by) - 2)
+
+ df = self.summary.summary_df
+ df = self.__drop_duplicates(df)
+ pvt_table = df.pivot_table(
+ index=["category", "test_type", "model"],
+ columns=["dataset_name"],
+ values="score",
+ )
+
+ # mean column
+ pvt_table.insert(0, "Avg", pvt_table.mean(axis=1))
+ pvt_table = pvt_table.sort_values(by=rank_by, ascending=ascending)
+
+ pvt_table = pvt_table.fillna("-")
+
+ return pvt_table
+
+ def category_wise(self, rank_by: Union[str, list] = "Avg"):
+ """
+ Get the score board for the models by category
+ """
+ # check if the rank_by is a string
+ if isinstance(rank_by, str):
+ rank_by = [rank_by]
+
+ ascending = [False] * len(rank_by)
+
+ df = self.summary.summary_df
+ df = self.__drop_duplicates(df)
+ pvt_table = df.pivot_table(
+ index=["category", "model"], columns=["dataset_name"], values="score"
+ )
+ pvt_table.insert(0, "Avg", pvt_table.mean(axis=1))
+ pvt_table = pvt_table.sort_values(by=rank_by, ascending=ascending)
+ pvt_table = pvt_table.fillna("-")
+
+ return pvt_table
+
+ def custom_wise(
+ self, indices: list, columns: list = [], rank_by: Union[str, list] = "Avg"
+ ):
+ """
+ Get the score board for the models by custom group
+ """
+ # check if the rank_by is a string
+ if isinstance(rank_by, str):
+ rank_by = [rank_by]
+
+ ascending = [False] * len(rank_by)
+
+ df = self.summary.summary_df
+ df = self.__drop_duplicates(df)
+ pvt_table = df.pivot_table(
+ index=["model", *indices],
+ columns=["dataset_name", *columns],
+ values="score",
+ aggfunc="first",
+ )
+ pvt_table.insert(0, "Avg", pvt_table.mean(axis=1))
+ pvt_table = pvt_table.fillna("-")
+ pvt_table = pvt_table.sort_values(by=rank_by, ascending=ascending)
+
+ return pvt_table
+
+ def __drop_duplicates(self, df: pd.DataFrame):
+ """
+ Drop duplicates from the dataframe
+ """
+ # arrange the dataframe by timestamp in descending order
+ df["timestamp"] = pd.to_datetime(df["timestamp"])
+ df = df.sort_values(by="timestamp", ascending=False)
+
+ # remove duplicates
+ df["timestamp"] = pd.to_datetime(df["timestamp"], format="%Y-%m-%d-%H-%M-%S")
+ df = df.sort_values(by="timestamp", ascending=False)
+ df.reset_index(drop=True, inplace=True)
+ unique_records = df.drop_duplicates(
+ subset=[
+ # "timestamp",
+ "category",
+ "test_type",
+ "model",
+ "hub",
+ "dataset_name",
+ "split",
+ "subset",
+ "task",
+ ],
+ # keep=,
+ )
+
+ unique_records.reset_index(drop=True, inplace=True)
+
+ return df
+
+ def __repr__(self) -> str:
+ return self.summary.summary_df.to_markdown()
+
+
+class Summary(Generic[TypeVar("T", bound="Summary")]):
+ """
+ Summary class to manage the summary report
+ """
+
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ """
+ Singleton pattern to ensure only one instance of the class is created
+ """
+ if not cls._instance:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self, path: str, *args, **kwargs) -> None:
+ """
+ Initialize the summary
+ """
+ self.save_dir = path
+ self.file_path = f"{path}summary.csv"
+
+ self.summary_df: pd.DataFrame = self.__update_summary_df()
+
+ def load_data_from_file(self, path: str, *args, **kwargs) -> pd.DataFrame:
+ """
+ Check if file exists
+ """
+ try:
+ if os.path.exists(path):
+ return self.__read_from_csv(path, *args, **kwargs)
+ else:
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ # Create a new file
+ df = pd.DataFrame(columns=self.__default_columns())
+ df.to_csv(path, index=False)
+ return df
+ except FileNotFoundError:
+ raise FileNotFoundError(f"File not found at {path}")
+
+ def __read_from_csv(self, path: str) -> pd.DataFrame:
+ """
+ Read data from csv file
+ """
+ df = pd.read_csv(path)
+ return df
+
+ def __default_columns(self):
+ """
+ Default columns for the summary report
+ """
+ cols = [
+ "timestamp",
+ "task",
+ "model",
+ "hub",
+ "category",
+ "test_type",
+ "dataset_name",
+ "split",
+ "subset",
+ "total_records",
+ "success_records",
+ "failure_records",
+ "score",
+ ]
+ return cols
+
+ def add_report(
+ self,
+ generated_results: pd.DataFrame,
+ ) -> None:
+ """
+ Add a new report to the summary
+ """
+ # Load and Update the summary dataframe
+ self.summary_df = self.__update_summary_df()
+
+ from datetime import datetime
+
+ # Filter the dataframe for accuracy, fairness and representation
+ afr_df = self.__afr(generated_results)
+ not_afr_df = self.__not_afr(generated_results)
+
+ # concatenate the dataframes
+ temp_summary_df = pd.concat([afr_df, not_afr_df], axis=0)
+
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ temp_summary_df["timestamp"] = timestamp
+
+ # insert row to the summary df
+ self.summary_df = pd.concat([self.summary_df, temp_summary_df], ignore_index=True)
+
+ # Save the summary to the file
+ self.save_summary()
+
+ def save_summary(self) -> None:
+ """
+ Save the summary to the file
+ """
+ self.summary_df.to_csv(self.file_path, index=False)
+
+ def __afr(self, df: pd.DataFrame) -> pd.DataFrame:
+ """
+ Filter the dataframe for accuracy, fairness and representation
+ to be used in the summary report
+ """
+ df = df[df["category"].isin(["accuracy", "fairness", "representation"])]
+ df = df[self.__group_by_cols() + ["actual_result"]]
+ df = df.rename(columns={"actual_result": "score"})
+
+ return df
+
+ def __not_afr(self, df: pd.DataFrame) -> pd.DataFrame:
+ """
+ Filter the dataframe for non accuracy, fairness and representation
+ to be used in the summary report
+ """
+ df = df[~df["category"].isin(["accuracy", "fairness", "representation"])]
+
+ grouped = df.groupby(self.__group_by_cols())
+
+ # Filter the columns
+ import numpy as np
+
+ total_records = grouped.size().reset_index(name="total_records")
+ success_records = grouped["pass"].sum().reset_index(name="success_records")
+ score = grouped["pass"].mean().reset_index(name="score")
+ failure_records = grouped.apply(
+ lambda x: np.size(x["pass"]) - np.sum(x["pass"])
+ ).reset_index(name="failure_records")
+
+ # concatenate the dataframes
+ result = pd.concat(
+ [
+ success_records,
+ failure_records["failure_records"],
+ total_records["total_records"],
+ score["score"],
+ ],
+ axis=1,
+ )
+
+ return result
+
+ def __group_by_cols(self):
+ """
+ Group by columns
+ """
+ return [
+ "category",
+ "dataset_name",
+ "test_type",
+ "model",
+ "hub",
+ "split",
+ "subset",
+ "task",
+ ]
+
+ @property
+ def df(self) -> pd.DataFrame:
+ return self.summary_df
+
+ def __update_summary_df(self):
+ """
+ Update the summary dataframe
+ """
+ if self.file_path.startswith("~"):
+ self.file_path = os.path.expanduser(self.file_path)
+ self.summary_df = self.load_data_from_file(self.file_path)
+ return self.summary_df
diff --git a/langtest/utils/config_utils.py b/langtest/utils/config_utils.py
index eab207c5d..9f762ce30 100644
--- a/langtest/utils/config_utils.py
+++ b/langtest/utils/config_utils.py
@@ -1,3 +1,4 @@
+from typing import List
from pkg_resources import resource_filename
@@ -47,3 +48,193 @@
"default": resource_filename("langtest", "data/config/wino_llm_config.yml"),
},
}
+
+BENCHMARK_DATASETS_DICT = {
+ "ASDiv": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "ASDiv", "split": "test-tiny"},
+ },
+ "BBQ": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "BBQ", "split": "test-tiny"},
+ },
+ "Bigbench": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {
+ "data_source": "Bigbench",
+ "subset": "Abstract-narrative-understanding",
+ "split": "test-tiny",
+ },
+ },
+ "BoolQ": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "BoolQ", "split": "test-tiny"},
+ },
+ "CommonsenseQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "CommonsenseQA", "split": "validation-tiny"},
+ },
+ "FIQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "Fiqa", "split": "test-tiny"},
+ },
+ "HellaSwag": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "HellaSwag", "split": "test-tiny"},
+ },
+ "Consumer-Contracts": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "Consumer-Contracts", "split": "test"},
+ },
+ "Contracts": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "Contracts", "split": "test"},
+ },
+ "Privacy-Policy": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "Privacy-Policy", "split": "test"},
+ },
+ "LogiQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "LogiQA", "split": "test-tiny"},
+ },
+ "MMLU": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "MMLU", "split": "test-tiny"},
+ },
+ "NarrativeQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "NarrativeQA", "split": "test-tiny"},
+ },
+ "NQ-open": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "NQ-open", "split": "test-tiny"},
+ },
+ "OpenBookQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "OpenBookQA", "split": "test-tiny"},
+ },
+ "PIQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "PIQA", "split": "test-tiny"},
+ },
+ "Quac": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "Quac", "split": "test-tiny"},
+ },
+ "SIQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "SIQA", "split": "test-tiny"},
+ },
+ "TruthfulQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "TruthfulQA", "split": "test-tiny"},
+ },
+ "XSum": {
+ "task": "summarization",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "XSum", "split": "test-tiny"},
+ },
+ "MultiLexSum": {
+ "task": "summarization",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {"data_source": "MultiLexSum", "split": "test-tiny"},
+ },
+ "MedMCQA": {
+ "task": "question-answering",
+ "model": {"model": "gpt-3.5-turbo-instruct", "hub": "openai"},
+ "data": {
+ "data_source": "MedMCQA",
+ "subset": "MedMCQA-Test",
+ "split": "Radiology",
+ },
+ },
+ "MedQA": {
+ "task": "question-answering",
+ "model": {"model": "mistralai/Mistral-7B-Instruct-v0.1", "hub": "huggingface"},
+ "data": {"data_source": "MedQA", "split": "test"},
+ "config": {
+ "evaluation": {
+ "metric": "string_distance",
+ "distance": "jaro",
+ "threshold": 0.1,
+ },
+ "tests": {
+ "defaults": {"min_pass_rate": 0.65},
+ "robustness": {
+ "add_ocr_typo": {"min_pass_rate": 0.66},
+ "dyslexia_word_swap": {"min_pass_rate": 0.60},
+ },
+ },
+ },
+ },
+ "PubMedQA": {
+ "task": "question-answering",
+ "model": {"model": "j2-jumbo-instruct", "hub": "ai21"},
+ "data": {"data_source": "PubMedQA", "split": "pqaa"},
+ "config": {
+ "evaluation": {
+ "metric": "string_distance",
+ "distance": "jaro",
+ "threshold": 0.1,
+ },
+ "tests": {
+ "defaults": {"min_pass_rate": 0.65},
+ "robustness": {
+ "add_ocr_typo": {"min_pass_rate": 0.66},
+ "dyslexia_word_swap": {"min_pass_rate": 0.60},
+ },
+ },
+ },
+ },
+}
+
+
+class BenchmarkDatasets:
+ def __init__(self, task: str, dataset_name: str):
+ self.__dataset_name = dataset_name
+ self.__task = task
+
+ @classmethod
+ def get_dataset_dict(cls, dataset_name: str, task: str) -> dict:
+ """Get the benchmark dataset configuration for the given dataset name."""
+ dataset_config = BENCHMARK_DATASETS_DICT.get(dataset_name, None)
+
+ if dataset_config is None:
+ raise ValueError(
+ f"Dataset {dataset_name} not found in the benchmark datasets list - {cls.get_datasets()}."
+ )
+
+ return dataset_config.get("data", None)
+
+ @classmethod
+ def get_datasets(cls) -> List[str]:
+ """Get the list of benchmark datasets available in the library."""
+ return list(BENCHMARK_DATASETS_DICT.keys())
+
+ @property
+ def dataset_name(self) -> str:
+ return self.__dataset_name
+
+ @property
+ def task(self) -> str:
+ return self.__task
diff --git a/langtest/utils/custom_types/helpers.py b/langtest/utils/custom_types/helpers.py
index 30bba5858..ab989f23c 100644
--- a/langtest/utils/custom_types/helpers.py
+++ b/langtest/utils/custom_types/helpers.py
@@ -113,6 +113,33 @@
"pubmedqa": "Context: {context}\nQuestion: {question}\n I've provided a question and context. From here on, I want you to become an intelligent bot that can only answer with one of these three choices: 'yes', 'no', or 'maybe'. If you think the answer to the question is yes, then say 'yes'. If it is no, then say 'no'. If the answer is uncertain or could be either yes or no, say 'maybe'. Do not say anything else other than that.",
}
+default_llm_chat_prompt = {
+ "ner": [
+ {
+ "role": "system",
+ "content": "You are an NER model designed to identify entities in the text. Your task is to classify each identified entity with its type, confidence score, word index, and the start and end positions in the input text. Respond with a list of dictionaries, each containing the keys 'entity', 'score', 'index', 'word', 'start', and 'end'. Note that all keys are mandatory and choose only ORG, and PER enities. Let's start with a few examples.",
+ },
+ {"role": "user", "content": "The sentence is: John is working at Google."},
+ {
+ "role": "assistant",
+ "content": "[{'entity': 'PER', 'score': 0.99, 'index': 1, 'word': 'John', 'start': 0, 'end': 4}, {'entity': 'ORG', 'score': 0.98, 'index': 5, 'word': 'Google', 'start': 19, 'end': 25}]",
+ },
+ {"role": "user", "content": "The sentence is: Elon Musk founded SpaceX."},
+ {
+ "role": "assistant",
+ "content": "[{'entity': 'PER', 'score': 0.99, 'index': 1, 'word': 'Elon Musk', 'start': 0, 'end': 9}, {'entity': 'ORG', 'score': 0.97, 'index': 4, 'word': 'SpaceX', 'start': 18, 'end': 24}]",
+ },
+ {
+ "role": "user",
+ "content": "The sentence is: Ada Lovelace is considered the first computer programmer.",
+ },
+ {
+ "role": "assistant",
+ "content": "[{'entity': 'PER', 'score': 0.98, 'index': 1, 'word': 'Ada Lovelace', 'start': 0, 'end': 12}]",
+ },
+ ]
+}
+
class Span(BaseModel):
"""Representation of a text's slice"""
diff --git a/langtest/utils/custom_types/sample.py b/langtest/utils/custom_types/sample.py
index 9fc752548..3a8d1abf2 100644
--- a/langtest/utils/custom_types/sample.py
+++ b/langtest/utils/custom_types/sample.py
@@ -373,7 +373,7 @@ class BaseQASample(BaseModel):
original_question: str
original_context: str
- options: str
+ options: str = None
test_type: str = None
perturbed_question: str = None
perturbed_context: str = None
@@ -948,7 +948,7 @@ def is_pass(self):
expected_unit = self.expected_results.split("/")[1]
actual_unit = self.actual_results.split("/")[1]
- return (expected_tokens >= actual_tokens) and (expected_unit == actual_unit)
+ return (expected_tokens <= actual_tokens) and (expected_unit == actual_unit)
class TranslationSample(BaseModel):
diff --git a/poetry.lock b/poetry.lock
index 7d22c5c89..807f568a5 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -414,35 +414,44 @@ numpy = [
[[package]]
name = "boto3"
-version = "1.7.84"
+version = "1.34.93"
description = "The AWS SDK for Python"
optional = true
-python-versions = "*"
+python-versions = ">=3.8"
files = [
- {file = "boto3-1.7.84-py2.py3-none-any.whl", hash = "sha256:0ed4b107c3b4550547aaec3c9bb17df068ff92d1f6f4781205800e2cb8a66de5"},
- {file = "boto3-1.7.84.tar.gz", hash = "sha256:64496f2c814e454e26c024df86bd08fb4643770d0e2b7a8fd70055fc6683eb9d"},
+ {file = "boto3-1.34.93-py3-none-any.whl", hash = "sha256:b59355bf4a1408563969526f314611dbeacc151cf90ecb22af295dcc4fe18def"},
+ {file = "boto3-1.34.93.tar.gz", hash = "sha256:e39516e4ca21612932599819662759c04485d53ca457996a913163da11f052a4"},
]
[package.dependencies]
-botocore = ">=1.10.84,<1.11.0"
-jmespath = ">=0.7.1,<1.0.0"
-s3transfer = ">=0.1.10,<0.2.0"
+botocore = ">=1.34.93,<1.35.0"
+jmespath = ">=0.7.1,<2.0.0"
+s3transfer = ">=0.10.0,<0.11.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
-version = "1.10.84"
+version = "1.34.93"
description = "Low-level, data-driven core of boto 3."
optional = true
-python-versions = "*"
+python-versions = ">=3.8"
files = [
- {file = "botocore-1.10.84-py2.py3-none-any.whl", hash = "sha256:380852e1adb9ba4ba9ff096af61f88a6888197b86e580e1bd786f04ebe6f9c0c"},
- {file = "botocore-1.10.84.tar.gz", hash = "sha256:d3e4b5a2c903ea30d19d41ea2f65d0e51dce54f4f4c4dfd6ecd7b04f240844a8"},
+ {file = "botocore-1.34.93-py3-none-any.whl", hash = "sha256:6fbd5a53a2adc9b3d4ebd90ae0ede83a91a41d96231f8a5984051f75495f246d"},
+ {file = "botocore-1.34.93.tar.gz", hash = "sha256:79d39b0b87e962991c6dd55e78ce15155099f6fb741be88b1b8a456a702cc150"},
]
[package.dependencies]
-docutils = ">=0.10"
-jmespath = ">=0.7.1,<1.0.0"
-python-dateutil = {version = ">=2.1,<3.0.0", markers = "python_version >= \"2.7\""}
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = [
+ {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""},
+ {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""},
+]
+
+[package.extras]
+crt = ["awscrt (==0.20.9)"]
[[package]]
name = "catalogue"
@@ -914,17 +923,6 @@ websocket-client = ">=0.32.0"
[package.extras]
ssh = ["paramiko (>=2.4.3)"]
-[[package]]
-name = "docutils"
-version = "0.20.1"
-description = "Docutils -- Python Documentation Utilities"
-optional = true
-python-versions = ">=3.7"
-files = [
- {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"},
- {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"},
-]
-
[[package]]
name = "en-core-web-sm"
version = "3.5.0"
@@ -1947,28 +1945,42 @@ extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.
[[package]]
name = "langchain-core"
-version = "0.1.29"
+version = "0.1.46"
description = "Building applications with LLMs through composability"
optional = true
-python-versions = ">=3.8.1,<4.0"
+python-versions = "<4.0,>=3.8.1"
files = [
- {file = "langchain_core-0.1.29-py3-none-any.whl", hash = "sha256:b96d599ff98810a7fcba726c151d473a4b938e0f90b9907c460b0bf0a1c7a0f7"},
- {file = "langchain_core-0.1.29.tar.gz", hash = "sha256:6731dabffad03b9213ada2640d54ed7f4ef6b99fce87ade3c71474ae154dd3cc"},
+ {file = "langchain_core-0.1.46-py3-none-any.whl", hash = "sha256:1c0befcd2665dd4aa153318aa9bf729071644b4c179e491769b8e583b4bf7441"},
+ {file = "langchain_core-0.1.46.tar.gz", hash = "sha256:17c416349f5c7a9808e70e3725749a3a2df5088f1ecca045c883871aa95f9c9e"},
]
[package.dependencies]
-anyio = ">=3,<5"
jsonpatch = ">=1.33,<2.0"
langsmith = ">=0.1.0,<0.2.0"
packaging = ">=23.2,<24.0"
pydantic = ">=1,<3"
PyYAML = ">=5.3"
-requests = ">=2,<3"
tenacity = ">=8.1.0,<9.0.0"
[package.extras]
extended-testing = ["jinja2 (>=3,<4)"]
+[[package]]
+name = "langchain-openai"
+version = "0.1.4"
+description = "An integration package connecting OpenAI and LangChain"
+optional = true
+python-versions = "<4.0,>=3.8.1"
+files = [
+ {file = "langchain_openai-0.1.4-py3-none-any.whl", hash = "sha256:a349ada8724921e380aab03ee312568f5ca99adbc806f6878d79ff9cd1d6d353"},
+ {file = "langchain_openai-0.1.4.tar.gz", hash = "sha256:1a3220464c270d73ea3987010617789adc2099d4d4740b15c7734ab07e1f054b"},
+]
+
+[package.dependencies]
+langchain-core = ">=0.1.46,<0.2.0"
+openai = ">=1.10.0,<2.0.0"
+tiktoken = ">=0.5.2,<1"
+
[[package]]
name = "langchain-text-splitters"
version = "0.0.1"
@@ -3629,17 +3641,20 @@ six = ">=1.14.0"
[[package]]
name = "s3transfer"
-version = "0.1.13"
+version = "0.10.1"
description = "An Amazon S3 Transfer Manager"
optional = true
-python-versions = "*"
+python-versions = ">= 3.8"
files = [
- {file = "s3transfer-0.1.13-py2.py3-none-any.whl", hash = "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f"},
- {file = "s3transfer-0.1.13.tar.gz", hash = "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1"},
+ {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"},
+ {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"},
]
[package.dependencies]
-botocore = ">=1.3.0,<2.0.0"
+botocore = ">=1.33.2,<2.0a.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"]
[[package]]
name = "safetensors"
@@ -4394,6 +4409,58 @@ files = [
{file = "threadpoolctl-3.2.0.tar.gz", hash = "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355"},
]
+[[package]]
+name = "tiktoken"
+version = "0.6.0"
+description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
+optional = true
+python-versions = ">=3.8"
+files = [
+ {file = "tiktoken-0.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:277de84ccd8fa12730a6b4067456e5cf72fef6300bea61d506c09e45658d41ac"},
+ {file = "tiktoken-0.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c44433f658064463650d61387623735641dcc4b6c999ca30bc0f8ba3fccaf5c"},
+ {file = "tiktoken-0.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afb9a2a866ae6eef1995ab656744287a5ac95acc7e0491c33fad54d053288ad3"},
+ {file = "tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c62c05b3109fefca26fedb2820452a050074ad8e5ad9803f4652977778177d9f"},
+ {file = "tiktoken-0.6.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ef917fad0bccda07bfbad835525bbed5f3ab97a8a3e66526e48cdc3e7beacf7"},
+ {file = "tiktoken-0.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e095131ab6092d0769a2fda85aa260c7c383072daec599ba9d8b149d2a3f4d8b"},
+ {file = "tiktoken-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:05b344c61779f815038292a19a0c6eb7098b63c8f865ff205abb9ea1b656030e"},
+ {file = "tiktoken-0.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cefb9870fb55dca9e450e54dbf61f904aab9180ff6fe568b61f4db9564e78871"},
+ {file = "tiktoken-0.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:702950d33d8cabc039845674107d2e6dcabbbb0990ef350f640661368df481bb"},
+ {file = "tiktoken-0.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d49d076058f23254f2aff9af603863c5c5f9ab095bc896bceed04f8f0b013a"},
+ {file = "tiktoken-0.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:430bc4e650a2d23a789dc2cdca3b9e5e7eb3cd3935168d97d43518cbb1f9a911"},
+ {file = "tiktoken-0.6.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:293cb8669757301a3019a12d6770bd55bec38a4d3ee9978ddbe599d68976aca7"},
+ {file = "tiktoken-0.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7bd1a288b7903aadc054b0e16ea78e3171f70b670e7372432298c686ebf9dd47"},
+ {file = "tiktoken-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac76e000183e3b749634968a45c7169b351e99936ef46f0d2353cd0d46c3118d"},
+ {file = "tiktoken-0.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:17cc8a4a3245ab7d935c83a2db6bb71619099d7284b884f4b2aea4c74f2f83e3"},
+ {file = "tiktoken-0.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:284aebcccffe1bba0d6571651317df6a5b376ff6cfed5aeb800c55df44c78177"},
+ {file = "tiktoken-0.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c1a3a5d33846f8cd9dd3b7897c1d45722f48625a587f8e6f3d3e85080559be8"},
+ {file = "tiktoken-0.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6318b2bb2337f38ee954fd5efa82632c6e5ced1d52a671370fa4b2eff1355e91"},
+ {file = "tiktoken-0.6.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f5f0f2ed67ba16373f9a6013b68da298096b27cd4e1cf276d2d3868b5c7efd1"},
+ {file = "tiktoken-0.6.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:75af4c0b16609c2ad02581f3cdcd1fb698c7565091370bf6c0cf8624ffaba6dc"},
+ {file = "tiktoken-0.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:45577faf9a9d383b8fd683e313cf6df88b6076c034f0a16da243bb1c139340c3"},
+ {file = "tiktoken-0.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7c1492ab90c21ca4d11cef3a236ee31a3e279bb21b3fc5b0e2210588c4209e68"},
+ {file = "tiktoken-0.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e2b380c5b7751272015400b26144a2bab4066ebb8daae9c3cd2a92c3b508fe5a"},
+ {file = "tiktoken-0.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f497598b9f58c99cbc0eb764b4a92272c14d5203fc713dd650b896a03a50ad"},
+ {file = "tiktoken-0.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e65e8bd6f3f279d80f1e1fbd5f588f036b9a5fa27690b7f0cc07021f1dfa0839"},
+ {file = "tiktoken-0.6.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5f1495450a54e564d236769d25bfefbf77727e232d7a8a378f97acddee08c1ae"},
+ {file = "tiktoken-0.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6c4e4857d99f6fb4670e928250835b21b68c59250520a1941618b5b4194e20c3"},
+ {file = "tiktoken-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:168d718f07a39b013032741867e789971346df8e89983fe3c0ef3fbd5a0b1cb9"},
+ {file = "tiktoken-0.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:47fdcfe11bd55376785a6aea8ad1db967db7f66ea81aed5c43fad497521819a4"},
+ {file = "tiktoken-0.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fb7d2ccbf1a7784810aff6b80b4012fb42c6fc37eaa68cb3b553801a5cc2d1fc"},
+ {file = "tiktoken-0.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ccb7a111ee76af5d876a729a347f8747d5ad548e1487eeea90eaf58894b3138"},
+ {file = "tiktoken-0.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2048e1086b48e3c8c6e2ceeac866561374cd57a84622fa49a6b245ffecb7744"},
+ {file = "tiktoken-0.6.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07f229a5eb250b6403a61200199cecf0aac4aa23c3ecc1c11c1ca002cbb8f159"},
+ {file = "tiktoken-0.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:432aa3be8436177b0db5a2b3e7cc28fd6c693f783b2f8722539ba16a867d0c6a"},
+ {file = "tiktoken-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:8bfe8a19c8b5c40d121ee7938cd9c6a278e5b97dc035fd61714b4f0399d2f7a1"},
+ {file = "tiktoken-0.6.0.tar.gz", hash = "sha256:ace62a4ede83c75b0374a2ddfa4b76903cf483e9cb06247f566be3bf14e6beed"},
+]
+
+[package.dependencies]
+regex = ">=2022.1.18"
+requests = ">=2.26.0"
+
+[package.extras]
+blobfile = ["blobfile (>=2)"]
+
[[package]]
name = "tokenizers"
version = "0.15.2"
@@ -4732,6 +4799,22 @@ files = [
{file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"},
]
+[[package]]
+name = "urllib3"
+version = "1.26.18"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+ {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
+ {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
+]
+
+[package.extras]
+brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
[[package]]
name = "urllib3"
version = "2.0.4"
@@ -5058,15 +5141,15 @@ datasets = ["datasets"]
evaluate = ["evaluate", "rouge-score", "seqeval"]
huggingface-hub = ["huggingface_hub", "langchain"]
johnsnowlabs = ["johnsnowlabs"]
-langchain = ["langchain"]
+langchain = ["langchain", "langchain-openai"]
matplotlib = ["matplotlib"]
metaflow = ["metaflow"]
mlflow = ["mlflow"]
-openai = ["langchain", "openai"]
+openai = ["langchain", "langchain-openai", "openai"]
spacy = ["spacy"]
transformers = ["accelerate", "datasets", "torch", "transformers"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0"
-content-hash = "86f5459a00c684b7232e2f9b5d4ae9d8b4975e942877b2accf238152a59a10d6"
+content-hash = "673eafe78578cc2d9f31a928a0ca5d898998f81a7abfaad65293a1ba8a27e8ed"
diff --git a/pyproject.toml b/pyproject.toml
index 8bb09a15a..21806b1b5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "langtest"
-version = "2.1.0"
+version = "2.2.0"
description = "John Snow Labs provides a library for delivering safe & effective NLP models."
authors = ["John Snow Labs "]
readme = "README.md"
@@ -70,14 +70,16 @@ matplotlib = {version = "^3.7.2", optional = true}
tenacity = {version = "^8.2.2", optional = true}
langchain = {version = "^0.1.11", optional = true}
typing-extensions = "^4.10.0"
+langchain-openai = {version = "^0.1.4", optional = true}
+boto3 = {version = "^1.34.93", optional = true}
[tool.poetry.extras]
transformers = ["transformers", "torch", "accelerate", "datasets"]
evaluate = ["evaluate", "rouge-score", "seqeval"]
spacy = ["spacy"]
-langchain = ["langchain"]
+langchain = ["langchain", "langchain-openai"]
johnsnowlabs = ["johnsnowlabs"]
-openai = ["openai", "langchain"]
+openai = ["openai", "langchain", "langchain-openai"]
cohere = ["cohere", "langchain"]
ai21 = ["ai21", "langchain"]
huggingface_hub = ["huggingface_hub", "langchain"]
diff --git a/tests/fixtures/test.conll b/tests/fixtures/test.conll
index 79bcd5603..577a4a17a 100644
--- a/tests/fixtures/test.conll
+++ b/tests/fixtures/test.conll
@@ -160,6 +160,71 @@ first JJ I-NP O
time NN I-NP O
. . O O
+A DT B-NP O
+London-based JJ I-NP B-MISC
+spokesman NN I-NP O
+of IN B-PP O
+the DT B-NP O
+Iraqi JJ I-NP B-ORG
+National NNP I-NP I-ORG
+Congress NNP B-NP I-ORG
+said VBD B-VP O
+Iraqi JJ B-NP B-MISC
+artillery NN I-NP O
+was VBD B-VP O
+shelling VBG I-VP O
+the DT B-NP O
+city NN I-NP O
+and CC I-NP O
+Iraqi JJ I-NP B-MISC
+tanks NNS I-NP O
+had VBD B-VP O
+advanced VBN I-VP O
+to TO B-PP O
+within IN B-NP O
+10 CD I-NP O
+km NN I-NP O
+( ( O O
+six CD B-NP O
+miles NNS I-NP O
+) ) O O
+of IN B-PP O
+Arbil NNP B-NP B-LOC
+, , O O
+the DT B-NP O
+administrative JJ I-NP O
+centre NN I-NP O
+of IN B-PP O
+the DT B-NP O
+Kurdish JJ I-NP B-MISC
+rebel-controlled JJ I-NP O
+region NN I-NP O
+of IN B-PP O
+northern JJ B-NP O
+Iraq NNP I-NP B-LOC
+. . O O
+
+Brokers NNS B-NP O
+said VBD B-VP O
+blue JJ B-NP O
+chips NNS I-NP O
+like IN B-PP O
+IDLC NNP B-NP B-ORG
+, , O O
+Bangladesh NNP B-NP B-ORG
+Lamps NNP I-NP I-ORG
+, , O O
+Chittagong NNP B-NP B-ORG
+Cement NNP I-NP I-ORG
+and CC O O
+Atlas NNP B-NP B-ORG
+Bangladesh NNP I-NP I-ORG
+were VBD B-VP O
+expected VBN I-VP O
+to TO B-PP O
+rise VB B-NP O
+. . O O
+
Despite IN B-PP O
winning VBG B-VP O
the DT B-NP O
@@ -1466,3 +1531,122 @@ China NNP B-NP B-LOC
0 CD I-NP O
2 CD I-NP O
0 CD I-NP O
+
+A DT B-NP O
+London-based JJ I-NP B-MISC
+spokesman NN I-NP O
+of IN B-PP O
+the DT B-NP O
+Iraqi JJ I-NP B-ORG
+National NNP I-NP I-ORG
+Congress NNP B-NP I-ORG
+said VBD B-VP O
+Iraqi JJ B-NP B-MISC
+artillery NN I-NP O
+was VBD B-VP O
+shelling VBG I-VP O
+the DT B-NP O
+city NN I-NP O
+and CC I-NP O
+Iraqi JJ I-NP B-MISC
+tanks NNS I-NP O
+had VBD B-VP O
+advanced VBN I-VP O
+to TO B-PP O
+within IN B-NP O
+10 CD I-NP O
+km NN I-NP O
+( ( O O
+six CD B-NP O
+miles NNS I-NP O
+) ) O O
+of IN B-PP O
+Arbil NNP B-NP B-LOC
+, , O O
+the DT B-NP O
+administrative JJ I-NP O
+centre NN I-NP O
+of IN B-PP O
+the DT B-NP O
+Kurdish JJ I-NP B-MISC
+rebel-controlled JJ I-NP O
+region NN I-NP O
+of IN B-PP O
+northern JJ B-NP O
+Iraq NNP I-NP B-LOC
+. . O O
+
+Brokers NNS B-NP O
+said VBD B-VP O
+blue JJ B-NP O
+chips NNS I-NP O
+like IN B-PP O
+IDLC NNP B-NP B-ORG
+, , O O
+Bangladesh NNP B-NP B-ORG
+Lamps NNP I-NP I-ORG
+, , O O
+Chittagong NNP B-NP B-ORG
+Cement NNP I-NP I-ORG
+and CC O O
+Atlas NNP B-NP B-ORG
+Bangladesh NNP I-NP I-ORG
+were VBD B-VP O
+expected VBN I-VP O
+to TO B-PP O
+rise VB B-NP O
+. . O O
+
+They PRP B-NP O
+said VBD B-VP O
+there EX B-NP O
+was VBD B-VP O
+still RB I-VP O
+demand VB I-VP O
+for IN B-PP O
+blue JJ B-NP O
+chips NNS I-NP O
+in IN B-PP O
+engineering NN B-NP O
+sector NN I-NP O
+despite IN B-PP O
+their PRP$ B-NP O
+persistent JJ I-NP O
+rise NN I-NP O
+over IN B-PP O
+the DT B-NP O
+past JJ I-NP O
+several JJ I-NP O
+sessions NNS I-NP O
+. . O O
+
+The DT B-NP O
+DSE NN I-NP B-ORG
+all DT B-NP O
+share NN I-NP O
+price NN I-NP O
+index NN I-NP O
+closed VBD B-VP O
+2.73 CD B-NP O
+points NNS I-NP O
+or CC O O
+0.22 CD B-NP O
+percent NN I-NP O
+up IN B-PP O
+at IN B-PP O
+1,196.35 CD B-NP O
+on IN B-PP O
+a DT B-NP O
+turnover NN I-NP O
+of IN B-PP O
+133.7 CD B-NP O
+million CD I-NP O
+taka NN I-NP O
+on IN B-PP O
+Thursday NNP B-NP O
+. . O O
+
+-- : O O
+Dhaka NNP B-NP B-ORG
+Newsroom NNP I-NP I-ORG
+880-2-506363 CD I-NP O
\ No newline at end of file
diff --git a/tests/test_datasource.py b/tests/test_datasource.py
index 55f2c0e29..d83054f7e 100644
--- a/tests/test_datasource.py
+++ b/tests/test_datasource.py
@@ -83,10 +83,10 @@ def test_load_raw_data(self, dataset, feature_col, target_col):
(
HuggingFaceDataset(
source_info={
- "data_source": "wikiann",
- "subset": "fo",
+ "data_source": "tner/wikiann",
+ "subset": "ace",
"feature_column": "tokens",
- "target_column": "ner_tags",
+ "target_column": "tags",
"split": "test",
},
task=TaskManager("ner"),