diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..268efb6b9 --- /dev/null +++ b/Pipfile @@ -0,0 +1,42 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +alabaster = "==0.7.12" +babel = "==2.9.1" +certifi = "==2021.10.8" +charset-normalizer = "==2.0.12" +commonmark = "==0.9.1" +docutils = "==0.17.1" +idna = "==3.3" +imagesize = "==1.3.0" +importlib-metadata = "==4.11.3" +jinja2 = "==3.0.3" +markupsafe = "==2.1.0" +packaging = "==21.3" +pyenchant = "==3.2.2" +pygments = "==2.11.2" +pyparsing = "==3.0.7" +pytz = "==2021.3" +recommonmark = "*" +requests = "==2.27.1" +snowballstemmer = "==2.2.0" +sphinx = "==4.4.0" +sphinxcontrib-applehelp = "==1.0.2" +sphinxcontrib-devhelp = "==1.0.2" +sphinxcontrib-htmlhelp = "==2.0.0" +sphinxcontrib-jsmath = "==1.0.1" +sphinxcontrib-qthelp = "==1.0.3" +sphinxcontrib-serializinghtml = "==1.1.5" +sphinxcontrib-spelling = "==7.3.2" +urllib3 = "==1.26.8" +zipp = "==3.7.0" +sphinx-autobuild = "*" +sphinx-material = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 000000000..8bea44e20 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,521 @@ +{ + "_meta": { + "hash": { + "sha256": "2cfa3808c6a2b5a65b9e66cc89f561b6c3fac0643fab4949ec97935e676bf3d8" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "alabaster": { + "hashes": [ + "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", + "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" + ], + "index": "pypi", + "version": "==0.7.12" + }, + "babel": { + "hashes": [ + "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9", + "sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.1" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da", + "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==4.12.2" + }, + "certifi": { + "hashes": [ + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" + ], + "index": "pypi", + "version": "==2021.10.8" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + ], + "index": "pypi", + "markers": "python_full_version >= '3.5.0'", + "version": "==2.0.12" + }, + "colorama": { + "hashes": [ + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==0.4.6" + }, + "commonmark": { + "hashes": [ + "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", + "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "css-html-js-minify": { + "hashes": [ + "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a", + "sha256:4704e04a0cd6dd56d61bbfa3bfffc630da6b2284be33519be0b456672e2a2438", + "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c" + ], + "version": "==2.5.5" + }, + "docutils": { + "hashes": [ + "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125", + "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.17.1" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "imagesize": { + "hashes": [ + "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c", + "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.3.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6", + "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.11.3" + }, + "jinja2": { + "hashes": [ + "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8", + "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==3.0.3" + }, + "livereload": { + "hashes": [ + "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869", + "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4" + ], + "version": "==2.6.3" + }, + "lxml": { + "hashes": [ + "sha256:05186a0f1346ae12553d66df1cfce6f251589fea3ad3da4f3ef4e34b2d58c6a3", + "sha256:075b731ddd9e7f68ad24c635374211376aa05a281673ede86cbe1d1b3455279d", + "sha256:081d32421db5df44c41b7f08a334a090a545c54ba977e47fd7cc2deece78809a", + "sha256:0a3d3487f07c1d7f150894c238299934a2a074ef590b583103a45002035be120", + "sha256:0bfd0767c5c1de2551a120673b72e5d4b628737cb05414f03c3277bf9bed3305", + "sha256:0c0850c8b02c298d3c7006b23e98249515ac57430e16a166873fc47a5d549287", + "sha256:0e2cb47860da1f7e9a5256254b74ae331687b9672dfa780eed355c4c9c3dbd23", + "sha256:120fa9349a24c7043854c53cae8cec227e1f79195a7493e09e0c12e29f918e52", + "sha256:1247694b26342a7bf47c02e513d32225ededd18045264d40758abeb3c838a51f", + "sha256:141f1d1a9b663c679dc524af3ea1773e618907e96075262726c7612c02b149a4", + "sha256:14e019fd83b831b2e61baed40cab76222139926b1fb5ed0e79225bc0cae14584", + "sha256:1509dd12b773c02acd154582088820893109f6ca27ef7291b003d0e81666109f", + "sha256:17a753023436a18e27dd7769e798ce302963c236bc4114ceee5b25c18c52c693", + "sha256:1e224d5755dba2f4a9498e150c43792392ac9b5380aa1b845f98a1618c94eeef", + "sha256:1f447ea5429b54f9582d4b955f5f1985f278ce5cf169f72eea8afd9502973dd5", + "sha256:23eed6d7b1a3336ad92d8e39d4bfe09073c31bfe502f20ca5116b2a334f8ec02", + "sha256:25f32acefac14ef7bd53e4218fe93b804ef6f6b92ffdb4322bb6d49d94cad2bc", + "sha256:2c74524e179f2ad6d2a4f7caf70e2d96639c0954c943ad601a9e146c76408ed7", + "sha256:303bf1edce6ced16bf67a18a1cf8339d0db79577eec5d9a6d4a80f0fb10aa2da", + "sha256:3331bece23c9ee066e0fb3f96c61322b9e0f54d775fccefff4c38ca488de283a", + "sha256:3e9bdd30efde2b9ccfa9cb5768ba04fe71b018a25ea093379c857c9dad262c40", + "sha256:411007c0d88188d9f621b11d252cce90c4a2d1a49db6c068e3c16422f306eab8", + "sha256:42871176e7896d5d45138f6d28751053c711ed4d48d8e30b498da155af39aebd", + "sha256:46f409a2d60f634fe550f7133ed30ad5321ae2e6630f13657fb9479506b00601", + "sha256:48628bd53a426c9eb9bc066a923acaa0878d1e86129fd5359aee99285f4eed9c", + "sha256:48d6ed886b343d11493129e019da91d4039826794a3e3027321c56d9e71505be", + "sha256:4930be26af26ac545c3dffb662521d4e6268352866956672231887d18f0eaab2", + "sha256:4aec80cde9197340bc353d2768e2a75f5f60bacda2bab72ab1dc499589b3878c", + "sha256:4c28a9144688aef80d6ea666c809b4b0e50010a2aca784c97f5e6bf143d9f129", + "sha256:4d2d1edbca80b510443f51afd8496be95529db04a509bc8faee49c7b0fb6d2cc", + "sha256:4dd9a263e845a72eacb60d12401e37c616438ea2e5442885f65082c276dfb2b2", + "sha256:4f1026bc732b6a7f96369f7bfe1a4f2290fb34dce00d8644bc3036fb351a4ca1", + "sha256:4fb960a632a49f2f089d522f70496640fdf1218f1243889da3822e0a9f5f3ba7", + "sha256:50670615eaf97227d5dc60de2dc99fb134a7130d310d783314e7724bf163f75d", + "sha256:50baa9c1c47efcaef189f31e3d00d697c6d4afda5c3cde0302d063492ff9b477", + "sha256:53ace1c1fd5a74ef662f844a0413446c0629d151055340e9893da958a374f70d", + "sha256:5515edd2a6d1a5a70bfcdee23b42ec33425e405c5b351478ab7dc9347228f96e", + "sha256:56dc1f1ebccc656d1b3ed288f11e27172a01503fc016bcabdcbc0978b19352b7", + "sha256:578695735c5a3f51569810dfebd05dd6f888147a34f0f98d4bb27e92b76e05c2", + "sha256:57aba1bbdf450b726d58b2aea5fe47c7875f5afb2c4a23784ed78f19a0462574", + "sha256:57d6ba0ca2b0c462f339640d22882acc711de224d769edf29962b09f77129cbf", + "sha256:5c245b783db29c4e4fbbbfc9c5a78be496c9fea25517f90606aa1f6b2b3d5f7b", + "sha256:5c31c7462abdf8f2ac0577d9f05279727e698f97ecbb02f17939ea99ae8daa98", + "sha256:64f479d719dc9f4c813ad9bb6b28f8390360660b73b2e4beb4cb0ae7104f1c12", + "sha256:65299ea57d82fb91c7f019300d24050c4ddeb7c5a190e076b5f48a2b43d19c42", + "sha256:6689a3d7fd13dc687e9102a27e98ef33730ac4fe37795d5036d18b4d527abd35", + "sha256:690dafd0b187ed38583a648076865d8c229661ed20e48f2335d68e2cf7dc829d", + "sha256:6fc3c450eaa0b56f815c7b62f2b7fba7266c4779adcf1cece9e6deb1de7305ce", + "sha256:704f61ba8c1283c71b16135caf697557f5ecf3e74d9e453233e4771d68a1f42d", + "sha256:71c52db65e4b56b8ddc5bb89fb2e66c558ed9d1a74a45ceb7dcb20c191c3df2f", + "sha256:71d66ee82e7417828af6ecd7db817913cb0cf9d4e61aa0ac1fde0583d84358db", + "sha256:7d298a1bd60c067ea75d9f684f5f3992c9d6766fadbc0bcedd39750bf344c2f4", + "sha256:8b77946fd508cbf0fccd8e400a7f71d4ac0e1595812e66025bac475a8e811694", + "sha256:8d7e43bd40f65f7d97ad8ef5c9b1778943d02f04febef12def25f7583d19baac", + "sha256:8df133a2ea5e74eef5e8fc6f19b9e085f758768a16e9877a60aec455ed2609b2", + "sha256:8ed74706b26ad100433da4b9d807eae371efaa266ffc3e9191ea436087a9d6a7", + "sha256:92af161ecbdb2883c4593d5ed4815ea71b31fafd7fd05789b23100d081ecac96", + "sha256:97047f0d25cd4bcae81f9ec9dc290ca3e15927c192df17331b53bebe0e3ff96d", + "sha256:9719fe17307a9e814580af1f5c6e05ca593b12fb7e44fe62450a5384dbf61b4b", + "sha256:9767e79108424fb6c3edf8f81e6730666a50feb01a328f4a016464a5893f835a", + "sha256:9a92d3faef50658dd2c5470af249985782bf754c4e18e15afb67d3ab06233f13", + "sha256:9bb6ad405121241e99a86efff22d3ef469024ce22875a7ae045896ad23ba2340", + "sha256:9e28c51fa0ce5674be9f560c6761c1b441631901993f76700b1b30ca6c8378d6", + "sha256:aca086dc5f9ef98c512bac8efea4483eb84abbf926eaeedf7b91479feb092458", + "sha256:ae8b9c6deb1e634ba4f1930eb67ef6e6bf6a44b6eb5ad605642b2d6d5ed9ce3c", + "sha256:b0a545b46b526d418eb91754565ba5b63b1c0b12f9bd2f808c852d9b4b2f9b5c", + "sha256:b4e4bc18382088514ebde9328da057775055940a1f2e18f6ad2d78aa0f3ec5b9", + "sha256:b6420a005548ad52154c8ceab4a1290ff78d757f9e5cbc68f8c77089acd3c432", + "sha256:b86164d2cff4d3aaa1f04a14685cbc072efd0b4f99ca5708b2ad1b9b5988a991", + "sha256:bb3bb49c7a6ad9d981d734ef7c7193bc349ac338776a0360cc671eaee89bcf69", + "sha256:bef4e656f7d98aaa3486d2627e7d2df1157d7e88e7efd43a65aa5dd4714916cf", + "sha256:c0781a98ff5e6586926293e59480b64ddd46282953203c76ae15dbbbf302e8bb", + "sha256:c2006f5c8d28dee289f7020f721354362fa304acbaaf9745751ac4006650254b", + "sha256:c41bfca0bd3532d53d16fd34d20806d5c2b1ace22a2f2e4c0008570bf2c58833", + "sha256:cd47b4a0d41d2afa3e58e5bf1f62069255aa2fd6ff5ee41604418ca925911d76", + "sha256:cdb650fc86227eba20de1a29d4b2c1bfe139dc75a0669270033cb2ea3d391b85", + "sha256:cef2502e7e8a96fe5ad686d60b49e1ab03e438bd9123987994528febd569868e", + "sha256:d27be7405547d1f958b60837dc4c1007da90b8b23f54ba1f8b728c78fdb19d50", + "sha256:d37017287a7adb6ab77e1c5bee9bcf9660f90ff445042b790402a654d2ad81d8", + "sha256:d3ff32724f98fbbbfa9f49d82852b159e9784d6094983d9a8b7f2ddaebb063d4", + "sha256:d73d8ecf8ecf10a3bd007f2192725a34bd62898e8da27eb9d32a58084f93962b", + "sha256:dd708cf4ee4408cf46a48b108fb9427bfa00b9b85812a9262b5c668af2533ea5", + "sha256:e3cd95e10c2610c360154afdc2f1480aea394f4a4f1ea0a5eacce49640c9b190", + "sha256:e4da8ca0c0c0aea88fd46be8e44bd49716772358d648cce45fe387f7b92374a7", + "sha256:eadfbbbfb41b44034a4c757fd5d70baccd43296fb894dba0295606a7cf3124aa", + "sha256:ed667f49b11360951e201453fc3967344d0d0263aa415e1619e85ae7fd17b4e0", + "sha256:f3df3db1d336b9356dd3112eae5f5c2b8b377f3bc826848567f10bfddfee77e9", + "sha256:f6bdac493b949141b733c5345b6ba8f87a226029cbabc7e9e121a413e49441e0", + "sha256:fbf521479bcac1e25a663df882c46a641a9bff6b56dc8b0fafaebd2f66fb231b", + "sha256:fc9b106a1bf918db68619fdcd6d5ad4f972fdd19c01d19bdb6bf63f3589a9ec5", + "sha256:fcdd00edfd0a3001e0181eab3e63bd5c74ad3e67152c84f93f13769a40e073a7", + "sha256:fe4bda6bd4340caa6e5cf95e73f8fea5c4bfc55763dd42f1b50a94c1b4a2fbd4" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.9.3" + }, + "markupsafe": { + "hashes": [ + "sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3", + "sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8", + "sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759", + "sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed", + "sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989", + "sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3", + "sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a", + "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c", + "sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c", + "sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8", + "sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454", + "sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad", + "sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d", + "sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635", + "sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61", + "sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea", + "sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49", + "sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce", + "sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e", + "sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f", + "sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f", + "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f", + "sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7", + "sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a", + "sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7", + "sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076", + "sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb", + "sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7", + "sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7", + "sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c", + "sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26", + "sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c", + "sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8", + "sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448", + "sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956", + "sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05", + "sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1", + "sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357", + "sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea", + "sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==2.1.0" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "pyenchant": { + "hashes": [ + "sha256:1cf830c6614362a78aab78d50eaf7c6c93831369c52e1bb64ffae1df0341e637", + "sha256:5a636832987eaf26efe971968f4d1b78e81f62bca2bde0a9da210c7de43c3bce", + "sha256:5facc821ece957208a81423af7d6ec7810dad29697cb0d77aae81e4e11c8e5a6", + "sha256:6153f521852e23a5add923dbacfbf4bebbb8d70c4e4bad609a8e0f9faeb915d1" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==3.2.2" + }, + "pygments": { + "hashes": [ + "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65", + "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==2.11.2" + }, + "pyparsing": { + "hashes": [ + "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", + "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==3.0.7" + }, + "python-slugify": { + "extras": [ + "unidecode" + ], + "hashes": [ + "sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395", + "sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27" + ], + "markers": "python_version >= '3.7'", + "version": "==8.0.1" + }, + "pytz": { + "hashes": [ + "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c", + "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326" + ], + "index": "pypi", + "version": "==2021.3" + }, + "recommonmark": { + "hashes": [ + "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f", + "sha256:bdb4db649f2222dcd8d2d844f0006b958d627f732415d399791ee436a3686d67" + ], + "index": "pypi", + "version": "==0.7.1" + }, + "requests": { + "hashes": [ + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.27.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "snowballstemmer": { + "hashes": [ + "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1", + "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a" + ], + "index": "pypi", + "version": "==2.2.0" + }, + "soupsieve": { + "hashes": [ + "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690", + "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7" + ], + "markers": "python_version >= '3.8'", + "version": "==2.5" + }, + "sphinx": { + "hashes": [ + "sha256:5da895959511473857b6d0200f56865ed62c31e8f82dd338063b84ec022701fe", + "sha256:6caad9786055cb1fa22b4a365c1775816b876f91966481765d7d50e9f0dd35cc" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==4.4.0" + }, + "sphinx-autobuild": { + "hashes": [ + "sha256:8fe8cbfdb75db04475232f05187c776f46f6e9e04cacf1e49ce81bdac649ccac", + "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==2021.3.14" + }, + "sphinx-material": { + "hashes": [ + "sha256:1d7f972cca7ebdfe135e28f18401673306d7c0d036d42c6e3d98b77394e61a60", + "sha256:eeff5f7d3dc016af32bafdf70c66e671d15c8754dbe0613dfbd629fbed912869" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==0.0.36" + }, + "sphinxcontrib-applehelp": { + "hashes": [ + "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", + "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==1.0.2" + }, + "sphinxcontrib-devhelp": { + "hashes": [ + "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", + "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==1.0.2" + }, + "sphinxcontrib-htmlhelp": { + "hashes": [ + "sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07", + "sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==2.0.0" + }, + "sphinxcontrib-jsmath": { + "hashes": [ + "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", + "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==1.0.1" + }, + "sphinxcontrib-qthelp": { + "hashes": [ + "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", + "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==1.0.3" + }, + "sphinxcontrib-serializinghtml": { + "hashes": [ + "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", + "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==1.1.5" + }, + "sphinxcontrib-spelling": { + "hashes": [ + "sha256:1b99cdb1a30271c7080ec5b968dfc243c2540a960afdc4c052cd59dfe8d94c54", + "sha256:9d66dc4990749c5ac52e7eaf17e82f4dc6b4aff6515d26bbf48821829d41bd02" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==7.3.2" + }, + "text-unidecode": { + "hashes": [ + "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", + "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" + ], + "version": "==1.3" + }, + "tornado": { + "hashes": [ + "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f", + "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5", + "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d", + "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3", + "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2", + "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a", + "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16", + "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a", + "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17", + "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0", + "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe" + ], + "markers": "python_version > '2.7'", + "version": "==6.3.3" + }, + "unidecode": { + "hashes": [ + "sha256:3c90b4662aa0de0cb591884b934ead8d2225f1800d8da675a7750cbc3bd94610", + "sha256:663a537f506834ed836af26a81b210d90cbde044c47bfbdc0fbbc9f94c86a6e4" + ], + "version": "==1.3.7" + }, + "urllib3": { + "hashes": [ + "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", + "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.8" + }, + "zipp": { + "hashes": [ + "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", + "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==3.7.0" + } + }, + "develop": {} +} diff --git a/README.md b/README.md index f3d775a9f..4cb7bf2b6 100644 --- a/README.md +++ b/README.md @@ -1,138 +1,66 @@ -# Clowder V2 (In Active Development) +![](docs/source/img/logo_full.png) -*For the previous version of Clowder, please see [Clowder V1](https://github.com/clowder-framework/clowder).* +# Clowder v2 (In active development) +[![Build Status](https://github.com/clowder-framework/clowder2/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/clowder-framework/clowder2/actions?query=branch%3Amain) +[![Slack](https://img.shields.io/badge/Slack-4A154B?&logo=slack&logoColor=white)](https://join.slack.com/t/clowder-software/shared_invite/zt-4e0vo0sh-YNndJEuLtPGRa7~uIlpcNA) -Clowder V2 is a reimagining of the [Clowder](https://clowderframework.org/) research data management system -using a different and newer technology stack. While the Clowder V1 has served us well, many of the underlying -technologies and libraries have not received enough support in recent years and new developers have a difficult -time learning how to contribute to it. Clowder V2 is also an opportunity to leverage our experience working with -research data in Clowder and deliver -a better solution to common problems researchers encounter when working with data. +[//]: # ([![Documentation Status](https://readthedocs.org/projects/clowder2/badge/?version=latest)](https://clowder2.readthedocs.io/en/latest/?badge=latest)) -In this version of Clowder, the application is clearly divided into backend and frontend modules. While this is somewhat -similar to Clowder V1, it used the [Play Framework](https://www.playframework.com/) and hence the fronted was created -at the server side, next to a standalone web Application Programming Interface (API). In Clowder V2, the -frontend module is a standalone [React](https://react.dev/) application and the backend, a -standalone [FastAPI](https://fastapi.tiangolo.com/lo/) web API. We continue to leverage -[MongoDB](https://www.mongodb.com/), [RabbitMQ](https://www.rabbitmq.com/), -and [Elasticsearch](https://www.elastic.co/). We also use [MinIO](https://min.io/) out of the box as the default object -store and [Traefik](https://traefik.io/traefik/) as the application proxy. +*For the previous version of Clowder, please see [Clowder v1](https://github.com/clowder-framework/clowder).* -## Running in Docker +Clowder v2 is a reimagining of the [Clowder research data management system](https://clowderframework.org/) +using a different and newer technology stack. Clowder is a cloud native data management framework to support any +research domain. Clowder was developed to help researchers and scientists in data intensive domains manage raw data, +complex metadata, and automatic data pipelines. -To run the full stack using [Docker](https://www.docker.com/) (recommended), please use the following instructions: +While the Clowder v1 has worked well over the years, many of the underlying +technologies and libraries have not received enough support in recent years and new developers have had a challenging +time learning how to contribute to it. +Clowder v2 is also an opportunity to leverage our experience working with +research data in Clowder and deliver a better solution to common problems researchers encounter when working with data. -1. Run all Docker services with `docker compose up --scale backend=4 --build`. This will start the services with four - instances of the backend module running in parallel. Note the `--build` flag used to build the images first. If using - default images, that flag can be removed. The images can also be built with `docker compose build`. +Clowder v2 provides: -2. The application will be running and available at `http://localhost`. +- a better user experience and user interface +- an easier code base to pick up and modify written in Python/FastAPI and Typescript/React +- new features based on our experience working with researchers -3. To access the Traefik dashboard, go to `http://localhost:8080`. To view the raw - settings, go to `http://localhost:8080/api/rawdata`. +## Documentation -## Developing +The v2 documentation is still work in progress. It's available at https://clowder2.readthedocs.io. -When developing, the required services can be run using Docker. You can then run the backend -and frontend modules from the command line or in your favorite IDE (to make debugging easier). We recommend -using [PyCharm](https://www.jetbrains.com/pycharm/) and have -made our run configurations available in the `.run` folder. PyCharm should automatically import it, but you will have -to change the path to the Python virtual environment to point to your path on your host (see Initial Dependencies -section below). +The v1 documentation is not fully compatible with v2, but it does provide some still relevant information. +It is available at https://clowder-framework.readthedocs.io. +There is a few other documentation links available on the [website](https://clowderframework.org/documentation.html). -### Initial Development Dependencies +## Installation -- Run `python3 -m venv venv` to create a Python Virtual Environment and add it to PyCharm by navigating to - `PyCharm -> Settings... -> Project: clowder2 -> Python Interpreter -> Add Interpreter`. -- Run `source venv/bin/activate && pip install --upgrade pip` to activate the created Python Virtual Environment and - upgrade - pip. -- Run `pip install pipenv` to install [Pipenv](https://pipenv.pypa.io/en/latest/). +The easiest way of running Clowder v2 is checking out the [code](https://github.com/clowder-framework/clowder2) +and running `docker compose up` in the main directory. -### Required Services +Helm charts are available for running Clowder v2 on Kubernetes. See the [helm](https://github.com/clowder-framework/clowder2/tree/main/deployments/kubernetes/charts) directory for more information. -- Running `./docker-dev.sh up` brings up the required services in the background. -- Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers, - provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`. -- Running `./docker-dev.sh down` brings down the required services. +## Contributing -**Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers -don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`. -If this is not used, the keycloak container will use the volume created with the other docker compose and it will be -unable to run as the information stored in the postgres database is different. +We are always looking for contributors. This could be anything from fixing bugs, adding new features, providing new +feature requests, reccomending UI/UX improvements, helping with the documentation, or just testing the system and +providing feedback. Here are a few ways to get started: -### Backend Module +- Join our [Slack](https://join.slack.com/t/clowder-software/shared_invite/zt-4e0vo0sh-YNndJEuLtPGRa7~uIlpcNA) + channel, introduce yourself, and ask questions about the specific aspects of the system you are interested in. +- Submit an issue (bug or feature request) on the [issue tracker](https://github.com/clowder-framework/clowder2/issues). +- Submit a [pull request](https://github.com/clowder-framework/clowder2/pulls) with a bug fix or new feature. For + larger changes, it's best to open an issue first or ask on Slack to discuss the changes. +- Develop new [information extractors](https://github.com/clowder-framework/pyclowder) and/or visualizations. -After starting up the required services, setup and run the backend module. +Please follow our [code of conduct](https://github.com/clowder-framework/clowder/blob/develop/CODE_OF_CONDUCT.md) when +interacting with the community. -The backend module is developed using [Python](https://www.python.org/), [FastAPI](https://fastapi.tiangolo.com/), -and [Motor](https://motor.readthedocs.io/en/stable/). -We recommend using [Python 3.9](https://www.python.org/downloads/) -and Pipenv for dependency management. +## Support & Contacts -#### Install Backend Dependencies +The easiest way to get in touch with us is [Slack](https://join.slack.com/t/clowder-software/shared_invite/zt-4e0vo0sh-YNndJEuLtPGRa7~uIlpcNA). +This is a public forum. If you prefer email, you can contact us at [clowder@lists.illinois.edu](mailto:clowder@lists.illinois.edu). -1. Switch to backend module directory `cd backend`. -2. Install dependencies using `pipenv install --dev`. +## License -#### Run Backend Module - -You can run the backend module using either of the below options: - -- Using the PyCharm's run configuration by navigating to `PyCharm -> Run -> Run...` and clicking `uvicorn`. Running - directly from PyCharm helps the developer by providing easy access to its debugging features. -- From the command line by running `pipenv run uvicorn app.main:app --reload` . - -Additional steps/details: - -1. API docs are available at `http://localhost:8000/docs`. The API base URL is `http://localhost:8000/api/v2`. -2. Create a user using `POST /api/v2/users` and getting a JWT token by using `POST /api/v2/login`. Place the token in - header of requests that require authentications using the `Authorization: Bearer ` HTTP header. - * You can also run the frontend module below and use the Login link available there. -3. Manually run tests before pushing with `pipenv run pytest -v` or right-clicking on `test` folder and clicking `Run` - in PyCharm. -4. Linting is done using [Black]((https://black.readthedocs.io/en/stable/)). You can set up PyCharm to automatically - run it when you save a file using - these [instructions](https://black.readthedocs.io/en/stable/integrations/editors.html). - The git repository includes an action to run Black on push and pull_request. -5. Before pushing new code, please make sure all files are properly formatted by running the following command in - the `/backend` directory: - ```pipenv run black app``` - -### Frontend Module - -To run the frontend, both required services and the backend module must be running successfully. - -The frontend module is developed using [TypeScript](https://www.typescriptlang.org/), [React](https://reactjs.org/), -[Material UI](https://mui.com/), [Redux](https://redux.js.org/), [webpack](https://webpack.js.org/), -[Node.js](https://nodejs.org). We recommend using Node v16.15 LTS. - -#### Install Frontend Dependencies - -1. Switch to frontend directory `cd ../frontend`. -2. Install dependencies: `npm install` - -#### Run Frontend Module - -You can run the frontend module using either of the below options: - -- Using the PyCharm's run configuration by navigating to `PyCharm -> Run -> Run...` and clicking `start:dev`. Running - directly from PyCharm helps the developer by providing easy access to its debugging features. -- From the command line by running `npm run start:dev` - - By default, the backend module runs at `http://localhost:8000`. If running at different URL/port, use: - `CLOWDER_REMOTE_HOSTNAME=http://: npm start` - - After modifying the backend module API, update autogenerated client function calls: - - Backend module must be running - - Run codegen: `npm run codegen:v2:dev` - -### Configuring Keycloak - -- If you are developer running the dev stack on your local machine, please import - the [Keycloak](https://www.keycloak.org/) realm setting file `/scripts/keycloak/clowder-realm-dev.json` -- If you are running production docker compose on local machine, please import the Keycloak realm setting file - `/scripts/keycloak/clowder-realm-prod.json` -- If you are deploying on the kubernetes cluster (https://clowder2.software-dev.ncsa.cloud/), please import the - Keycloak realm setting file `/scripts/keycloak/mini-kube-clowder-realm-prod.json` - -**For more details on how to set up Keycloak, please refer to -this [Documentation](docs/source/configure-keycloak-realm.md)** +Clowder v2 is licensed under the [Apache 2.0 license](https://github.com/clowder-framework/clowder2/blob/main/LICENSE). diff --git a/deployments/kubernetes/charts/README.md b/deployments/kubernetes/charts/README.md index 20ca465da..6ae975d09 100644 --- a/deployments/kubernetes/charts/README.md +++ b/deployments/kubernetes/charts/README.md @@ -1,6 +1,6 @@ -# Clowder 2 +# Clowder v2 Helm Charts -This depends on some subcharts, make sure to have them installed if you plan on modifying the helm chart: +Helm charts depend on some subcharts, make sure to have them installed if you plan on modifying the helm chart: ```bash helm repo add bitnami https://charts.bitnami.com/bitnami @@ -28,9 +28,9 @@ Now you can install (or upgrade) clowder using: helm upgrade --install --namespace clowder2 --create-namespace --values local.yaml clowder2 . ``` -# Docker Desktop +## Ingress Controller -You will need an ingress controller, I like Traefik as my ingress controller. You install this with: +You will need an ingress controller. Traefik works well as ingress controller. You can install it with: ```bash helm install --namespace traefik --create-namespace traefik traefik/traefik diff --git a/docs/README.md b/docs/README.md index f2d669201..ab2dbb858 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,18 +5,32 @@ Uses [Sphinx](https://www.sphinx-doc.org). Requires [enchant](https://pyenchant. Currently deployed at https://clowder2.readthedocs.io. +If you have installed `sphinx-autobuild docs/source docs/build/html` you can use it to automatically rebuild the docs +when you make changes. + +```shell +sphinx-autobuild source build/html + +open http://localhost:8000/ +``` + +If you don't have `sphinx-autobuild` installed, you can use the Makefile. + ```shell # build make html # view -python3 -m http.server --directory build/html +python3 -m http.server 7000 --directory build/html +open http://localhost:7000/ # check links make linkcheck +``` -# spell checking +You can check spelling with `make spelling`. This requires `enchant` to be installed. +```shell # install enchant once, on mac brew install enchant diff --git a/docs/requirements.txt b/docs/requirements.txt index cf1a893a6..3fe678ca6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -18,6 +18,7 @@ recommonmark==0.7.1 requests==2.27.1 snowballstemmer==2.2.0 Sphinx==4.4.0 +sphinx-autobuild==2021.3.14 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.0 diff --git a/docs/source/architecture.md b/docs/source/architecture.md new file mode 100644 index 000000000..278adfd75 --- /dev/null +++ b/docs/source/architecture.md @@ -0,0 +1,9 @@ +# Architecture + +Clowder v2 follows the architecture of v1, we the additional split of the server side into +multiple standalone containers. +Each box below is a docker service as defined in `docker-compose.yml`. +Boxes are grouped logically into backend, keycloak, extractors, database boxes. +Lines show interactions between containers over the docker network. + +![Clowder v2 architecture](img/architecture.jpg) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index b4c182935..0d1f28e60 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,12 +17,12 @@ # -- Project information ----------------------------------------------------- -project = "Clowder v2" -copyright = "2022, Luigi Marini" -author = "Luigi Marini" +project = "Clowder2" +copyright = "2022, Clowder Devs" +author = "Clowder Devs" # The full version, including alpha/beta/rc tags -release = "0.1" +release = "2.0.0-beta.1" # -- General configuration --------------------------------------------------- @@ -40,13 +40,51 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +html_theme = "sphinx_material" + +# Set link name generated in the top bar. +html_title = "Clowder v2" + +# Material theme options (see theme.conf for more information) +html_sidebars = { + "**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"] +} +html_theme_options = { + # Set the name of the project to appear in the navigation. + "nav_title": "Clowder v2", + # Set you GA account ID to enable tracking + "google_analytics_account": "UA-XXXXX", + # Specify a base_url used to generate sitemap.xml. If not + # specified, then no sitemap will be built. + "base_url": "https://clowder2.readthedocs.io/", + # Set the color and the accent color + "color_primary": "blue", + "color_accent": "light-blue", + # Set the repo location to get a badge with stats + "repo_url": "https://github.com/clowder-framework/clowder2", + "repo_name": "Clowder2", + # Visible levels of the global TOC; -1 means unlimited + "globaltoc_depth": 2, + # If False, expand all TOC entries + "globaltoc_collapse": True, + # If True, show hidden TOC entries + "globaltoc_includehidden": False, +} + +import os + +FORCE_CLASSIC = os.environ.get("SPHINX_MATERIAL_FORCE_CLASSIC", False) +FORCE_CLASSIC = FORCE_CLASSIC in ("1", "true") +if FORCE_CLASSIC: + print("!!!!!!!!! Forcing classic !!!!!!!!!!!") + html_theme = "classic" + html_theme_options = {} + html_sidebars = {"**": ["globaltoc.html", "localtoc.html", "searchbox.html"]} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/source/dev-get-started.md b/docs/source/dev-get-started.md new file mode 100644 index 000000000..f12274eb1 --- /dev/null +++ b/docs/source/dev-get-started.md @@ -0,0 +1,129 @@ +# Getting Started + +The core app is divided into two modules: backend and frontend. The frontend module is a standalone [React](https://react.dev/) +application and the backend, a standalone [FastAPI](https://fastapi.tiangolo.com/lo/) web API. We leverage [MongoDB](https://www.mongodb.com/), [RabbitMQ](https://www.rabbitmq.com/), +and [Elasticsearch](https://www.elastic.co/). We also use [MinIO](https://min.io/) out of the box as the default object store and [Traefik](https://traefik.io/traefik/) as +the web proxy. + +There are two simple ways to run Clowder v2: +1. using Docker to run the full stack in Docker using the `docker-compose.yml` +2. running required services in docker using `docker-compose.dev.yml` and running backend and frontend modules from the +command line or in your favorite IDE (to make debugging easier). + +## Running Full Stack in Docker + +To run the full stack using [Docker](https://www.docker.com/) (recommended), please use the following instructions: + +1. Run all Docker services with `docker compose up --scale backend=4 --build`. This will start the services with four + instances of the backend module running in parallel. Note the `--build` flag used to build the images first. If using + default images, that flag can be removed. The images can also be built with `docker compose build`. + +2. The application will be running and available at `http://localhost`. + +3. To access the Traefik dashboard, go to `http://localhost:8080`. To view the raw + settings, go to `http://localhost:8080/api/rawdata`. + +## Running in Development Mode + +When developing, the required services can be run using Docker. You can then run the backend +and frontend modules from the command line or in your favorite IDE (to make debugging easier). We recommend +using [PyCharm](https://www.jetbrains.com/pycharm/) and have +made our run configurations available in the `.run` folder. PyCharm should automatically import it, but you will have +to change the path to the Python virtual environment to point to your path on your host (see Initial Dependencies +section below). + +### Initial Development Dependencies + +- Run `python3 -m venv venv` to create a Python Virtual Environment and add it to PyCharm by navigating to + `PyCharm -> Settings... -> Project: clowder2 -> Python Interpreter -> Add Interpreter`. +- Run `source venv/bin/activate && pip install --upgrade pip` to activate the created Python Virtual Environment and + upgrade + pip. +- Run `pip install pipenv` to install [Pipenv](https://pipenv.pypa.io/en/latest/). + +### Required Services + +- Running `./docker-dev.sh up` brings up the required services in the background. +- Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers, + provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`. +- Running `./docker-dev.sh down` brings down the required services. + +**Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers +don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`. +If this is not used, the keycloak container will use the volume created with the other docker compose and it will be +unable to run as the information stored in the postgres database is different. + +### Backend Module + +After starting up the required services, setup and run the backend module. + +The backend module is developed using [Python](https://www.python.org/), [FastAPI](https://fastapi.tiangolo.com/), +and [Motor](https://motor.readthedocs.io/en/stable/). +We recommend using [Python 3.9](https://www.python.org/downloads/) +and Pipenv for dependency management. + +#### Install Backend Dependencies + +1. Switch to backend module directory `cd backend`. +2. Install dependencies using `pipenv install --dev`. + +#### Run Backend Module + +You can run the backend module using either of the below options: + +- Using the PyCharm's run configuration by navigating to `PyCharm -> Run -> Run...` and clicking `uvicorn`. Running + directly from PyCharm helps the developer by providing easy access to its debugging features. +- From the command line by running `pipenv run uvicorn app.main:app --reload` . + +Additional steps/details: + +1. API docs are available at `http://localhost:8000/docs`. The API base URL is `http://localhost:8000/api/v2`. +2. Create a user using `POST /api/v2/users` and getting a JWT token by using `POST /api/v2/login`. Place the token in + header of requests that require authentications using the `Authorization: Bearer ` HTTP header. + * You can also run the frontend module below and use the Login link available there. +3. Manually run tests before pushing with `pipenv run pytest -v` or right-clicking on `test` folder and clicking `Run` + in PyCharm. +4. Linting is done using [Black]((https://black.readthedocs.io/en/stable/)). You can set up PyCharm to automatically + run it when you save a file using + these [instructions](https://black.readthedocs.io/en/stable/integrations/editors.html). + The git repository includes an action to run Black on push and pull_request. +5. Before pushing new code, please make sure all files are properly formatted by running the following command in + the `/backend` directory: + ```pipenv run black app``` + +### Frontend Module + +To run the frontend, both required services and the backend module must be running successfully. + +The frontend module is developed using [TypeScript](https://www.typescriptlang.org/), [React](https://reactjs.org/), +[Material UI](https://mui.com/), [Redux](https://redux.js.org/), [webpack](https://webpack.js.org/), +[Node.js](https://nodejs.org). We recommend using Node v16.15 LTS. + +#### Install Frontend Dependencies + +1. Switch to frontend directory `cd ../frontend`. +2. Install dependencies: `npm install` + +#### Run Frontend Module + +You can run the frontend module using either of the below options: + +- Using the PyCharm's run configuration by navigating to `PyCharm -> Run -> Run...` and clicking `start:dev`. Running + directly from PyCharm helps the developer by providing easy access to its debugging features. +- From the command line by running `npm run start:dev` + - By default, the backend module runs at `http://localhost:8000`. If running at different URL/port, use: + `CLOWDER_REMOTE_HOSTNAME=http://: npm start` + - After modifying the backend module API, update autogenerated client function calls: + - Backend module must be running + - Run codegen: `npm run codegen:v2:dev` + +### Configuring Keycloak + +- If you are developer running the dev stack on your local machine, please import + the [Keycloak](https://www.keycloak.org/) realm setting file `/scripts/keycloak/clowder-realm-dev.json` +- If you are running production docker compose on local machine, please import the Keycloak realm setting file + `/scripts/keycloak/clowder-realm-prod.json` +- If you are deploying on the kubernetes cluster (https://clowder2.software-dev.ncsa.cloud/), please import the + Keycloak realm setting file `/scripts/keycloak/mini-kube-clowder-realm-prod.json` + +For more details on how to set up Keycloak, please refer to this [Documentation](docs/source/configure-keycloak-realm.md) \ No newline at end of file diff --git a/docs/source/elasticsearch.md b/docs/source/elasticsearch.md index 3301f20ed..855a42557 100644 --- a/docs/source/elasticsearch.md +++ b/docs/source/elasticsearch.md @@ -1,18 +1,18 @@ -# Elasticsearch Integration Notes +# Elasticsearch -For a brief introduction to elasticsearch, please refer to https://www.elastic.co/guide/en/app-search/current/getting-started.html +For a brief introduction to Elasticsearch, please take a look [here https://www.elastic.co/guide/en/app-search/current/getting-started.html -For some examples in python, please refer to to https://www.elastic.co/guide/en/elasticsearch/client/python-api/master/examples.html +For some examples in Python, please refer to https://www.elastic.co/guide/en/elasticsearch/client/python-api/master/examples.html -We are running elasticsearch inside docker at port 9200. Once the elasticsearch docker container is up and running, -you should be able to connect to http://localhost:9200 and see something like below +We are running Elasticsearch inside Docker at port 9200. Once the elasticsearch docker container is up and running, +you should be able to connect to http://localhost:9200 and see something like below: ![Connect to elasticsearch](img/elasticsearch1.png) -If you see error in elasticsearch docker container and that is related to heap memory, try upgrading memory to +If you see error in Elasticsearch docker container and that is related to heap memory, try upgrading memory to 8Gb in docker setting. -For all the implemented elasticsearch APIs, please refere to the code in backend/app/elasticsearch/connect.py +For all the implemented elasticsearch APIs, please refer to the code in `backend/app/elasticsearch/connect.py`. ## Common Commands diff --git a/docs/source/img/architecture.jpg b/docs/source/img/architecture.jpg new file mode 100644 index 000000000..a294bbd8f Binary files /dev/null and b/docs/source/img/architecture.jpg differ diff --git a/docs/source/img/clowder-collaborators.jpg b/docs/source/img/clowder-collaborators.jpg new file mode 100644 index 000000000..18a4ffeb5 Binary files /dev/null and b/docs/source/img/clowder-collaborators.jpg differ diff --git a/docs/source/img/logo.png b/docs/source/img/logo.png new file mode 100644 index 000000000..173c4e0af Binary files /dev/null and b/docs/source/img/logo.png differ diff --git a/docs/source/img/logo_full.png b/docs/source/img/logo_full.png new file mode 100644 index 000000000..0ecf1c841 Binary files /dev/null and b/docs/source/img/logo_full.png differ diff --git a/backend/docs/source/img/metadata_dropdown.png b/docs/source/img/metadata_dropdown.png similarity index 100% rename from backend/docs/source/img/metadata_dropdown.png rename to docs/source/img/metadata_dropdown.png diff --git a/backend/docs/source/img/multiple_widgets.png b/docs/source/img/multiple_widgets.png similarity index 100% rename from backend/docs/source/img/multiple_widgets.png rename to docs/source/img/multiple_widgets.png diff --git a/docs/source/img/visualizations.jpg b/docs/source/img/visualizations.jpg new file mode 100644 index 000000000..a20112f67 Binary files /dev/null and b/docs/source/img/visualizations.jpg differ diff --git a/docs/source/index.rst b/docs/source/index.rst index 26c29201a..f5f1fcae1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,21 +3,44 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Clowder v2's documentation! +Clowder v2 ====================================== +`Clowder `_ is a cloud native data management framework to support any research domain. Clowder was developed to help +researchers and scientists in data intensive domains manage raw data, complex metadata, and automatic data pipelines. +Clowder is not a central system, but software that research labs and individual users can install on local clusters or +in the cloud. Clowder is data agnostic, but can be customized for specific data types through a sophisticated +information extraction bus and web based visualization plugins. Clowder is also a community of practitioners working +on data intensive research projects and developing tools that work across research domains. + +.. image:: img/clowder-collaborators.jpg + :alt: Clowder collaborators and research domains + +The first version of Clowder (`v1 `_) has been in development for over ten years and is currently being used in a variety +of `use cases `_. A new version of Clowder (`v2 `_) is currently in development. v2 builds on lessons learned from current and +prior use cases and is being developed with two major goals in mind: 1) a modernized web interface for end users 2) an +accessible and easy to maintain code base for contributors. + +We are always looking for new collaborators, contributors, and ideas in the space of research data management. If you +are interested in contributing to Clowder v2 please come say hi in `Slack `_. If you are ready to dive into the code, take +a look at `first time issues `_ in v2. + .. toctree:: - :maxdepth: 2 - :caption: Contents: + :maxdepth: 1 + :caption: Developers + dev-get-started.md + architecture.md mongo-fastapi.md pycharm.md + elasticsearch.md + configure-keycloak-realm.md + listeners.md + mongo-fastapi.md +.. toctree:: + :maxdepth: 1 + :caption: Users - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` + metadata.md + visualizations.md \ No newline at end of file diff --git a/backend/docs/source/metadata.md b/docs/source/metadata.md similarity index 96% rename from backend/docs/source/metadata.md rename to docs/source/metadata.md index 1e063e5d8..71edbbb6e 100644 --- a/backend/docs/source/metadata.md +++ b/docs/source/metadata.md @@ -1,7 +1,7 @@ -# Metadata in Clowder +# Metadata In Clowder 2 we decide to continue supporting both user-defined and machine-defined metadata in a flexible -representation based on JSON-LD. We enforce that, to register a new metadata, either the definition +representation based on [JSON-LD](https://json-ld.org/). We enforce that, to register a new metadata, either the definition or a context URL that contains the schema of given vocabulary needs to be provided. Leveraging the React Component, we would like to explore dynamic rendering as well as the reusability of the metadata user interface. Our metadata definition are designed to facilitate that process. diff --git a/docs/source/mongo-fastapi.md b/docs/source/mongo-fastapi.md index 5cdb61730..9037e2a13 100644 --- a/docs/source/mongo-fastapi.md +++ b/docs/source/mongo-fastapi.md @@ -1,22 +1,13 @@ # MongoDB FastAPI Integration Notes -Mongo ObjectId -https://github.com/tiangolo/fastapi/issues/1515 - -ODM -https://github.com/art049/odmantic/ - -pymongo and fastapi -https://medium.com/fastapi-tutorials/integrating-fastapi-and-mongodb-8ef4f2ca68ad - -mongoengine and fastapi -https://stackoverflow.com/questions/60277170/how-to-convert-mongoengine-class-in-pedantic-basemodel-in-python-fastapi - -pydantic and objectid -https://stackoverflow.com/questions/59503461/how-to-parse-objectid-in-a-pydantic-model - -mongoengine and fastapi -https://stackoverflow.com/questions/60277170/how-to-convert-mongoengine-class-in-pedantic-basemodel-in-python-fastapi - -beanie odm -https://github.com/roman-right/beanie \ No newline at end of file +We ended up using [beanie](https://github.com/roman-right/beanie) odm and +[motor](https://motor.readthedocs.io) to interact with MongoDB. +Below are links to some resources consulted during the development of this integration. +Some are not relevant anymore and are here just for reference. + +* [Mongo ObjectId](https://github.com/tiangolo/fastapi/issues/1515) +* [ODM](https://github.com/art049/odmantic/) +* [pymongo and fastapi](https://medium.com/fastapi-tutorials/integrating-fastapi-and-mongodb-8ef4f2ca68ad) +* [mongoengine and fastapi](https://stackoverflow.com/questions/60277170/how-to-convert-mongoengine-class-in-pedantic-basemodel-in-python-fastapi) +* [pydantic and objectid](https://stackoverflow.com/questions/59503461/how-to-parse-objectid-in-a-pydantic-model) +* [mongoengine and fastapi](https://stackoverflow.com/questions/60277170/how-to-convert-mongoengine-class-in-pedantic-basemodel-in-python-fastap) \ No newline at end of file diff --git a/docs/source/pycharm.md b/docs/source/pycharm.md index c9becf1ff..1d751dd38 100644 --- a/docs/source/pycharm.md +++ b/docs/source/pycharm.md @@ -1,4 +1,4 @@ -### How to setup pycharm to work with Clowder V2 +# Pycharm Setup From the main menu select Run -> Edit Configurations... diff --git a/backend/docs/source/visualizations.md b/docs/source/visualizations.md similarity index 89% rename from backend/docs/source/visualizations.md rename to docs/source/visualizations.md index 8ce5f8f61..ede09a5ef 100644 --- a/backend/docs/source/visualizations.md +++ b/docs/source/visualizations.md @@ -1,4 +1,4 @@ -# Visualizations in Clowder +# Visualizations When browsing Clowder, there are two primary ways users can see aspects of a file or dataset without downloading it: @@ -20,7 +20,7 @@ To programatically add a thumbnail to a file or dataset: 3. Retrieve the thumbnail (e.g. to render on a frontend GUI) by issuing a GET request to `api/v2/files/:file_id/thumbnail`. -The `ncsa.image.preview` extractor found in https://github.com/clowder-framework/extractors-core will create file +The `ncsa.image.preview` extractor found in the [core extractors](https://github.com/clowder-framework/extractors-core) will create file thumbnails automatically. ## Visualizations diff --git a/scripts/metadata/definitions/alternativeTitle.json b/scripts/metadata/definitions/alternativeTitle.json index 8fde1148d..cd281c9fd 100644 --- a/scripts/metadata/definitions/alternativeTitle.json +++ b/scripts/metadata/definitions/alternativeTitle.json @@ -14,7 +14,7 @@ "config": { "type" : "str" }, - "required" : true + "required" : false } ] } \ No newline at end of file diff --git a/scripts/metadata/definitions/coordinates.json b/scripts/metadata/definitions/coordinates.json index 920e63a6c..476e4591e 100644 --- a/scripts/metadata/definitions/coordinates.json +++ b/scripts/metadata/definitions/coordinates.json @@ -14,7 +14,7 @@ "config": { "type" : "tuple" }, - "required" : true + "required" : false } ] } \ No newline at end of file diff --git a/scripts/metadata/definitions/doi.json b/scripts/metadata/definitions/doi.json index 9ad476d1e..a43633214 100644 --- a/scripts/metadata/definitions/doi.json +++ b/scripts/metadata/definitions/doi.json @@ -14,7 +14,7 @@ "config": { "type" : "str" }, - "required" : true + "required" : false } ] } \ No newline at end of file diff --git a/scripts/metadata/definitions/latLon.json b/scripts/metadata/definitions/latLon.json index fe9c39ee0..cf6a36825 100644 --- a/scripts/metadata/definitions/latLon.json +++ b/scripts/metadata/definitions/latLon.json @@ -15,7 +15,7 @@ "config": { "type" : "float" }, - "required" : true + "required" : false }, { "name" : "latitude", @@ -24,7 +24,7 @@ "config": { "type" : "float" }, - "required" : true + "required" : false } ] } \ No newline at end of file diff --git a/scripts/metadata/definitions/time.json b/scripts/metadata/definitions/time.json index cb4c03d27..177016008 100644 --- a/scripts/metadata/definitions/time.json +++ b/scripts/metadata/definitions/time.json @@ -14,7 +14,7 @@ "config": { "type" : "str" }, - "required" : true + "required" : false } ] } \ No newline at end of file diff --git a/scripts/metadata/definitions/unit.json b/scripts/metadata/definitions/unit.json index 323fd56ca..b569a7ba2 100644 --- a/scripts/metadata/definitions/unit.json +++ b/scripts/metadata/definitions/unit.json @@ -15,7 +15,7 @@ "type" : "enum", "options": ["Ampere", "Kelvin", "Second"] }, - "required" : true + "required" : false } ] } \ No newline at end of file