From 0a6e59ed750582be7dcf846ee13fc2e0a5907a04 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 12 Nov 2020 11:47:55 -0800 Subject: [PATCH 1/8] update readme --- README.md | 13 ++----------- example/README.md | 34 +++++++++++----------------------- 2 files changed, 13 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 26573dee3..3ba39ef2e 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,9 @@ The MMS SDVC is a collection of modules built on top of the Spring Framework and is a part of Open-MBEE. For more information about Open-MBEE, visit the [Open-MBEE Website](https://openmbee.org/) -For a reference on how to create an application, see [MMSRI](https://github.com/Open-MBEE/mmsri) or the "example" package. Also see README inside each module. +#### If you are interested in deploying MMS, please see the [MMSRI](https://github.com/Open-MBEE/mmsri) quickstart. -## Quick Start -### Docker -Installation instructions are found here: [Docker documentation](https://docs.docker.com/) - -1. Copy the example properties file in `example/src/main/resources/` as `application.properties` -1. In the command line, run `docker-compose up --build` to create and start all the services from the configuration. -1. Swagger ui at [http://localhost:8080/v3/swagger-ui.html](http://localhost:8080/v3/swagger-ui.html) -1. Use the command `docker-compose down` to stop any containers from running and to remove the containers, networks, and images created by the `docker-compose up` command. This command should always be done before any new attempts to restart the services from the configuration. - -## Developer Setup for example project +## Developer Setup ### Docker We suggest using Docker to set up PostgreSQL and Elasticsearch. Installation instructions are found here: [Docker documentation](https://docs.docker.com/) diff --git a/example/README.md b/example/README.md index 61431ee18..8a9466acc 100644 --- a/example/README.md +++ b/example/README.md @@ -1,6 +1,15 @@ +## Quick Start +### Docker +Installation instructions are found here: [Docker documentation](https://docs.docker.com/) + +1. At repo's root level, run `docker-compose up --build` to create and start all the services from the configuration. This uses the `test` Spring profile +1. Swagger ui at [http://localhost:8080/v3/swagger-ui.html](http://localhost:8080/v3/swagger-ui.html) +1. Use the command `docker-compose down` to stop any containers from running and to remove the containers, networks, and images created by the `docker-compose up` command. This command should always be done before any new attempts to restart the services from the configuration. + + ## Run command line api test -1. run example app on localhost (see top level readme) +1. run example app on localhost 1. if using docker-compose to bring up example app, can do the following to run the tests instead of installing node docker run -v $PWD:/etc/newman -t --network container:mms postman/newman:alpine run crud.postman_collection.json --environment="test-env.json" --delay-request=300 @@ -12,27 +21,6 @@ newman run crud.postman_collection.json -e localhost-env.json --delay-request 300 -## Swagger codegen - -[Gradle Plugin](https://github.com/int128/gradle-swagger-generator-plugin) - - ../gradlew generateSwaggerCode - -Results in build/swagger-code-* - ## Swagger UI of running app -Swagger 3 UI at http://localhost:8080/v3/swagger-ui.html (not dependent on gradle codegen) - -yaml at http://localhost:8080/v3/api-docs.yaml - will need to fix securitySchemes for codegen - -### Fixes needed from generated yaml to use in codegen - -- add basic auth security scheme - - basicAuth: - type: http - scheme: basic - -- change schema of ElementJson to just `type: object` (otherwise client generation ignores `additionalProperties` unless using `python-experimental` which needs other changes) -- change `addtionalProperties` in openapi spec to be just `addtionalProperties: true` where it appears +Swagger 3 UI at http://localhost:8080/v3/swagger-ui.html \ No newline at end of file From 44a394d9117aa316caee9f786422be3f1d26b792 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 2 Dec 2020 12:42:16 -0800 Subject: [PATCH 2/8] chunk elastic gets --- .../mms/crud/services/DefaultNodeService.java | 4 +- .../mms/elastic/BaseElasticDAOImpl.java | 37 +++++++++++++------ .../resources/application-test.properties | 1 + .../resources/application.properties.example | 1 + 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 81ec448fc..a823df6ca 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -168,9 +168,9 @@ protected void commitChanges(NodeChangeInfo info) { commit.setDocId(cmjs.getId()); commit.setTimestamp(now); commit.setComment(cmjs.getComment()); - - this.commitIndex.index(cmjs); + this.commitRepository.save(commit); + this.commitIndex.index(cmjs); this.nodeRepository.getTransactionManager().commit(status); } catch (Exception e) { logger.error("commitChanges error: ", e); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index cc870b4d8..b33a70d26 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -46,6 +46,8 @@ public abstract class BaseElasticDAOImpl> { protected int resultLimit; @Value("${elasticsearch.limit.term}") protected int termLimit; + @Value("${elasticsearch.limit.get}") + protected int getLimit; protected static int readTimeout = 1000000000; protected RestHighLevelClient client; private static final RequestOptions REQUEST_OPTIONS; @@ -121,28 +123,41 @@ public List findAllById(String index, Set docIds) { if (docIds.isEmpty()) { return listOfResponses; } + int cur = 0; MultiGetRequest request = new MultiGetRequest(); for (String eid : docIds) { request.add(index, eid); - } - MultiGetResponse response = client.mget(request, REQUEST_OPTIONS); - - for (MultiGetItemResponse res : response.getResponses()) { - GetResponse item = res.getResponse(); - if (item != null && item.isExists()) { - E ob = newInstance(); - ob.putAll(item.getSourceAsMap()); - listOfResponses.add(ob); - } else { - continue; + cur++; + if (cur == getLimit) { + getResponses(request, listOfResponses); + cur = 0; + request = new MultiGetRequest(); } } + if (cur > 0) { + getResponses(request, listOfResponses); + } return listOfResponses; } catch (IOException e) { throw new RuntimeException(e); } } + private void getResponses(MultiGetRequest request, List listOfResponses) throws IOException { + MultiGetResponse response = client.mget(request, REQUEST_OPTIONS); + + for (MultiGetItemResponse res : response.getResponses()) { + GetResponse item = res.getResponse(); + if (item != null && item.isExists()) { + E ob = newInstance(); + ob.putAll(item.getSourceAsMap()); + listOfResponses.add(ob); + } else { + continue; + } + } + } + public void indexAll(String index, Collection jsons) { BulkProcessor bulkProcessor = getBulkProcessor(client); for (BaseJson json : jsons) { diff --git a/example/src/main/resources/application-test.properties b/example/src/main/resources/application-test.properties index 692a63551..2cb508f36 100644 --- a/example/src/main/resources/application-test.properties +++ b/example/src/main/resources/application-test.properties @@ -51,6 +51,7 @@ elasticsearch.limit.insert=80 elasticsearch.limit.result=10000 elasticsearch.limit.term=1000 elasticsearch.limit.scrollTimeout=1000 +elasticsearch.limit.get=10000 #Configuration for TWC #port is for REST interface diff --git a/example/src/main/resources/application.properties.example b/example/src/main/resources/application.properties.example index b0a3ee78c..6d0d11e24 100644 --- a/example/src/main/resources/application.properties.example +++ b/example/src/main/resources/application.properties.example @@ -49,6 +49,7 @@ elasticsearch.limit.insert=80 elasticsearch.limit.result=10000 elasticsearch.limit.term=1000 elasticsearch.limit.scrollTimeout=1000 +elasticsearch.limit.get=100000 #Configuration for TWC #port is for REST interface From f1fde45f4e841edc58c6a50b0996434bb56546b6 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 2 Dec 2020 13:35:53 -0800 Subject: [PATCH 3/8] repo order --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a4ddf2f8b..7c02a7ca2 100644 --- a/build.gradle +++ b/build.gradle @@ -50,9 +50,9 @@ ext.milestoneBuild = !(snapshotBuild || releaseBuild) subprojects { repositories { - maven { url 'https://repo.spring.io/plugins-release' } - gradlePluginPortal() mavenCentral() + gradlePluginPortal() + maven { url 'https://repo.spring.io/plugins-release' } } plugins.withType(JavaPlugin) { From b46cb3ce2f4874e06993d05f81f16fb180f69c18 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 3 Dec 2020 10:52:56 -0800 Subject: [PATCH 4/8] add get users --- .../openmbee/mms/data/domains/global/Base.java | 2 +- .../controllers/LocalUserController.java | 10 ++++++++++ .../security/UserDetailsServiceImpl.java | 5 +++++ .../mms/localuser/security/UsersResponse.java | 17 +++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 localuser/src/main/java/org/openmbee/mms/localuser/security/UsersResponse.java diff --git a/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java b/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java index 3adaff6a2..f4f372c77 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java @@ -15,7 +15,7 @@ import org.springframework.data.annotation.LastModifiedDate; @MappedSuperclass -@JsonIgnoreProperties(value = {"created", "modified"}, allowGetters = true) +@JsonIgnoreProperties(value = {"created", "modified", "id"}) public abstract class Base implements Serializable { private static final long serialVersionUID = 8389104517465359723L; diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java b/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java index 524b44a9f..7627cbeae 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java @@ -8,11 +8,13 @@ import org.openmbee.mms.core.utils.AuthenticationUtils; import org.openmbee.mms.localuser.security.UserCreateRequest; import org.openmbee.mms.localuser.security.UserDetailsServiceImpl; +import org.openmbee.mms.localuser.security.UsersResponse; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.Authentication; import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @@ -40,6 +42,14 @@ public UserCreateRequest createUser(@RequestBody UserCreateRequest req) { throw new BadRequestException("User already exists"); } + @GetMapping(value = "/users") + @PreAuthorize(AuthorizationConstants.IS_MMSADMIN) + public UsersResponse getUsers() { + UsersResponse res = new UsersResponse(); + res.setUsers(userDetailsService.getUsers()); + return res; + } + @PostMapping(value = "/password", consumes = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("isAuthenticated()") public Object updatePassword(@RequestBody UserCreateRequest req, diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java index baef8bf6b..e8c560ee3 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java @@ -1,5 +1,6 @@ package org.openmbee.mms.localuser.security; +import java.util.List; import java.util.Optional; import org.openmbee.mms.core.exceptions.ForbiddenException; @@ -39,6 +40,10 @@ public UserDetailsImpl loadUserByUsername(String username) throws UsernameNotFou return new UserDetailsImpl(user.get()); } + public List getUsers() { + return userRepository.findAll(); + } + @Transactional public User register(String username, String password, boolean isAdmin) { User user = new User(); diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/security/UsersResponse.java b/localuser/src/main/java/org/openmbee/mms/localuser/security/UsersResponse.java new file mode 100644 index 000000000..f119e7cf8 --- /dev/null +++ b/localuser/src/main/java/org/openmbee/mms/localuser/security/UsersResponse.java @@ -0,0 +1,17 @@ +package org.openmbee.mms.localuser.security; + +import java.util.List; +import org.openmbee.mms.data.domains.global.User; + +public class UsersResponse { + + private List users; + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users; + } +} From f5646f8f56fde753b1e14929dab69fc865ffca56 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 3 Dec 2020 11:01:20 -0800 Subject: [PATCH 5/8] fill out rest of user info on create user --- .../controllers/LocalUserController.java | 2 +- .../localuser/security/UserCreateRequest.java | 27 +++++++++++++++++++ .../security/UserDetailsServiceImpl.java | 11 +++++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java b/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java index 7627cbeae..eda199b8e 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/controllers/LocalUserController.java @@ -36,7 +36,7 @@ public UserCreateRequest createUser(@RequestBody UserCreateRequest req) { try { userDetailsService.loadUserByUsername(req.getUsername()); } catch (UsernameNotFoundException e) { - userDetailsService.register(req.getUsername(), req.getPassword(), req.isAdmin()); + userDetailsService.register(req); return req; } throw new BadRequestException("User already exists"); diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserCreateRequest.java b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserCreateRequest.java index 48e50111b..485096140 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserCreateRequest.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserCreateRequest.java @@ -8,8 +8,35 @@ public class UserCreateRequest implements Serializable { private String username; private String password; + private String email; + private String firstname; + private String lastname; private boolean admin; + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getFirstname() { + return firstname; + } + + public void setFirstname(String firstname) { + this.firstname = firstname; + } + + public String getLastname() { + return lastname; + } + + public void setLastname(String lastname) { + this.lastname = lastname; + } + public String getUsername() { return this.username; } diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java index e8c560ee3..aaa4c13e9 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/security/UserDetailsServiceImpl.java @@ -45,12 +45,15 @@ public List getUsers() { } @Transactional - public User register(String username, String password, boolean isAdmin) { + public User register(UserCreateRequest req) { User user = new User(); - user.setUsername(username); - user.setPassword(passwordEncoder.encode(password)); + user.setEmail(req.getEmail()); + user.setFirstName(req.getFirstname()); + user.setLastName(req.getLastname()); + user.setUsername(req.getUsername()); + user.setPassword(passwordEncoder.encode(req.getPassword())); user.setEnabled(true); - user.setAdmin(isAdmin); + user.setAdmin(req.isAdmin()); return userRepository.save(user); } From e2fc643638a4678554a96e74eb90f8069a1439e9 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 3 Dec 2020 11:09:46 -0800 Subject: [PATCH 6/8] fix --- .../openmbee/mms/localuser/config/AuthProviderConfig.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java b/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java index 8621ffaed..5116e9a3f 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java @@ -1,5 +1,6 @@ package org.openmbee.mms.localuser.config; +import org.openmbee.mms.localuser.security.UserCreateRequest; import org.openmbee.mms.localuser.security.UserDetailsServiceImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +40,11 @@ public DaoAuthenticationProvider daoAuthenticationProvider() { try { userDetailsService.loadUserByUsername(adminUsername); } catch (UsernameNotFoundException e) { - userDetailsService.register(adminUsername, adminPassword, true); + UserCreateRequest req = new UserCreateRequest(); + req.setAdmin(true); + req.setPassword(adminPassword); + req.setUsername(adminUsername); + userDetailsService.register(req); logger.info(String.format("Creating root user: %s with specified password.", adminUsername)); } From 4b4ae4086901a26f541891b00fd59f2794d4c826 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 3 Dec 2020 11:26:42 -0800 Subject: [PATCH 7/8] fix test --- .../main/java/org/openmbee/mms/data/domains/global/Base.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java b/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java index f4f372c77..1d7c2abd2 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/global/Base.java @@ -15,7 +15,7 @@ import org.springframework.data.annotation.LastModifiedDate; @MappedSuperclass -@JsonIgnoreProperties(value = {"created", "modified", "id"}) +@JsonIgnoreProperties(value = {"created", "modified", "id"}, allowGetters = true) public abstract class Base implements Serializable { private static final long serialVersionUID = 8389104517465359723L; From a205f8d6681466bf18edbcbb29a8c54aca37859a Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 3 Dec 2020 12:07:21 -0800 Subject: [PATCH 8/8] Bump version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 54c95dcf2..40226acdf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0 +version=4.0.0-b3 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE