Adding filters to RSD Library

api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt

@@ -13,6 +13,7 @@ object RoutePaths {
     const val SKILLS_PATH = "$API/skills"
     const val SKILLS_LIST = SKILLS_PATH
     const val SKILLS_CREATE = SKILLS_PATH
+    const val SKILLS_FILTER = "$SKILLS_PATH/filter"
     const val SKILL_PUBLISH = "$SKILLS_PATH/publish"
     const val SKILL_DETAIL = "$SKILLS_PATH/{uuid}"
     const val SKILL_UPDATE = "$SKILL_DETAIL/update"

api/src/main/kotlin/edu/wgu/osmt/api/model/ApiSearch.kt

@@ -11,6 +11,9 @@ data class ApiSearch(
     @JsonProperty("advanced")
     val advanced: ApiAdvancedSearch? = null,
 
+    @JsonProperty("filtered")
+    val filtered: ApiFilteredSearch? = null,
+
     @JsonProperty("uuids")
     val uuids: List<String>? = null
 )
@@ -58,3 +61,34 @@ data class ApiSkillListUpdate(
     @JsonProperty("remove")
     val remove: ApiSearch? = null
 )
+
+@JsonInclude(JsonInclude.Include.ALWAYS)
+data class ApiFilteredSearch(
+
+    @JsonProperty("categories")
+    val categories: List<String>? = null,
+
+    @JsonProperty("keywords")
+    val keywords: List<String>? = null,
+
+    @JsonProperty("standards")
+    val standards: List<String>? = null,
+
+    @JsonProperty("certifications")
+    val certifications: List<String>? = null,
+
+    @JsonProperty("alignments")
+    val alignments: List<String>? = null,
+
+    @JsonProperty("jobcodes")
+    val jobCodes: List<String>? = null,
+
+    @JsonProperty("employers")
+    val employers: List<String>? = null,
+
+    @JsonProperty("authors")
+    val authors: List<String>? = null,
+
+    @JsonProperty("occupations")
+    val occupations: List<String>? = null
+)

api/src/main/kotlin/edu/wgu/osmt/config/Constants.kt

@@ -18,6 +18,7 @@ const val INDEX_JOBCODE_DOC = "jobcode_v1"
 const val INDEX_KEYWORD_DOC = "keyword"
 
 // ElasticSearch Sort Criteria
-const val NAME_SORT_INSENSITIVE = "name.sort_insensitive"
-const val CATEGORY_SORT_INSENSITIVE = "category.sort_insensitive"
+const val SORT_INSENSITIVE = ".sort_insensitive"
+const val NAME_SORT_INSENSITIVE = "name${SORT_INSENSITIVE}"
+const val CATEGORY_SORT_INSENSITIVE = "category${SORT_INSENSITIVE}"
 

api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt

@@ -6,6 +6,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
 import org.elasticsearch.index.query.Operator
 import org.elasticsearch.index.query.QueryBuilders.*
 import org.elasticsearch.search.sort.SortBuilders
+import org.elasticsearch.search.sort.SortOrder
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Configuration
 import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate
@@ -28,13 +29,18 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea
     CustomJobCodeRepository {
 
     override fun typeAheadSearch(query: String): SearchHits<JobCode> {
-        val limitedPageable = OffsetPageable(0, 10, null)
-        val disjunctionQuery = JobCodeQueries.multiPropertySearch(query)
+        val nsq: NativeSearchQueryBuilder
 
-        val nsq: NativeSearchQueryBuilder =
-            NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery)
-                .withSort(SortBuilders.scoreSort())
+        val limitedPageable: OffsetPageable = if (query.isEmpty()) {
+            OffsetPageable(0, 10000, null)
+        } else {
+            OffsetPageable(0, 20, null)
 
+        }
+        val disjunctionQuery = JobCodeQueries.multiPropertySearch(query)
+        nsq =
+            NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery)
+                .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC))
         return elasticSearchTemplate.search(nsq.build(), JobCode::class.java)
     }
 }

api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt

@@ -1,13 +1,23 @@
 package edu.wgu.osmt.keyword
 
 import edu.wgu.osmt.config.INDEX_KEYWORD_DOC
-import edu.wgu.osmt.db.*
+import edu.wgu.osmt.db.DatabaseData
+import edu.wgu.osmt.db.HasUpdateDate
+import edu.wgu.osmt.db.NullableFieldUpdate
+import edu.wgu.osmt.db.TableWithUpdate
+import edu.wgu.osmt.db.UpdateObject
 import org.elasticsearch.core.Nullable
 import org.jetbrains.exposed.dao.id.LongIdTable
 import org.jetbrains.exposed.sql.Column
 import org.jetbrains.exposed.sql.`java-time`.datetime
 import org.springframework.data.annotation.Id
-import org.springframework.data.elasticsearch.annotations.*
+import org.springframework.data.elasticsearch.annotations.DateFormat
+import org.springframework.data.elasticsearch.annotations.Document
+import org.springframework.data.elasticsearch.annotations.Field
+import org.springframework.data.elasticsearch.annotations.FieldType
+import org.springframework.data.elasticsearch.annotations.InnerField
+import org.springframework.data.elasticsearch.annotations.MultiField
+import org.springframework.data.elasticsearch.annotations.Setting
 import java.time.LocalDateTime
 
 @Document(indexName = INDEX_KEYWORD_DOC, createIndex = true)
@@ -32,7 +42,8 @@ data class Keyword(
         otherFields = [
             InnerField(suffix = "", type = FieldType.Search_As_You_Type),
             InnerField(suffix = "raw", analyzer = "whitespace_exact", type = FieldType.Text),
-            InnerField(suffix = "keyword", type = FieldType.Keyword)
+            InnerField(suffix = "keyword", type = FieldType.Keyword),
+            InnerField(suffix = "sort_insensitive", type = FieldType.Keyword, normalizer = "lowercase_normalizer")
         ]
     )
     val value: String? = null,

api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt

@@ -1,9 +1,11 @@
 package edu.wgu.osmt.keyword
 
 import edu.wgu.osmt.config.INDEX_KEYWORD_DOC
+import edu.wgu.osmt.config.SORT_INSENSITIVE
 import edu.wgu.osmt.elasticsearch.OffsetPageable
 import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.search.sort.SortBuilders
+import org.elasticsearch.search.sort.SortOrder
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Configuration
 import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate
@@ -25,26 +27,39 @@ interface CustomKeywordRepository {
 class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) :
     CustomKeywordRepository {
     override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits<Keyword> {
-        val limitedPageable = OffsetPageable(0, 10, null)
+        val limitedPageable: OffsetPageable
         val bq = QueryBuilders.boolQuery()
+        val nsq: NativeSearchQueryBuilder
 
-        val nsq: NativeSearchQueryBuilder =
-            NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq).withSort(SortBuilders.scoreSort())
-
-        bq
-            .must(QueryBuilders.termQuery(Keyword::type.name, type.name))
-            .should(
-                QueryBuilders.matchBoolPrefixQuery(
-                    Keyword::value.name,
-                    query
+        if(query.isEmpty()){ //retrieve all
+            limitedPageable = OffsetPageable(0, 10000, null)
+            nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq)
+                .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC))
+            bq
+                .must(QueryBuilders.termQuery(Keyword::type.name, type.name))
+                .should(
+                    QueryBuilders.matchAllQuery()
+                )
+        }
+        else {
+            limitedPageable  = OffsetPageable(0, 20, null)
+            nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq)
+                .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC))
+            bq
+                .must(QueryBuilders.termQuery(Keyword::type.name, type.name))
+                .should(
+                    QueryBuilders.matchBoolPrefixQuery(
+                        Keyword::value.name,
+                        query
+                    )
                 )
-            )
-            .should(
-                QueryBuilders.matchPhraseQuery(
-                    Keyword::value.name,
-                    query
-                ).boost(5f)
-            ).minimumShouldMatch(1)
+                .should(
+                    QueryBuilders.matchPhraseQuery(
+                        Keyword::value.name,
+                        query
+                    ).boost(5f)
+                ).minimumShouldMatch(1)
+        }
 
         return elasticSearchTemplate.search(nsq.build(), Keyword::class.java)
     }

api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt

@@ -2,17 +2,20 @@ package edu.wgu.osmt.richskill
 
 import edu.wgu.osmt.HasAllPaginated
 import edu.wgu.osmt.RoutePaths
+import edu.wgu.osmt.RoutePaths.SKILLS_FILTER
 import edu.wgu.osmt.api.GeneralApiException
 import edu.wgu.osmt.api.model.ApiSearch
 import edu.wgu.osmt.api.model.ApiSkill
 import edu.wgu.osmt.api.model.ApiSkillUpdate
 import edu.wgu.osmt.api.model.SkillSortEnum
+import edu.wgu.osmt.api.model.SortOrder
 import edu.wgu.osmt.auditlog.AuditLog
 import edu.wgu.osmt.auditlog.AuditLogRepository
 import edu.wgu.osmt.auditlog.AuditLogSortEnum
 import edu.wgu.osmt.config.AppConfig
 import edu.wgu.osmt.db.PublishStatus
 import edu.wgu.osmt.elasticsearch.OffsetPageable
+import edu.wgu.osmt.elasticsearch.PaginatedLinks
 import edu.wgu.osmt.keyword.KeywordDao
 import edu.wgu.osmt.security.OAuthHelper
 import edu.wgu.osmt.task.AppliesToType
@@ -23,6 +26,7 @@ import edu.wgu.osmt.task.PublishTask
 import edu.wgu.osmt.task.Task
 import edu.wgu.osmt.task.TaskMessageService
 import edu.wgu.osmt.task.TaskResult
+import org.apache.commons.lang3.StringUtils
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.HttpEntity
 import org.springframework.http.HttpHeaders
@@ -75,6 +79,50 @@ class RichSkillController @Autowired constructor(
         }
         return super.allPaginated(uriComponentsBuilder, size, from, status, sort, user)
     }
+    @PostMapping(SKILLS_FILTER, produces = [MediaType.APPLICATION_JSON_VALUE])
+    @ResponseBody
+    fun allPaginatedWithFilters(
+        uriComponentsBuilder: UriComponentsBuilder,
+        size: Int,
+        from: Int,
+        status: Array<String>,
+        @RequestBody apiSearch: ApiSearch,
+        sort: String?,
+        @AuthenticationPrincipal user: Jwt?
+    ): HttpEntity<List<RichSkillDoc>> {
+
+        val publishStatuses = status.mapNotNull {
+            val status = PublishStatus.forApiValue(it)
+            if (user == null && (status == PublishStatus.Deleted  || status == PublishStatus.Draft)) null else status
+        }.toSet()
+        val sortEnum: SortOrder = sortOrderCompanion.forValueOrDefault(sort)
+        val pageable = OffsetPageable(from, size, sortEnum.sort)
+        val searchHits = richSkillEsRepo.byApiSearch(
+            apiSearch,
+            publishStatuses,
+            pageable,
+            StringUtils.EMPTY
+        )
+        val countAllFiltered: Long = searchHits.totalHits
+        val responseHeaders = HttpHeaders()
+        responseHeaders.add("X-Total-Count", countAllFiltered.toString())
+
+        uriComponentsBuilder
+            .path(SKILLS_FILTER)
+            .queryParam(RoutePaths.QueryParams.FROM, from)
+            .queryParam(RoutePaths.QueryParams.SIZE, size)
+            .queryParam(RoutePaths.QueryParams.SORT, sort)
+            .queryParam(RoutePaths.QueryParams.STATUS, status.joinToString(",").toLowerCase())
+
+        PaginatedLinks(
+            pageable,
+            searchHits.totalHits.toInt(),
+            uriComponentsBuilder
+        ).addToHeaders(responseHeaders)
+
+        return ResponseEntity.status(200).headers(responseHeaders)
+            .body(searchHits.map { it.content }.toList())
+    }
 
     @PostMapping(RoutePaths.SKILLS_CREATE, produces = [MediaType.APPLICATION_JSON_VALUE])
     @ResponseBody

api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt

@@ -115,7 +115,7 @@ data class RichSkillDoc(
             InnerField(suffix = "keyword", type = Keyword)
         ]
     )
-    @get:JsonIgnore
+    @get:JsonProperty("standards")
     val standards: List<String> = listOf(),
 
     @MultiField(