diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 2261cb731..7d8c20e12 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -9,8 +9,6 @@ import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* -import org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery -import org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -93,17 +91,17 @@ class CustomCollectionQueriesImpl @Autowired constructor( publishStatus: Set, pageable: Pageable ): SearchHits { - val nqb = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) + val nsqb1 = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) val bq = QueryBuilders.boolQuery() - //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(publishStatus.name, publishStatus.map { ps -> ps.toString() }) + val filterDslQuery = createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }) val filter = BoolQueryBuilder().must( QueryBuilders.termsQuery( RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() } ) ) - nqb.withFilter(filter) - nqb.withQuery(bq) + nsqb1.withFilter(filter) + nsqb1.withQuery(bq) var collectionMultiPropertyResults: List = listOf() @@ -113,84 +111,57 @@ class CustomCollectionQueriesImpl @Autowired constructor( bq.should( BoolQueryBuilder() .must(richSkillEsRepo.richSkillPropertiesMultiMatch(apiSearch.query)) - .must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + .must(createNestedQueryBuilder()) ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) - val nqb = NativeSearchQueryBuilder() + val nsqb = NativeSearchQueryBuilder() .withQuery( collectionPropertiesMultiMatch(apiSearch.query) ) .withPageable(Pageable.unpaged()) .withFilter(filter) // search on collection specific properties - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log) + val query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb, "CustomCollectionQueriesImpl.byApiSearch()1", log) collectionMultiPropertyResults = elasticSearchTemplate - .search(query, CollectionDoc::class.java) - .searchHits - .map { it.content.uuid } + .search(query, CollectionDoc::class.java) + .searchHits + .map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { - if (apiSearch.advanced.collectionName.contains("\"")) { - val nqb = NativeSearchQueryBuilder() - .withQuery( simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) - .withPageable(Pageable.unpaged()) - .withFilter(filter) - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log) - collectionMultiPropertyResults = elasticSearchTemplate - .search( query, CollectionDoc::class.java ) - .searchHits - .map { it.content.uuid } - } else { - val nqb = NativeSearchQueryBuilder() - .withQuery( matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) - .withPageable(Pageable.unpaged()) - .withFilter(filter) - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log) - collectionMultiPropertyResults = elasticSearchTemplate - .search( query, CollectionDoc::class.java ) - .searchHits - .map { it.content.uuid } - } + collectionMultiPropertyResults = getCollectionUuids(pageable, filterDslQuery, apiSearch.advanced.collectionName ) } else { - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } } else { // query nor advanced search was provided, return all collections - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } - var query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log) - val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java) + var query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb1, "CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", log) + val innerHitCollectionUuids = elasticSearchTemplate + .search(query, RichSkillDoc::class.java) + .searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } + .flatten() + .map { it.uuid } + .distinct() + return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct()) + } - val innerHitCollectionUuids = - results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } - .flatten().map { it.uuid }.distinct() + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNestQueryDslQuery"), DeprecationLevel.WARNING ) + private fun createNestedQueryBuilder(): NestedQueryBuilder { + return QueryBuilders.nestedQuery( + RichSkillDoc::collections.name, + QueryBuilders.matchAllQuery(), + ScoreMode.Avg + ).innerHit(InnerHitBuilder()) + } - val nqb2 = NativeSearchQueryBuilder() - .withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ) - .withFilter(filter) - .withPageable(pageable) - query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log) - return elasticSearchTemplate.search(query, CollectionDoc::class.java) + private fun getCollectionUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + return if (collectionName.contains("\"")) + getCollectionUuidsFromComplexName(pageable, filter, collectionName) + else + getCollectionUuidsFromName(pageable, filter, collectionName) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt index ca4bf1517..a1e43d240 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt @@ -43,28 +43,28 @@ fun SchemaUtils.addMissingColumnsStatementsPublic(vararg tables: Table): List { .stream() .map { ps -> ps.name} .collect(Collectors.toList()) - var filter = createTermsDslQuery(false, RichSkillDoc::publishStatus.name, filterValues) + var filter = createTermsDslQuery( RichSkillDoc::publishStatus.name, filterValues, false) return NativeQueryBuilder() .withPageable(pageable) .withQuery(MATCH_ALL) @@ -59,9 +62,45 @@ interface FindsAllByPublishStatus { } /** - * Create a query_dsl.Query instance that ElasticSearchTemplate v8.x mandates for filtering. + * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt */ - fun createTermsDslQuery(andFlag: Boolean, fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { + fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val oldQuery = nsqb.build() + val nuQuery = NativeQuery.builder() + .withFilter(filter) + .withQuery(StringQuery(oldQuery.query.toString())) + .withPageable(pageable) + .build() + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) + return nuQuery + } + + /* + * Below methods are all leveraging the latest ElasticSearch v8.7.X Java API + */ + fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return simpleQueryString { qb -> qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) } + } + + fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + query ?: matchAll { b-> b } + innerHits ?: InnerHits.Builder().build() + return nested { qb -> qb.path(path) + .scoreMode(ChildScoreMode.Avg) + .innerHits(innerHits) + .query(matchAll { b-> b }) } + } + + fun createTermsDslQuery(fieldName: String, filterValues: List, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query { val values = filterValues .stream() .map { FieldValue.of(it) } @@ -77,9 +116,9 @@ interface FindsAllByPublishStatus { /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } return bool { qb -> if (andFlag) - qb.must(terms) + qb.must(terms2) else - qb.should(terms) } + qb.should(terms2) } */ return bool() @@ -89,31 +128,39 @@ interface FindsAllByPublishStatus { ._toQuery() } - fun createTermsDslQuery(fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { - return createTermsDslQuery(true, fieldName, filterValues) + fun getCollectionUuidsFromComplexName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + val query = NativeQuery + .builder() + .withFilter(filter) + .withQuery(createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName)) + .withPageable(pageable) + .build() + return elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } - @Deprecated("", ReplaceWith("convertToNativeQuery"), DeprecationLevel.WARNING) - fun convertToStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - //NOTE: this causes us to lose the filter query - return StringQuery(query.query.toString()) + fun getCollectionUuidsFromName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + val query = NativeQuery + .builder() + .withFilter(filter) + .withQuery(createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName)) + .withPageable(pageable) + .build() + return elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } - /** - * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt - */ - fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { - val oldQuery = nsqb.build() - val nuQuery = NativeQuery.builder() + fun getCollectionFromUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List ): SearchHits { + val query = NativeQuery + .builder() .withFilter(filter) - .withQuery(StringQuery(oldQuery.query.toString())) + .withQuery(createTermsDslQuery("_id", uuids)) .withPageable(pageable) .build() - log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) - return nuQuery + return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 23687e873..226408aca 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -71,7 +71,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea } } -@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("Replacement method"), DeprecationLevel.WARNING ) +@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("JobCodeQueriesEx"), DeprecationLevel.WARNING ) object JobCodeQueries { //TODO Convert to ES v8.7.x apis and return the newer BoolQuery.Builder instance; see KeywordEsRep.kt fun multiPropertySearch(query: String, parentDocPath: String? = null): BoolQueryBuilder { diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 0f5008322..997b7093a 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -1,10 +1,11 @@ package edu.wgu.osmt.richskill +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.query_dsl.Query +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import edu.wgu.osmt.PaginationDefaults -import edu.wgu.osmt.api.model.ApiAdvancedSearch -import edu.wgu.osmt.api.model.ApiFilteredSearch -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSimilaritySearch +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.config.QUOTED_SEARCH_REGEX_PATTERN import edu.wgu.osmt.db.PublishStatus @@ -32,6 +33,8 @@ import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt +import java.util.function.Consumer +import java.util.stream.Collectors const val collectionsUuid = "collections.uuid" @@ -102,6 +105,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return uuids } + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) override fun occupationQueries(query: String): NestedQueryBuilder { val jobCodePath = RichSkillDoc::jobCodes.name return QueryBuilders.nestedQuery( @@ -111,6 +115,21 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ) } + /** + * ElasticSearch v8.7.X version + */ + fun occupationQueriesNu(query: String): Query? { + /* + val multiPropQuery = null + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.nested { + qb -> qb.path(RichSkillDoc::jobCodes.name) + .query(JobCodeQueries.multiPropertySearch(query, jobCodePath),) + .scoreMode( ChildScoreMode.Max)} + */ + return null; + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) private fun buildNestedQueries(path: String?=null, queryParams: List) : BoolQueryBuilder { val disjunctionQuery = disMaxQuery() val queries = ArrayList() @@ -128,8 +147,23 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return boolQuery().must(existsQuery("$path.keyword")).must(disjunctionQuery) } + /** + * ElasticSearch v8.7.X version + */ + private fun buildNestedQueriesNu(path: String?=null, queryParams: List) : Query { + val prefixQueries = ArrayList() + queryParams.forEach(Consumer { s: String? -> + val q = prefix { qb -> qb.field( "$path.keyword").value(s) } + prefixQueries.add(q) + }) + + val disMaxQuery = disMax {qb -> qb.queries(prefixQueries)} + val existQuery = exists { qb -> qb.field("$path.keyword")} + return bool { qb -> qb.must(disMaxQuery).must(existQuery)} + } // Query clauses for Rich Skill properties + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearch(bq: BoolQueryBuilder, advancedQuery: ApiAdvancedSearch) { with(advancedQuery) { // boolQuery.must for logical AND @@ -199,8 +233,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear it.mapNotNull { it.name }.map { s -> if (s.contains("\"")) { bq.must( - simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND) + simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw").defaultOperator(Operator.AND) ) } else { bq.must(matchBoolPrefixQuery(RichSkillDoc::certifications.name, s)) @@ -234,7 +267,50 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * ElasticSearch v8.7.X version + */ + fun generateBoolQueriesFromApiSearchNu(advancedQuery: ApiAdvancedSearch): Query { + with(advancedQuery) { + return bool { bq -> + skillName.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + category.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::categories.name, it, QUOTED_SEARCH_REGEX_PATTERN)) } + author.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::authors.name, it)) } + skillStatement.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::statement.name, it)) } + keywords?.let { bq.must(createQueryFromStringList(RichSkillDoc::searchingKeywords.name, it)) } +//TODO implement this +// occupations.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + + standards?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::standards.name, it)) } + certifications?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::certifications.name, it)) } + employers?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::employers.name, it)) } + alignments?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::alignments.name, it)) } + } + } + } + + private fun createQueryFromString(fieldName: String, searchStr: String, regEx: String? = null): Query { + val isComplex = searchStr.contains("\"") || (regEx != null && searchStr.matches(Regex(regEx))) + return if (isComplex) + createSimpleQueryDslQuery(String.format("%s.raw", fieldName), searchStr) + else + createMatchBoolPrefixDslQuery(fieldName, searchStr) + } + + private fun createQueryFromStringList(fieldName: String, searchStrList: List): List { + return searchStrList + .stream() + .map { createQueryFromString(fieldName, it ?: "") } + .collect(Collectors.toList()) + } + + private fun createQueryFromApiNameList(fieldName: String, searchStrList: List): List { + return createQueryFromStringList(fieldName, searchStrList.map { it.name?: "" }) + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchWithFiltersNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearchWithFilters(bq: BoolQueryBuilder, filteredQuery: ApiFilteredSearch, publishStatus: Set) { + bq.must( termsQuery( RichSkillDoc::publishStatus.name, @@ -283,25 +359,76 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * TODO Fix the NPE at the return. + fun generateBoolQueriesFromApiSearchWithFiltersNu(filteredQuery: ApiFilteredSearch, publishStatus: Set) : Query { + val values = publishStatus + .stream() + .map { FieldValue.of(it.toString()) } + .collect(Collectors.toList()) + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms2 = terms { qb -> qb.field(RichSkillDoc::publishStatus.name).terms(tqf) } + val qb = bool().must(terms2) + + with(filteredQuery) { + categories?. let { qb.must(buildNestedQueriesNu(RichSkillDoc::categories.name, it)) } + keywords?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::searchingKeywords.name, keywords)) } + } + standards?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::standards.name, standards)) } + } + certifications?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::certifications.name, certifications)) } + } + alignments?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::alignments.name, alignments)) } + } + employers?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::employers.name, employers)) } + } + authors?. let { + qb.must(buildNestedQueriesNu(RichSkillDoc::authors.name, it)) + } + occupations?.let { + it.mapNotNull { value -> qb.must( occupationQueriesNu(value) ) } + } + } + val s = qb.build()._toQuery() + return s + } + */ + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateTermsSetQueryBuilderNu"), DeprecationLevel.WARNING) private fun generateTermsSetQueryBuilder(fieldName: String, list: List): TermsSetQueryBuilder { return TermsSetQueryBuilder("$fieldName.keyword", list).setMinimumShouldMatchScript(Script(list.size.toString())) } + /** + * ElasticSearch v8.7.X version + */ + private fun generateTermsSetQueryBuilderNu(fieldName: String, list: List): Query { + val sb = co.elastic.clients.elasticsearch._types.Script.Builder().inline { il -> il.source(list.size.toString())} + return termsSet { + qb -> qb.field("$fieldName.keyword") + .terms(list) + .minimumShouldMatchScript(sb.build()) + } + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("richSkillPropertiesMultiMatchNu"), DeprecationLevel.WARNING) override fun richSkillPropertiesMultiMatch(query: String): BoolQueryBuilder { val isComplex = query.contains("\"") - val boolQuery = boolQuery() - val complexQueries = listOf( - simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::statement.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::categories.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::standards.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::employers.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::alignments.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::authors.name}.raw").defaultOperator(Operator.AND) @@ -324,10 +451,48 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } else { queries.map { boolQuery.should(it) } } - return boolQuery } + /** + * ElasticSearch v8.7.X version + */ + fun richSkillPropertiesMultiMatchNu(searchStr: String): Query { + var queries = if (searchStr.contains("\"")) + createComplexMultiMatchQueries(searchStr) + else + createMultiMatchQueries(searchStr) + return bool {qb -> qb.should(queries) } + } + + private fun createComplexMultiMatchQueries(searchStr: String) : List{ + return listOf( + createSimpleQueryDslQuery("${RichSkillDoc::name.name}.raw", searchStr, 2.0f), + createSimpleQueryDslQuery("${RichSkillDoc::statement.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::categories.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::searchingKeywords.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::standards.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::certifications.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::employers.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::alignments.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::authors.name}.raw", searchStr) + ) + } + + private fun createMultiMatchQueries(searchStr: String) : List{ + return listOf( + createMatchPhrasePrefixDslQuery(RichSkillDoc::name.name, searchStr, 2.0f), + createMatchPhrasePrefixDslQuery(RichSkillDoc::statement.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::categories.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::searchingKeywords.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::standards.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::certifications.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::employers.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::alignments.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::authors.name, searchStr) + ) + } + override fun byApiSearch( apiSearch: ApiSearch, publishStatus: Set, @@ -481,6 +646,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } + }