diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java index d47e96c2de..77704361eb 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java @@ -27,7 +27,9 @@ import org.apache.usergrid.persistence.index.impl.IndexProducer; import org.apache.usergrid.persistence.model.field.DistanceField; import org.apache.usergrid.persistence.model.field.DoubleField; +import org.apache.usergrid.persistence.model.field.EntityObjectField; import org.apache.usergrid.persistence.model.field.Field; +import org.apache.usergrid.persistence.model.field.value.EntityObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,9 +52,9 @@ /** - * Loads entities from an incoming CandidateResult emissions into entities, then streams them on - * performs internal buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. It is up to the - * collector to determine when to stop streaming entities. + * Loads entities from an incoming CandidateResult emissions into entities, then streams them on performs internal + * buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. + * It is up to the collector to determine when to stop streaming entities. */ public class CandidateEntityFilter extends AbstractFilter, FilterResult> { @@ -91,11 +93,12 @@ public Observable> call( entityCollectionManagerFactory.createCollectionManager( applicationScope ); - final EntityIndex applicationIndex = - entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); + final EntityIndex applicationIndex = entityIndexFactory + .createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); //buffer them to get a page size we can make 1 network hop - final Observable> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() ) + final Observable> searchIdSetObservable = + candidateResultsObservable.buffer( pipelineContext.getLimit() ) //load them .flatMap( candidateResults -> { @@ -123,12 +126,26 @@ public Observable> call( if (mappings.size() > 0) { Map fieldMap = new HashMap(mappings.size()); rx.Observable.from(mappings) - .filter(mapping -> entity.getFieldMap().containsKey(mapping.getSourceFieldName())) + + .filter(mapping -> { + if ( entity.getFieldMap().containsKey(mapping.getSourceFieldName())) { + return true; + } + String[] parts = mapping.getSourceFieldName().split("\\."); + return nestedFieldCheck( parts, entity.getFieldMap() ); + }) + .doOnNext(mapping -> { Field field = entity.getField(mapping.getSourceFieldName()); - field.setName(mapping.getTargetFieldName()); - fieldMap.put(mapping.getTargetFieldName(),field); - }).toBlocking().last(); + if ( field != null ) { + field.setName( mapping.getTargetFieldName() ); + fieldMap.put( mapping.getTargetFieldName(), field ); + } else { + String[] parts = mapping.getSourceFieldName().split("\\."); + nestedFieldSet( fieldMap, parts, entity.getFieldMap() ); + } + }).toBlocking().lastOrDefault(null); + entity.setFieldMap(fieldMap); } return entityFilterResult; @@ -144,10 +161,65 @@ public Observable> call( } + /** + * Sets field in result map with support for nested fields via recursion. + * + * @param result The result map of filtered fields + * @param parts The parts of the field name (more than one if field is nested) + * @param fieldMap Map of fields of the object + */ + private void nestedFieldSet( Map result, String[] parts, Map fieldMap) { + if ( parts.length > 0 ) { + + if ( fieldMap.containsKey( parts[0] )) { + Field field = fieldMap.get( parts[0] ); + if ( field instanceof EntityObjectField ) { + EntityObjectField eof = (EntityObjectField)field; + result.putIfAbsent( parts[0], new EntityObjectField( parts[0], new EntityObject() ) ); + + // recursion + nestedFieldSet( + ((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(), + Arrays.copyOfRange(parts, 1, parts.length), + eof.getValue().getFieldMap()); + + } else { + result.put( parts[0], field ); + } + } + } + } + + + /** + * Check to see if field should be included in filtered result with support for nested fields via recursion. + * + * @param parts The parts of the field name (more than one if field is nested) + * @param fieldMap Map of fields of the object + */ + private boolean nestedFieldCheck( String[] parts, Map fieldMap) { + if ( parts.length > 0 ) { + + if ( fieldMap.containsKey( parts[0] )) { + Field field = fieldMap.get( parts[0] ); + if ( field instanceof EntityObjectField ) { + EntityObjectField eof = (EntityObjectField)field; + + // recursion + return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap()); + + } else { + return true; + } + } + } + return false; + } /** - * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally + * Our collector to collect entities. Not quite a true collector, but works within our operational + * flow as this state is mutable and difficult to represent functionally */ private static final class EntityVerifier { diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java index f4aa204ea7..d62f88ef2a 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java @@ -17,6 +17,7 @@ package org.apache.usergrid.persistence; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; @@ -461,4 +462,101 @@ public void testPropertyUpdateWithConnectionEntityIndexEntryAudit() throws Excep } + + @Test + public void testSelectMappings() throws Exception { + + UUID applicationId = app.getId(); + + EntityManager em = setup.getEmf().getEntityManager(applicationId); + + Map entity1 = new HashMap() {{ + put("name","name_1"); + put("status", "pickled"); + put("data", new HashMap() {{ + put("xfactor", 5.1); + put("rando", "bar"); + put("mondo", "2000"); + put("frosting", "chocolate"); + put("misc", new HashMap() {{ + put("range", "open"); + }}); + }}); + }}; + em.create("names", entity1); + + Map entity2 = new HashMap() {{ + put("name","name_2"); + put("status", "pickled"); + put("data", new HashMap() {{ + put("xfactor", 5.1); + put("rando", "bar"); + put("mondo", "2001"); + put("frosting", "orange"); + put("misc", new HashMap() {{ + put("range", "open"); + }}); + }}); + }}; + em.create("names", entity2); + + app.refreshIndex(); + + // simple single-field select mapping + { + Query query = Query.fromQL("select status where status = 'pickled'"); + Results r = em.searchCollection( em.getApplicationRef(), "names", query ); + assertTrue(r.getEntities() != null && r.getEntities().size() == 2); + Entity first = r.getEntities().get(0); + assertTrue(first.getDynamicProperties().size() == 2); + } + + // simple single-field plus nested field select mapping + { + Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" ); + Results r = em.searchCollection( em.getApplicationRef(), "names", query ); + assertTrue( r.getEntities() != null && r.getEntities().size() == 2 ); + + Entity first = r.getEntities().get( 0 ); + + assertNotNull( first.getProperty("status") ); + assertEquals( first.getProperty("status"), "pickled" ); + + assertNotNull( first.getProperty("data") ); + assertEquals( ((Map)first.getProperty("data")).get("rando"), "bar" ); + + assertTrue( first.getDynamicProperties().size() == 3 ); + } + + // multiple nested fields with one doubly-nested field + { + Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" ); + Results r = em.searchCollection( em.getApplicationRef(), "names", query ); + assertTrue( r.getEntities() != null && r.getEntities().size() == 2 ); + + Entity first = r.getEntities().get( 0 ); + + Map data = ((Map)first.getProperty("data")); + assertNotNull( data ); + assertEquals( data.get("rando"), "bar" ); + assertEquals( data.get("mondo"), "2001" ); + assertNull( data.get("frosting") ); + + Map misc = (Map)data.get("misc"); + assertEquals( misc.get("range"), "open" ); + + assertTrue( first.getDynamicProperties().size() == 2 ); + } + + // query for one bogus field name should return empty entities + { + Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" ); + Results r = em.searchCollection( em.getApplicationRef(), "names", query ); + assertTrue( r.getEntities() != null && r.getEntities().size() == 2 ); + Entity first = r.getEntities().get( 0 ); + assertTrue( first.getDynamicProperties().size() == 1 ); + } + + } + } diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java index db44e87adf..a157029aa7 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java @@ -19,10 +19,7 @@ package org.apache.usergrid.persistence.model.field.value; import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.usergrid.persistence.model.field.Field; @@ -41,11 +38,19 @@ public class EntityObject implements Serializable { @JsonIgnore private long size; + // field names are treated in case-insensitive way by design + static class CaseInsensitiveComparator implements Comparator { + public int compare(String o1, String o2) { + return o1.compareToIgnoreCase(o2); + } + } + public static final CaseInsensitiveComparator INSTANCE = new CaseInsensitiveComparator(); + /** * Fields the users can set */ @JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" ) - private final Map fields = new HashMap(); + private Map fields = new TreeMap(INSTANCE); /** * Add the field, return the old one if it existed diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java new file mode 100644 index 0000000000..fd33c15424 --- /dev/null +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.applications.queries; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.usergrid.rest.test.resource.model.Collection; +import org.apache.usergrid.rest.test.resource.model.Entity; +import org.apache.usergrid.rest.test.resource.model.QueryParameters; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + + +public class SelectMappingsQueryTest extends QueryTestBase { + private static final Logger logger = LoggerFactory.getLogger(OrderByTest.class); + + + /** + * Select field mappings may include nested entity fields. + */ + @Test + public void testNestedSelectFieldNames() throws Exception { + + generateTestEntities(20, "things"); + + QueryParameters params = new QueryParameters() + .setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'"); + Collection things = this.app().collection("things").get(params); + assertEquals( 10, things.getNumOfEntities() ); + + Iterator iter = things.iterator(); + while ( iter.hasNext() ) { + + Entity entity = iter.next(); + assertEquals( 5, entity.getDynamicProperties().size() ); + + assertNotNull( entity.getDynamicProperties().get("uuid") ); + assertNotNull( entity.getDynamicProperties().get("type") ); + assertNotNull( entity.getDynamicProperties().get("metadata") ); + assertNotNull( entity.getDynamicProperties().get("sometestprop") ); + + Map actor = (Map)entity.getDynamicProperties().get("actor"); + assertNotNull( actor ); + assertNotNull( actor.get("displayName") ); + + } + } + + + /** + * When entity posted with two duplicate names with different cases, last one wins. + */ + @Test + public void testMixedCaseDupField() throws Exception { + + String collectionName = "things"; + + String value = RandomStringUtils.randomAlphabetic( 20 ); + String otherValue = RandomStringUtils.randomAlphabetic( 20 ); + + // create entity with testProp=value + Entity entity = new Entity() + .withProp( "testProp", value ) + .withProp( "TESTPROP", otherValue); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // testProp and TESTPROP should now have otherValue + + QueryParameters params = new QueryParameters() + .setQuery( "select * where testProp='" + otherValue + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + params = new QueryParameters() + .setQuery( "select * where TESTPROP='" + otherValue + "'" ); + things = app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + } + + + /** + * Field named testProp can be over-written by field named TESTPROP. + */ + @Test + public void testFieldOverride1() throws Exception { + + String collectionName = "things"; + + // create entity with testProp=value + String value = RandomStringUtils.randomAlphabetic( 20 ); + Entity entity = new Entity().withProp( "testProp", value ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // override with TESTPROP=newValue + String newValue = RandomStringUtils.randomAlphabetic( 20 ); + entity = new Entity().withProp( "TESTPROP", newValue ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // testProp and TESTPROP should new be queryable by new value + + QueryParameters params = new QueryParameters() + .setQuery( "select * where testProp='" + newValue + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + params = new QueryParameters() + .setQuery( "select * where TESTPROP='" + newValue + "'" ); + things = app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + } + + /** + * Field named testProp can be over-written by field named TESTPROP. + */ + @Test + public void testFieldOverride2() throws Exception { + + String collectionName = "things"; + + // create entity with TESTPROP=value + String value = RandomStringUtils.randomAlphabetic( 20 ); + Entity entity = new Entity().withProp( "TESTPROP", value ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // override with testProp=newValue + String newValue = RandomStringUtils.randomAlphabetic( 20 ); + entity = new Entity().withProp( "testProp", newValue ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // testProp and TESTPROP should new be queryable by new value + + QueryParameters params = new QueryParameters() + .setQuery( "select * where testProp='" + newValue + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + params = new QueryParameters() + .setQuery( "select * where TESTPROP='" + newValue + "'" ); + things = app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + } + +}