Support for nested fields in Query Select Mappings

stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java

@@ -27,7 +27,9 @@
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.apache.usergrid.persistence.model.field.DistanceField;
 import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
 import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -50,9 +52,9 @@
 
 
 /**
- * Loads entities from an incoming CandidateResult emissions into entities, then streams them on
- * performs internal buffering for efficiency.  Note that all entities may not be emitted if our load crosses page boundaries.  It is up to the
- * collector to determine when to stop streaming entities.
+ * Loads entities from an incoming CandidateResult emissions into entities, then streams them on performs internal
+ * buffering for efficiency.  Note that all entities may not be emitted if our load crosses page boundaries.
+ * It is up to the collector to determine when to stop streaming entities.
  */
 public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate>, FilterResult<Entity>> {
 
@@ -91,11 +93,12 @@ public Observable<FilterResult<Entity>> call(
             entityCollectionManagerFactory.createCollectionManager( applicationScope );
 
 
-        final EntityIndex applicationIndex =
-            entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
+        final EntityIndex applicationIndex = entityIndexFactory
+            .createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
 
         //buffer them to get a page size we can make 1 network hop
-        final Observable<FilterResult<Entity>> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() )
+        final Observable<FilterResult<Entity>> searchIdSetObservable =
+            candidateResultsObservable.buffer( pipelineContext.getLimit() )
 
             //load them
             .flatMap( candidateResults -> {
@@ -123,12 +126,26 @@ public Observable<FilterResult<Entity>> call(
                                 if (mappings.size() > 0) {
                                     Map<String,Field> fieldMap = new HashMap<String, Field>(mappings.size());
                                     rx.Observable.from(mappings)
-                                        .filter(mapping -> entity.getFieldMap().containsKey(mapping.getSourceFieldName()))
+
+                                        .filter(mapping -> {
+                                            if ( entity.getFieldMap().containsKey(mapping.getSourceFieldName())) {
+                                                return true;
+                                            }
+                                            String[] parts = mapping.getSourceFieldName().split("\\.");
+                                            return nestedFieldCheck( parts, entity.getFieldMap() );
+                                        })
+
                                         .doOnNext(mapping -> {
                                             Field field = entity.getField(mapping.getSourceFieldName());
-                                            field.setName(mapping.getTargetFieldName());
-                                            fieldMap.put(mapping.getTargetFieldName(),field);
-                                        }).toBlocking().last();
+                                            if ( field != null ) {
+                                                field.setName( mapping.getTargetFieldName() );
+                                                fieldMap.put( mapping.getTargetFieldName(), field );
+                                            } else {
+                                                String[] parts = mapping.getSourceFieldName().split("\\.");
+                                                nestedFieldSet( fieldMap, parts, entity.getFieldMap() );
+                                            }
+                                        }).toBlocking().lastOrDefault(null);
+
                                     entity.setFieldMap(fieldMap);
                                 }
                                 return entityFilterResult;
@@ -144,10 +161,65 @@ public Observable<FilterResult<Entity>> call(
     }
 
 
+    /**
+     * Sets field in result map with support for nested fields via recursion.
+     *
+     * @param result The result map of filtered fields
+     * @param parts The parts of the field name (more than one if field is nested)
+     * @param fieldMap Map of fields of the object
+     */
+    private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
+        if ( parts.length > 0 ) {
+
+            if ( fieldMap.containsKey( parts[0] )) {
+                Field field = fieldMap.get( parts[0] );
+                if ( field instanceof EntityObjectField ) {
+                    EntityObjectField eof = (EntityObjectField)field;
+                    result.putIfAbsent( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
+
+                    // recursion
+                    nestedFieldSet(
+                        ((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(),
+                        Arrays.copyOfRange(parts, 1, parts.length),
+                        eof.getValue().getFieldMap());
+
+                } else {
+                    result.put( parts[0], field );
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Check to see if field should be included in filtered result with support for nested fields via recursion.
+     *
+     * @param parts The parts of the field name (more than one if field is nested)
+     * @param fieldMap Map of fields of the object
+     */
+    private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
+        if ( parts.length > 0 ) {
+
+            if ( fieldMap.containsKey( parts[0] )) {
+                Field field = fieldMap.get( parts[0] );
+                if ( field instanceof EntityObjectField ) {
+                    EntityObjectField eof = (EntityObjectField)field;
+
+                    // recursion
+                    return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap());
+
+                } else {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
 
 
     /**
-     * Our collector to collect entities.  Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally
+     * Our collector to collect entities.  Not quite a true collector, but works within our operational
+     * flow as this state is mutable and difficult to represent functionally
      */
     private static final class EntityVerifier {
 

stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java

@@ -17,6 +17,7 @@
 package org.apache.usergrid.persistence;
 
 
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -461,4 +462,101 @@ public void testPropertyUpdateWithConnectionEntityIndexEntryAudit() throws Excep
 
 
     }
+
+    @Test
+    public void testSelectMappings() throws Exception {
+
+        UUID applicationId = app.getId();
+
+        EntityManager em = setup.getEmf().getEntityManager(applicationId);
+
+        Map<String, Object> entity1 = new HashMap<String, Object>() {{
+            put("name","name_1");
+            put("status", "pickled");
+            put("data", new HashMap() {{
+                put("xfactor", 5.1);
+                put("rando", "bar");
+                put("mondo", "2000");
+                put("frosting", "chocolate");
+                put("misc", new HashMap() {{
+                    put("range", "open");
+                }});
+            }});
+        }};
+        em.create("names", entity1);
+
+        Map<String, Object> entity2 = new HashMap<String, Object>() {{
+            put("name","name_2");
+            put("status", "pickled");
+            put("data", new HashMap() {{
+                put("xfactor", 5.1);
+                put("rando", "bar");
+                put("mondo", "2001");
+                put("frosting", "orange");
+                put("misc", new HashMap() {{
+                    put("range", "open");
+                }});
+            }});
+        }};
+        em.create("names", entity2);
+
+        app.refreshIndex();
+
+        // simple single-field select mapping
+        {
+            Query query = Query.fromQL("select status where status = 'pickled'");
+            Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+            assertTrue(r.getEntities() != null && r.getEntities().size() == 2);
+            Entity first =  r.getEntities().get(0);
+            assertTrue(first.getDynamicProperties().size() == 2);
+        }
+
+        // simple single-field plus nested field select mapping
+        {
+            Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" );
+            Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+            assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+            Entity first = r.getEntities().get( 0 );
+
+            assertNotNull( first.getProperty("status") );
+            assertEquals( first.getProperty("status"), "pickled" );
+
+            assertNotNull( first.getProperty("data") );
+            assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+
+            assertTrue( first.getDynamicProperties().size() == 3 );
+        }
+
+        // multiple nested fields with one doubly-nested field
+        {
+            Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" );
+            Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+            assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+            Entity first = r.getEntities().get( 0 );
+
+            Map<String, Object> data = ((Map<String, Object>)first.getProperty("data"));
+            assertNotNull( data );
+            assertEquals( data.get("rando"), "bar" );
+            assertEquals( data.get("mondo"), "2001" );
+            assertNull( data.get("frosting") );
+
+            Map<String, Object> misc = (Map<String, Object>)data.get("misc");
+            assertEquals( misc.get("range"), "open" );
+
+            assertTrue( first.getDynamicProperties().size() == 2 );
+        }
+
+        //  query for one bogus field name should return empty entities
+        {
+            Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" );
+            Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+            assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+            Entity first = r.getEntities().get( 0 );
+            assertTrue( first.getDynamicProperties().size() == 1 );
+        }
+
+    }
+
 }

stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java

@@ -19,10 +19,7 @@
 package org.apache.usergrid.persistence.model.field.value;
 
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.usergrid.persistence.model.field.Field;
 
@@ -41,11 +38,19 @@ public class EntityObject implements Serializable {
     @JsonIgnore
     private long size;
 
+    // field names are treated in case-insensitive way by design
+    static class CaseInsensitiveComparator implements Comparator<String> {
+        public int compare(String o1, String o2) {
+            return o1.compareToIgnoreCase(o2);
+        }
+    }
+    public static final CaseInsensitiveComparator INSTANCE = new CaseInsensitiveComparator();
+
     /**
      * Fields the users can set
      */
     @JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
-    private final Map<String, Field> fields = new HashMap<String, Field>();
+    private Map<String, Field> fields = new TreeMap<String, Field>(INSTANCE);
 
     /**
      * Add the field, return the old one if it existed