datastore: allow nested unindexed properties

packages/datastore/src/entity.js

@@ -362,14 +362,20 @@ entity.entityFromEntityProto = entityFromEntityProto;
  *
  * @example
  * entityToEntityProto({
- *   name: 'Burcu',
- *   legit: true
+ *   excludeFromIndexes: [
+ *     'name'
+ *   ],
+ *   data: {
+ *     name: 'Burcu',
+ *     legit: true
+ *   }
  * });
  * // {
  * //   key: null,
  * //   properties: {
  * //     name: {
  * //       stringValue: 'Burcu'
+ * //       excludeFromIndexes: true
  * //     },
  * //     legit: {
  * //       booleanValue: true
@@ -378,14 +384,70 @@ entity.entityFromEntityProto = entityFromEntityProto;
  * // }
  */
 function entityToEntityProto(entityObject) {
-  return {
+  var properties = entityObject.data;
+  var excludeFromIndexes = entityObject.excludeFromIndexes;
+
+  var entityProto = {
     key: null,
 
-    properties: Object.keys(entityObject).reduce(function(properties, key) {
-      properties[key] = entity.encodeValue(entityObject[key]);
-      return properties;
+    properties: Object.keys(properties).reduce(function(encoded, key) {
+      encoded[key] = entity.encodeValue(properties[key]);
+      return encoded;
     }, {})
   };
+
+  if (excludeFromIndexes && excludeFromIndexes.length > 0) {
+    excludeFromIndexes.forEach(function(excludePath) {
+      excludePathFromEntity(entityProto, excludePath);
+    });
+  }
+
+  return entityProto;
+
+  function excludePathFromEntity(entity, path) {
+    var arrayIndex = path.indexOf('[]');
+    var entityIndex = path.indexOf('.');
+
+    var hasArrayPath = arrayIndex > -1;
+    var hasEntityPath = entityIndex > -1;
+
+    if (!hasArrayPath && !hasEntityPath) {
+      if (entity.properties[path]) {
+        // This is the property to exclude!
+        entity.properties[path].excludeFromIndexes = true;
+      }
+      return;
+    }
+
+    var delimiterIndex;
+    if (hasArrayPath && hasEntityPath) {
+      delimiterIndex = Math.min(arrayIndex, entityIndex);
+    } else {
+      delimiterIndex = Math.max(arrayIndex, entityIndex);
+    }
+
+    var firstPathPartIsArray = delimiterIndex === arrayIndex;
+    var firstPathPartIsEntity = delimiterIndex === entityIndex;
+
+    var delimiter = firstPathPartIsArray ? '[]' : '.';
+    var splitPath = path.split(delimiter);
+    var firstPathPart = splitPath.shift();
+    var remainderPath = splitPath.join(delimiter).replace(/^(\.|[])/, '');
+
+    if (!entity.properties[firstPathPart]) {
+      return;
+    }
+
+    if (firstPathPartIsArray) {
+      var array = entity.properties[firstPathPart].arrayValue;
+      array.values.forEach(function(arrayValue) {
+        excludePathFromEntity(arrayValue.entityValue, remainderPath);
+      });
+    } else if (firstPathPartIsEntity) {
+      var parentEntity = entity.properties[firstPathPart].entityValue;
+      excludePathFromEntity(parentEntity, remainderPath);
+    }
+  }
 }
 
 entity.entityToEntityProto = entityToEntityProto;

packages/datastore/src/request.js

@@ -463,10 +463,11 @@ DatastoreRequest.prototype.get = function(keys, options, callback) {
  *
  * @param {object|object[]} entities - Datastore key object(s).
  * @param {Key} entities.key - Datastore key object.
- * @param {object|object[]} entities.data - Data to save with the provided key.
- *     If you provide an array of objects, you must use the explicit syntax:
- *     `name` for the name of the property and `value` for its value. You may
- *     also specify an `excludeFromIndexes` property, set to `true` or `false`.
+ * @param {string[]=} entities.excludeFromIndexes - Exclude properties from
+ *     indexing using a simple JSON path notation. See the examples in
+ *     {module:datastore#save} to see how to target properties at different
+ *     levels of nesting within your entity.
+ * @param {object} entities.data - Data to save with the provided key.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request
  * @param {object} callback.apiResponse - The full API response.
@@ -720,21 +721,22 @@ DatastoreRequest.prototype.runQueryStream = function(query, options) {
  * the entity will be updated with the data specified.
  *
  * By default, all properties are indexed. To prevent a property from being
- * included in *all* indexes, you must supply an entity's `data` property as an
- * array. See below for an example.
+ * included in *all* indexes, you must supply an `excludeFromIndexes` array. See
+ * below for an example.
  *
  * @borrows {module:datastore/transaction#save} as save
  *
  * @throws {Error} If an unrecognized method is provided.
  *
  * @param {object|object[]} entities - Datastore key object(s).
  * @param {Key} entities.key - Datastore key object.
+ * @param {string[]=} entities.excludeFromIndexes - Exclude properties from
+ *     indexing using a simple JSON path notation. See the example below to see
+ *     how to target properties at different levels of nesting within your
+ *     entity.
  * @param {string=} entities.method - Explicit method to use, either 'insert',
  *     'update', or 'upsert'.
- * @param {object|object[]} entities.data - Data to save with the provided key.
- *     If you provide an array of objects, you must use the explicit syntax:
- *     `name` for the name of the property and `value` for its value. You may
- *     also specify an `excludeFromIndexes` property, set to `true` or `false`.
+ * @param {object} entities.data - Data to save with the provided key.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request
  * @param {object} callback.apiResponse - The full API response.
@@ -840,18 +842,27 @@ DatastoreRequest.prototype.runQueryStream = function(query, options) {
  * datastore.save(entity, function(err, apiResponse) {});
  *
  * //-
- * // To specify an `excludeFromIndexes` value for a Datastore entity, pass in
- * // an array for the key's data.
+ * // Use an array, `excludeFromIndexes`, to exclude properties from indexing.
+ * // This will allow storing string values larger than 1500 bytes.
  * //-
  * var entity = {
  *   key: datastore.key('Company'),
- *   data: [
- *     {
- *       name: 'rating',
- *       value: 10,
- *       excludeFromIndexes: true
- *     }
- *   ]
+ *   excludeFromIndexes: [
+ *     'description',
+ *     'embeddedEntity.description',
+ *     'arrayValue[].description'
+ *   ],
+ *   data: {
+ *     description: 'Long string (...)',
+ *     embeddedEntity: {
+ *       description: 'Long string (...)'
+ *     },
+ *     arrayValue: [
+ *       {
+ *         description: 'Long string (...)'
+ *       }
+ *     ]
+ *   }
  * };
  *
  * datastore.save(entity, function(err, apiResponse) {});
@@ -931,6 +942,9 @@ DatastoreRequest.prototype.save = function(entities, callback) {
         insertIndexes[index] = true;
       }
 
+      // @TODO remove in @google-cloud/datastore@2.0.0
+      // This was replaced with a more efficient mechanism in the top-level
+      // `excludeFromIndexes` option.
       if (is.array(entityObject.data)) {
         entityProto.properties = entityObject.data.reduce(function(acc, data) {
           var value = entity.encodeValue(data.value);
@@ -951,7 +965,7 @@ DatastoreRequest.prototype.save = function(entities, callback) {
           return acc;
         }, {});
       } else {
-        entityProto = entity.entityToEntityProto(entityObject.data);
+        entityProto = entity.entityToEntityProto(entityObject);
       }
 
       entityProto.key = entity.keyToKeyProto(entityObject.key);
@@ -1003,10 +1017,11 @@ DatastoreRequest.prototype.save = function(entities, callback) {
  *
  * @param {object|object[]} entities - Datastore key object(s).
  * @param {Key} entities.key - Datastore key object.
- * @param {object|object[]} entities.data - Data to save with the provided key.
- *     If you provide an array of objects, you must use the explicit syntax:
- *     `name` for the name of the property and `value` for its value. You may
- *     also specify an `excludeFromIndexes` property, set to `true` or `false`.
+ * @param {string[]=} entities.excludeFromIndexes - Exclude properties from
+ *     indexing using a simple JSON path notation. See the examples in
+ *     {module:datastore#save} to see how to target properties at different
+ *     levels of nesting within your entity.
+ * @param {object} entities.data - Data to save with the provided key.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request
  * @param {object} callback.apiResponse - The full API response.
@@ -1024,10 +1039,11 @@ DatastoreRequest.prototype.update = function(entities, callback) {
  *
  * @param {object|object[]} entities - Datastore key object(s).
  * @param {Key} entities.key - Datastore key object.
- * @param {object|object[]} entities.data - Data to save with the provided key.
- *     If you provide an array of objects, you must use the explicit syntax:
- *     `name` for the name of the property and `value` for its value. You may
- *     also specify an `excludeFromIndexes` property, set to `true` or `false`.
+ * @param {string[]=} entities.excludeFromIndexes - Exclude properties from
+ *     indexing using a simple JSON path notation. See the examples in
+ *     {module:datastore#save} to see how to target properties at different
+ *     levels of nesting within your entity.
+ * @param {object} entities.data - Data to save with the provided key.
  * @param {function} callback - The callback function.
  * @param {?error} callback.err - An error returned while making this request
  * @param {object} callback.apiResponse - The full API response.

packages/datastore/src/transaction.js

@@ -407,15 +407,16 @@ Transaction.prototype.run = function(callback) {
  * the entity will be updated with the data specified.
  *
  * By default, all properties are indexed. To prevent a property from being
- * included in *all* indexes, you must supply an entity's `data` property as an
- * array. See below for an example.
+ * included in *all* indexes, you must supply an `excludeFromIndexes` array. See
+ * below for an example.
  *
  * @param {object|object[]} entities - Datastore key object(s).
  * @param {Key} entities.key - Datastore key object.
- * @param {object|object[]} entities.data - Data to save with the provided key.
- *     If you provide an array of objects, you must use the explicit syntax:
- *     `name` for the name of the property and `value` for its value. You may
- *     also specify an `excludeFromIndexes` property, set to `true` or `false`.
+ * @param {string[]=} entities.excludeFromIndexes - Exclude properties from
+ *     indexing using a simple JSON path notation. See the example below to see
+ *     how to target properties at different levels of nesting within your
+ *     entity.
+ * @param {object} entities.data - Data to save with the provided key.
  *
  * @example
  * //-
@@ -447,8 +448,8 @@ Transaction.prototype.run = function(callback) {
  * });
  *
  * //-
- * // To specify an `excludeFromIndexes` value for a Datastore entity, pass in
- * // an array for the key's data. The above example would then look like:
+ * // Use an array, `excludeFromIndexes`, to exclude properties from indexing.
+ * // This will allow storing string values larger than 1500 bytes.
  * //-
  * transaction.run(function(err) {
  *   if (err) {
@@ -457,13 +458,22 @@ Transaction.prototype.run = function(callback) {
  *
  *   transaction.save({
  *     key: key,
- *     data: [
- *       {
- *         name: 'rating',
- *         value: '10',
- *         excludeFromIndexes: false
- *       }
- *     ]
+ *     excludeFromIndexes: [
+ *       'description',
+ *       'embeddedEntity.description',
+ *       'arrayValue[].description'
+ *     ],
+ *     data: {
+ *       description: 'Long string (...)',
+ *       embeddedEntity: {
+ *         description: 'Long string (...)'
+ *       },
+ *       arrayValue: [
+ *         {
+ *           description: 'Long string (...)'
+ *         }
+ *       ]
+ *     }
  *   });
  *
  *   transaction.commit(function(err) {

packages/datastore/system-test/datastore.js

@@ -81,6 +81,75 @@ describe('Datastore', function() {
       }
     };
 
+    it('should excludeFromIndexes correctly', function(done) {
+      var longString = Buffer.alloc(1501, '.').toString();
+      var postKey = datastore.key(['Post', 'post1']);
+
+      var data = {
+        longString: longString,
+        notMetadata: true,
+        metadata: {
+          longString: longString,
+          otherProperty: 'value',
+          obj: {
+            longStringArray: [
+              {
+                longString: longString,
+                nestedLongStringArray: [
+                  {
+                    longString: longString,
+                    nestedProperty: true
+                  },
+                  {
+                    longString: longString
+                  }
+                ]
+              }
+            ]
+          },
+          longStringArray: [
+            {
+              longString: longString,
+              nestedLongStringArray: [
+                {
+                  longString: longString,
+                  nestedProperty: true
+                },
+                {
+                  longString: longString
+                }
+              ]
+            }
+          ]
+        }
+      };
+
+      datastore.save({
+        key: postKey,
+        data: data,
+        excludeFromIndexes: [
+          'longString',
+          'metadata.obj.longString',
+          'metadata.obj.longStringArray[].longString',
+          'metadata.obj.longStringArray[].nestedLongStringArray[].longString',
+          'metadata.longString',
+          'metadata.longStringArray[].longString',
+          'metadata.longStringArray[].nestedLongStringArray[].longString'
+        ]
+      }, function(err) {
+        assert.ifError(err);
+
+        datastore.get(postKey, function(err, entity) {
+          assert.ifError(err);
+
+          assert.deepEqual(entity, data);
+          assert.deepEqual(entity[datastore.KEY], postKey);
+
+          datastore.delete(postKey, done);
+        });
+      });
+    });
+
     it('should save/get/delete with a key name', function(done) {
       var postKey = datastore.key(['Post', 'post1']);