datastore emulator invokes callback early with multiple save/delete

[rommguy]

Environment details

• OS: OSX 10.11.3
• Node.js version: 6.2.1
• npm version: 3.10.7
• google-cloud-node version:
  • Google Cloud SDK 128.0.0
  • beta 2016.01.12
  • bq 2.0.24
  • bq-nix 2.0.24
  • cloud-datastore-emulator 1.2.1
  • core 2016.09.23
  • core-nix 2016.06.24
  • gcd-emulator v1beta3-1.0.0
  • gcloud
  • gsutil 4.21
  • gsutil-nix 4.19

Steps to reproduce

1. require google-cloud
2. create datastore with project ID
3. call datastore.save with multiple entities
4. in the save callback, read the existing entities in the db
5. only some of the entities that were saved are there, number of existing entities looks random, usually not all of them have been created.

[stephenplusplus]

Thanks for reporting. Just to confirm, is this the latest version of google-cloud on npm, 0.41.2?

[rommguy]

Yes, updated version

[stephenplusplus]

Can you provide some sample code that I can use to reproduce?

[rommguy]

I wrote an isolated example -
DS module -

'use strict';
let gcloud = require('google-cloud');

let ds = gcloud.datastore({projectId: 'bug-example'});
const kind = 'myKind';

function createMultiple(count, cb) {
    let myEntities = [];
    for (let i = 0; i < count; i++) {
        myEntities.push({
            data: toDatastore({
                hello: 'world-' + i
            }),
            key: ds.key([kind])
        });
    }
    ds.save(myEntities, (err) => {
        cb(err, err ? null : myEntities);
    });
}

function list(limit, token, cb) {
    const q = ds.createQuery([kind])
        .limit(limit)
        .start(token);

    ds.runQuery(q, (err, entities, nextQuery) => {
        if (err) {
            return cb(err);
        }
        const hasMore = entities.length === limit ? nextQuery.startVal : false;
        return cb(null, entities.map(fromDatastore), hasMore);
    });
}

function fromDatastore(obj) {
    obj.data.id = obj.key.id;
    return obj.data;
}

function toDatastore(obj, nonIndexed) {
    nonIndexed = nonIndexed || [];
    let results = [];
    Object.keys(obj).forEach((k) => {
        if (obj[k] === undefined) {
            return;
        }
        results.push({
            name: k,
            value: obj[k],
            excludeFromIndexes: nonIndexed.indexOf(k) !== -1
        });
    });
    return results;
}

module.exports = {
    createMultiple,
    list
};

Failing test

'use strict';

const {describe, it} = require('mocha');
const {expect} = require('chai');

const ds = require('../../main/server/model-datastore-bug');

describe.only('model datastore', () => {
    it('should find any created entity when creating multiple entities', (done) => {
        ds.createMultiple(100, () => {
            ds.list(1000, null, (err, entities) => {
                expect(entities.length).to.equal(100);
                done();
            });
        });
    });
});

Test result (86 is random depends on the load the emulator is experiencing):
AssertionError: expected 86 to equal 100
Expected :100
Actual :86

[stephenplusplus]

Thank you! And confirmed.

@pcostell does the API response from saving data to the emulator "beat" the entities actually being available for querying?

[rommguy]

Thanks!
This bug makes testing db related code impossible, the tests are very flaky

[pcostell]

By default the Cloud Datastore Emulator emulates production Cloud Datastore semantics. In this case, you are running an eventually consistent query.

When testing db code, I would recommend starting the emulator with the --testing flag. This will start the emulator without eventual consistency which allows you to write non-flaky tests.

[stephenplusplus]

Thanks @pcostell!

[rommguy]

@pcostell The --testing flag is not recognised.
I can't find any mentions of --testing flags in the emulator documentation.
When trying to run with it - getting this error:
ERROR: (gcloud.beta.emulators.datastore.start) unrecognized arguments: --testing

@stephenplusplus maybe reopen this ?

After a long search, I eventually found in another issue's comments that --testing flag is equivalent to --consistency=1.0 and that worked.
Thanks.
Please update some kind of documentation about this