From 1f09a9c5d8d6c5742fe857133994cdda898018a8 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 12 Jun 2017 15:10:15 -0400 Subject: [PATCH 01/67] pubsub: api redesign --- packages/pubsub/package.json | 2 +- packages/pubsub/src/batch.js | 220 ++++++++ packages/pubsub/src/index.js | 540 +++++++------------- packages/pubsub/src/message.js | 90 ++++ packages/pubsub/src/publisher.js | 106 ++++ packages/pubsub/src/snapshot.js | 89 ++-- packages/pubsub/src/subscription.js | 709 +++++++------------------- packages/pubsub/src/topic.js | 525 ++++++------------- packages/pubsub/system-test/pubsub.js | 121 ++--- 9 files changed, 1028 insertions(+), 1374 deletions(-) create mode 100644 packages/pubsub/src/batch.js create mode 100644 packages/pubsub/src/message.js create mode 100644 packages/pubsub/src/publisher.js diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index 5f0c14e6422..8bde63a6141 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -59,7 +59,7 @@ "google-proto-files": "^0.12.0", "is": "^3.0.1", "modelo": "^4.2.0", - "propprop": "^0.3.0", + "propprop": "^0.3.1", "uuid": "^3.0.1" }, "devDependencies": { diff --git a/packages/pubsub/src/batch.js b/packages/pubsub/src/batch.js new file mode 100644 index 00000000000..1c2a0698802 --- /dev/null +++ b/packages/pubsub/src/batch.js @@ -0,0 +1,220 @@ +/*! + * Copyright 2014 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module pubsub/batch + */ + +'use strict'; + +var prop = require('propprop'); + +/** + * Semi-generic Batch object used to queue up multiple requests while limiting + * the number of concurrent requests. + * + * @constructor + * @alias module:pubsub/batch + * + * @param {object} options - Batch configuration settings. + * @param {number} options.maxBytes - Max size of data to be sent per request. + * @param {number} options.maxMessages - Max number of messages to be sent per + * request. + * @param {number} options.maxMilliseconds - Max time to wait before sending + * data. + * @param {number} options.maxRequests - Max number of concurrent requests + * allowed. + * @param {function(messages, callback)} options.sendHandler - Function to be + * called with message batch. + * + * @example + * var batch = new Batch({ + * maxBytes: Math.pow(1024, 2) * 5, + * maxMessages: 1000, + * maxMilliseconds: 1000, + * maxRequests: 5, + * send: function(messages, done) { + * var reqOpts = { + * topic: self.topic.name, + * messages = messages.map(function(message) { + * return { + * data: message.data, + * attributes: message.attrs + * }; + * }) + * }; + * + * return self.api.Publisher.publish(reqOpts, function(err, resp) { + * if (err) { + * done(err); + * return; + * } + * + * done(null, { + * responses: resp.messageIds + * }) + * }); + * } + * }); + */ +function Batch(options) { + this.maxBytes = options.maxBytes || Math.pow(1024, 2) * 5; + this.maxMessages = options.maxMessages || 1000; + this.maxMilliseconds = options.maxMilliseconds || 1000; + this.maxRequests = options.maxRequests || 5; + this.sendHandler = options.send; + + this.inventory = { + queued: new Set(), + inFlight: new Set() + }; + + Object.defineProperty(this.inventory, 'queueBytes', { + get: function() { + var size = 0; + + this.queued.forEach(function(message) { + size += message.size; + }); + + return size; + } + }); + + this.intervalHandle_ = null; + this.activeRequests_ = 0; +} + +/** + * Adds message to the queue. + * + * @param {*} data - The message data. + * @param {function} callback - Callback to be ran once message is sent. + * + * @example + * batch.add(1234, function(err, resp) {}); + */ +Batch.prototype.add = function(data, callback) { + this.inventory.queued.add({ + data: data, + callback: callback + }); + + var reachedMaxMessages = this.inventory.queued.size >= this.maxMessages; + var reachedMaxBytes = this.inventory.queueBytes >= this.maxBytes; + var reachedMaxRequests = this.activeRequests_ >= this.maxRequests; + + if ((reachedMaxMessages || reachedMaxBytes) && !reachedMaxRequests) { + this.send(this.getNextMessageBatch()); + } else if (!this.intervalHandle_) { + this.beginSending(); + } +}; + +/** + * Starts the send interval. + * + * @example + * batch.beginSending(); + */ +Batch.prototype.beginSending = function() { + if (this.intervalHandle_) { + return; + } + + var self = this; + + this.intervalHandle_ = setInterval(function() { + self.send(self.getNextMessageBatch()); + + if (self.inventory.queued.size === 0) { + clearInterval(self.intervalHandle_); + self.intervalHandle_ = null; + } + }, this.maxMilliseconds); +}; + +/** + * Prepares next batch of messages to be sent. + * + * @return array + * + * @example + * var messages = batch.getNextMessageBatch(); + * batch.send(messages); + */ +Batch.prototype.getNextMessageBatch = function() { + var size = 0; + var messages = []; + + var queued = this.inventory.queued[Symbol.iterator](); + var message; + + while (message = queued.next().value) { + var isOverSizeLimit = (size + message.size) > this.maxBytes; + var isOverMessageLimit = (messages.length + 1) > this.maxMessages; + + if (isOverSizeLimit || isOverMessageLimit) { + break; + } + + this.inventory.queued.delete(message); + messages.push(message); + } + + return messages; +}; + +/** + * Sends out messages + * + * @param {array} messages - Array of messages. + * + * @example + * var messages = [{ + * data: 123, + * callback: function(err, resp) {} + * }]; + * + * batch.send(messages); + */ +Batch.prototype.send = function(messages) { + if (this.activeRequests_ >= this.maxRequests) { + this.scheduleSend(); + return; + } + + var self = this; + + messages.forEach(function(message) { + self.inventory.inFlight.add(message); + }); + + this.activeRequests_ += 1; + + this.sendHandler(messages.map(prop('data')), function(err, resp) { + self.activeRequests_ -= 1; + + messages.forEach(function(message, i) { + var response = resp && resp.responses ? resp.responses[i] : resp; + + message.callback(err, response); + self.inventory.inFlight.delete(message); + }); + }); +}; + +module.exports = Batch; diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index a5b13483b11..4d5c6fe265a 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -20,14 +20,14 @@ 'use strict'; -var arrify = require('arrify'); var common = require('@google-cloud/common'); -var commonGrpc = require('@google-cloud/common-grpc'); var extend = require('extend'); var is = require('is'); var path = require('path'); var util = require('util'); +var v1 = require('./v1'); + /** * @type {module:pubsub/snapshot} * @private @@ -46,17 +46,6 @@ var Subscription = require('./subscription.js'); */ var Topic = require('./topic.js'); -/** - * @type {object} - GAX's default configuration. - * @private - */ -var GAX_CONFIG = { - Publisher: require('./v1/publisher_client_config.json'). - interfaces['google.pubsub.v1.Publisher'], - Subscriber: require('./v1/subscriber_client_config.json'). - interfaces['google.pubsub.v1.Subscriber'] -}; - /** * [Cloud Pub/Sub](https://developers.google.com/pubsub/overview) is a * reliable, many-to-many, asynchronous messaging service from Cloud @@ -80,35 +69,141 @@ function PubSub(options) { } this.defaultBaseUrl_ = 'pubsub.googleapis.com'; - this.determineBaseUrl_(options.apiEndpoint); - - var config = { - baseUrl: this.baseUrl_, - customEndpoint: this.customEndpoint_, - protosDir: path.resolve(__dirname, '../protos'), - protoServices: { - Publisher: { - path: 'google/pubsub/v1/pubsub.proto', - service: 'pubsub.v1' - }, - Subscriber: { - path: 'google/pubsub/v1/pubsub.proto', - service: 'pubsub.v1' - } - }, - scopes: [ - 'https://www.googleapis.com/auth/pubsub', - 'https://www.googleapis.com/auth/cloud-platform' - ], - packageJson: require('../package.json') + + if (options.servicePath) { + this.defaultBaseUrl_ = options.servicePath; + + if (options.port) { + this.defaultBaseUrl_ += ':' + options.port; + } + } + + this.api = { + Publisher: v1(options).publisherClient(options), + Subscriber: v1(options).subscriberClient(options) }; this.options = options; - - commonGrpc.Service.call(this, config, options); + this.projectId = options.projectId; } -util.inherits(PubSub, commonGrpc.Service); +/** + * Create a subscription to a topic. + * + * All generated subscription names share a common prefix, `autogenerated-`. + * + * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} + * + * @throws {Error} If a Topic instance or topic name is not provided. + * @throws {Error} If a subName is not provided. + * + * @param {module:pubsub/topic|string} topic - The Topic to create a + * subscription to. + * @param {string=} subName - The name of the subscription. If a name is not + * provided, a random subscription name will be generated and created. + * @param {object=} options - See a + * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + * @param {number} options.ackDeadlineSeconds - The maximum time after receiving + * a message that you must ack a message before it is redelivered. + * @param {string} options.encoding - When pulling for messages, this type is + * used when converting a message's data to a string. (default: 'utf-8') + * @param {number|date} options.messageRetentionDuration - Set this to override + * the default duration of 7 days. This value is expected in seconds. + * Acceptable values are in the range of 10 minutes and 7 days. + * @param {string} options.pushEndpoint - A URL to a custom endpoint that + * messages should be pushed to. + * @param {boolean} options.retainAckedMessages - If set, acked messages are + * retained in the subscription's backlog for 7 days (unless overriden by + * `options.messageRetentionDuration`). Default: `false` + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request + * @param {module:pubsub/subscription} callback.subscription - The subscription. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Subscribe to a topic. (Also see {module:pubsub/topic#subscribe}). + * //- + * var topic = 'messageCenter'; + * var name = 'newMessages'; + * + * var callback = function(err, subscription, apiResponse) {}; + * + * pubsub.createSubscription(topic, name, callback); + * + * //- + * // Omit the name to have one generated automatically. All generated names + * // share a common prefix, `autogenerated-`. + * //- + * pubsub.createSubscription(topic, function(err, subscription, apiResponse) { + * // subscription.name = The generated name. + * }); + * + * //- + * // Customize the subscription. + * //- + * pubsub.createSubscription(topic, name, { + * ackDeadlineSeconds: 90 + * }, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * pubsub.createSubscription(topic, name).then(function(data) { + * var subscription = data[0]; + * var apiResponse = data[1]; + * }); + */ +PubSub.prototype.createSubscription = function(topic, name, options, callback) { + if (!is.string(topic) && !(topic instanceof Topic)) { + throw new Error('A Topic is required for a new subscription.'); + } + + if (!is.string(name)) { + throw new Error('A subscription name is required.'); + } + + if (is.string(topic)) { + topic = this.topic(topic); + } + + if (is.fn(options)) { + callback = options; + options = {}; + } + + var subscription = this.subscription(name, options); + + var reqOpts = extend({ + topic: topic.name, + name: subscription.name + }, options); + + if (reqOpts.messageRetentionDuration) { + reqOpts.retainAckedMessages = true; + + reqOpts.messageRetentionDuration = { + seconds: reqOpts.messageRetentionDuration, + nanos: 0 + }; + } + + if (reqOpts.pushEndpoint) { + reqOpts.pushConfig = { + pushEndpoint: reqOpts.pushEndpoint + }; + } + + this.api.Subscriber.createSubscription(reqOpts, function(err, resp) { + if (err && err.code !== 6) { + callback(err, null, resp); + return; + } + + subscription.metadata = resp; + callback(null, subscription, resp); + }); +}; /** * Create a topic with the given name. @@ -138,28 +233,19 @@ util.inherits(PubSub, commonGrpc.Service); * }); */ PubSub.prototype.createTopic = function(name, callback) { - var self = this; - - callback = callback || common.util.noop; - - var protoOpts = { - service: 'Publisher', - method: 'createTopic' - }; + var topic = this.topic(name); var reqOpts = { - name: Topic.formatName_(this.projectId, name) + name: topic.name }; - this.request(protoOpts, reqOpts, function(err, resp) { + this.api.Publisher.createTopic(reqOpts, function(err, resp) { if (err) { callback(err, null, resp); return; } - var topic = self.topic(name); topic.metadata = resp; - callback(null, topic, resp); }); }; @@ -216,35 +302,22 @@ PubSub.prototype.getSnapshots = function(options, callback) { options = {}; } - var protoOpts = { - service: 'Subscriber', - method: 'listSnapshots' - }; - - var reqOpts = extend({}, options); - - reqOpts.project = 'projects/' + this.projectId; - - this.request(protoOpts, reqOpts, function(err, resp) { - if (err) { - callback(err, null, null, resp); - return; - } - - var snapshots = arrify(resp.snapshots).map(function(snapshot) { - var snapshotInstance = self.snapshot(snapshot.name); - snapshotInstance.metadata = snapshot; - return snapshotInstance; - }); + var reqOpts = extend({ + project: 'projects/' + this.projectId + }, options); - var nextQuery = null; + this.api.Subscriber.listSnapshots(reqOpts, function() { + var snapshots = arguments[1]; - if (resp.nextPageToken) { - nextQuery = options; - nextQuery.pageToken = resp.nextPageToken; + if (snapshots) { + arguments[1] = snapshots.map(function(snapshot) { + var snapshotInstance = self.snapshot(snapshot.name); + snapshotInstance.metadata = snapshot; + return snapshotInstance; + }); } - callback(null, snapshots, nextQuery, resp); + callback.apply(null, arguments); }); }; @@ -343,56 +416,39 @@ PubSub.prototype.getSubscriptions = function(options, callback) { options = {}; } - var protoOpts = {}; var reqOpts = extend({}, options); + var method; if (options.topic) { - protoOpts = { - service: 'Publisher', - method: 'listTopicSubscriptions' - }; - if (options.topic instanceof Topic) { reqOpts.topic = options.topic.name; - } else { - reqOpts.topic = options.topic; } - } else { - protoOpts = { - service: 'Subscriber', - method: 'listSubscriptions' - }; + method = this.api.Publisher.listTopicSubscriptions.bind(this.api.Publisher); + } else { reqOpts.project = 'projects/' + this.projectId; + method = this.api.Subscriber.listSubscriptions.bind(this.api.Subscriber); } - this.request(protoOpts, reqOpts, function(err, resp) { - if (err) { - callback(err, null, null, resp); - return; - } - - var subscriptions = arrify(resp.subscriptions).map(function(sub) { - // Depending on if we're using a subscriptions.list or - // topics.subscriptions.list API endpoint, we will get back a - // Subscription resource or just the name of the subscription. - var subscriptionInstance = self.subscription(sub.name || sub); - - if (sub.name) { - subscriptionInstance.metadata = sub; - } + method(reqOpts, function() { + var subscriptions = arguments[1]; - return subscriptionInstance; - }); + if (subscriptions) { + arguments[1] = subscriptions.map(function(sub) { + // Depending on if we're using a subscriptions.list or + // topics.subscriptions.list API endpoint, we will get back a + // Subscription resource or just the name of the subscription. + var subscriptionInstance = self.subscription(sub.name || sub); - var nextQuery = null; + if (sub.name) { + subscriptionInstance.metadata = sub; + } - if (resp.nextPageToken) { - nextQuery = options; - nextQuery.pageToken = resp.nextPageToken; + return subscriptionInstance; + }); } - callback(null, subscriptions, nextQuery, resp); + callback.apply(null, arguments); }); }; @@ -481,42 +537,34 @@ PubSub.prototype.getSubscriptionsStream = * var topics = data[0]; * }); */ -PubSub.prototype.getTopics = function(query, callback) { +PubSub.prototype.getTopics = function(options, callback) { var self = this; - if (!callback) { - callback = query; - query = {}; + if (is.fn(options)) { + callback = options; + options = {}; } - var protoOpts = { - service: 'Publisher', - method: 'listTopics' - }; - var reqOpts = extend({ project: 'projects/' + this.projectId - }, query); + }, options); - this.request(protoOpts, reqOpts, function(err, result) { - if (err) { - callback(err, null, result); - return; - } + var options = { + autoPaginate: !is.number(reqOpts.pageSize) + }; - var topics = arrify(result.topics).map(function(topic) { - var topicInstance = self.topic(topic.name); - topicInstance.metadata = topic; - return topicInstance; - }); + this.api.Publisher.listTopics(reqOpts, options, function() { + var topics = arguments[1]; - var nextQuery = null; - if (result.nextPageToken) { - nextQuery = query; - nextQuery.pageToken = result.nextPageToken; + if (topics) { + arguments[1] = topics.map(function(topic) { + var topicInstance = self.topic(topic.name); + topicInstance.metadata = topic; + return topicInstance; + }); } - callback(null, topics, nextQuery, result); + callback.apply(null, arguments); }); }; @@ -549,154 +597,6 @@ PubSub.prototype.getTopics = function(query, callback) { */ PubSub.prototype.getTopicsStream = common.paginator.streamify('getTopics'); -/** - * Create a subscription to a topic. - * - * All generated subscription names share a common prefix, `autogenerated-`. - * - * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} - * - * @throws {Error} If a Topic instance or topic name is not provided. - * @throws {Error} If a subName is not provided. - * - * @param {module:pubsub/topic|string} topic - The Topic to create a - * subscription to. - * @param {string=} subName - The name of the subscription. If a name is not - * provided, a random subscription name will be generated and created. - * @param {object=} options - See a - * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadlineSeconds - The maximum time after receiving - * a message that you must ack a message before it is redelivered. - * @param {boolean} options.autoAck - Automatically acknowledge the message once - * it's pulled. (default: false) - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') - * @param {number} options.interval - Interval in milliseconds to check for new - * messages. (default: 10) - * @param {number} options.maxInProgress - Maximum messages to consume - * simultaneously. - * @param {number|date} options.messageRetentionDuration - Set this to override - * the default duration of 7 days. This value is expected in seconds. - * Acceptable values are in the range of 10 minutes and 7 days. - * @param {string} options.pushEndpoint - A URL to a custom endpoint that - * messages should be pushed to. - * @param {boolean} options.retainAckedMessages - If set, acked messages are - * retained in the subscription's backlog for 7 days (unless overriden by - * `options.messageRetentionDuration`). Default: `false` - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds on an HTTP request to pull new messages to wait for a - * response before the connection is broken. - * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this request - * @param {module:pubsub/subscription} callback.subscription - The subscription. - * @param {object} callback.apiResponse - The full API response. - * - * @example - * //- - * // Subscribe to a topic. (Also see {module:pubsub/topic#subscribe}). - * //- - * var topic = 'messageCenter'; - * var name = 'newMessages'; - * - * pubsub.subscribe(topic, name, function(err, subscription, apiResponse) {}); - * - * //- - * // Omit the name to have one generated automatically. All generated names - * // share a common prefix, `autogenerated-`. - * //- - * pubsub.subscribe(topic, function(err, subscription, apiResponse) { - * // subscription.name = The generated name. - * }); - * - * //- - * // Customize the subscription. - * //- - * pubsub.subscribe(topic, name, { - * ackDeadlineSeconds: 90, - * autoAck: true, - * interval: 30 - * }, function(err, subscription, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * pubsub.subscribe(topic, name).then(function(data) { - * var subscription = data[0]; - * var apiResponse = data[1]; - * }); - */ -PubSub.prototype.subscribe = function(topic, subName, options, callback) { - if (!is.string(topic) && !(topic instanceof Topic)) { - throw new Error('A Topic is required for a new subscription.'); - } - - if (is.string(topic)) { - topic = this.topic(topic); - } - - if (is.object(subName)) { - callback = options; - options = subName; - subName = ''; - } - - if (is.fn(subName)) { - callback = subName; - subName = ''; - } - - if (is.fn(options)) { - callback = options; - options = {}; - } - - options = options || {}; - - var subscription = this.subscription(subName, options); - - var protoOpts = { - service: 'Subscriber', - method: 'createSubscription', - timeout: options.timeout - }; - - var reqOpts = extend(true, {}, options, { - topic: topic.name, - name: subscription.name - }); - - if (reqOpts.messageRetentionDuration) { - reqOpts.retainAckedMessages = true; - - reqOpts.messageRetentionDuration = { - seconds: reqOpts.messageRetentionDuration, - nanos: 0 - }; - } - - if (reqOpts.pushEndpoint) { - reqOpts.pushConfig = { - pushEndpoint: reqOpts.pushEndpoint - }; - } - - delete reqOpts.autoAck; - delete reqOpts.encoding; - delete reqOpts.interval; - delete reqOpts.maxInProgress; - delete reqOpts.pushEndpoint; - delete reqOpts.timeout; - - this.request(protoOpts, reqOpts, function(err, resp) { - if (err && err.code !== 409) { - callback(err, null, resp); - return; - } - - callback(null, subscription, resp); - }); -}; - /** * Create a Snapshot object. See {module:pubsub/subscription#createSnapshot} to * create a snapshot. @@ -727,17 +627,8 @@ PubSub.prototype.snapshot = function(name) { * @param {string=} name - The name of the subscription. If a name is not * provided, a random subscription name will be generated. * @param {object=} options - Configuration object. - * @param {boolean} options.autoAck - Automatically acknowledge the message once - * it's pulled. (default: false) * @param {string} options.encoding - When pulling for messages, this type is * used when converting a message's data to a string. (default: 'utf-8') - * @param {number} options.interval - Interval in milliseconds to check for new - * messages. (default: 10) - * @param {number} options.maxInProgress - Maximum messages to consume - * simultaneously. - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds on an HTTP request to pull new messages to wait for a - * response before the connection is broken. * @return {module:pubsub/subscription} * * @example @@ -749,20 +640,16 @@ PubSub.prototype.snapshot = function(name) { * // message.id = ID of the message. * // message.ackId = ID used to acknowledge the message receival. * // message.data = Contents of the message. - * // message.attributes = Attributes of the message. - * // message.timestamp = Timestamp when Pub/Sub received the message. + * // message.attrs = Attributes of the message. + * // message.publishTime = Timestamp when Pub/Sub received the message. * }); */ PubSub.prototype.subscription = function(name, options) { - if (is.object(name)) { - options = name; - name = undefined; + if (!name) { + throw new Error('A name must be specified for a subscription.'); } - options = options || {}; - options.name = name; - - return new Subscription(this, options); + return new Subscription(this, name, options); }; /** @@ -780,53 +667,12 @@ PubSub.prototype.subscription = function(name, options) { * data: 'New message!' * }, function(err) {}); */ -PubSub.prototype.topic = function(name) { +PubSub.prototype.topic = function(name, options) { if (!name) { - throw new Error('A name must be specified for a new topic.'); + throw new Error('A name must be specified for a topic.'); } - return new Topic(this, name); -}; - -/** - * Intercept the call to {module:common/grpc-service#request}, making sure the - * correct timeouts are set. - * - * @private - */ -PubSub.prototype.request = function(protoOpts) { - var method = protoOpts.method; - var camelCaseMethod = method[0].toUpperCase() + method.substr(1); - var config = GAX_CONFIG[protoOpts.service].methods[camelCaseMethod]; - - if (is.undefined(arguments[0].timeout)) { - arguments[0].timeout = config.timeout_millis; - } - - commonGrpc.Service.prototype.request.apply(this, arguments); -}; - -/** - * Determine the appropriate endpoint to use for API requests, first trying the - * local `apiEndpoint` parameter. If the `apiEndpoint` parameter is null we try - * Pub/Sub emulator environment variable (PUBSUB_EMULATOR_HOST), otherwise the - * default JSON API. - * - * @private - */ -PubSub.prototype.determineBaseUrl_ = function(apiEndpoint) { - var baseUrl = this.defaultBaseUrl_; - var leadingProtocol = new RegExp('^https*://'); - var trailingSlashes = new RegExp('/*$'); - - if (apiEndpoint || process.env.PUBSUB_EMULATOR_HOST) { - this.customEndpoint_ = true; - baseUrl = apiEndpoint || process.env.PUBSUB_EMULATOR_HOST; - } - - this.baseUrl_ = baseUrl - .replace(leadingProtocol, '') - .replace(trailingSlashes, ''); + return new Topic(this, name, options); }; /*! Developer Documentation @@ -846,15 +692,11 @@ common.paginator.extend(PubSub, [ */ common.util.promisifyAll(PubSub, { exclude: [ - 'request', 'snapshot', 'subscription', 'topic' ] }); -PubSub.Subscription = Subscription; -PubSub.Topic = Topic; - module.exports = PubSub; -module.exports.v1 = require('./v1'); +module.exports.v1 = v1; diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js new file mode 100644 index 00000000000..93bfff726ab --- /dev/null +++ b/packages/pubsub/src/message.js @@ -0,0 +1,90 @@ +/*! + * Copyright 2014 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module pubsub/message + */ + +'use strict'; + +var common = require('@google-cloud/common'); + +/** + * + */ +function Message(subscription, resp) { + this.subscription = subscription; + this.api = subscription.api; + + this.ackId = resp.ackId; + + this.id = resp.message.messageId; + this.data = resp.message.data; + this.attrs = resp.message.attributes; + + var pt = resp.message.publishTime; + var milliseconds = parseInt(pt.nanos, 10) / 1e6; + + this.publishTime = new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds); +} + +/** + * + */ +Message.prototype.ack = function(callback) { + this.subscription.ack(this.ackId, callback) +}; + +/** + * + */ +Message.prototype.modifyAckDeadline = function(milliseconds, callback) { + var seconds = milliseconds / 1000; + + if (this.subscription.connection) { + this.subscription.connection.write({ + modifyDeadlineAckIds: [this.ackId], + modifyDeadlineSeconds: [seconds] + }); + return; + } + + callback = callback || common.util.noop; + + reqOpts = { + subscription: this.subscription.name, + ackIds: [this.ackId], + ackDeadlineSeconds: seconds + }; + + this.api.Subscriber.modifyAckDeadline(reqOpts, callback); +}; + +/** + * + */ +Message.prototype.nack = function(callback) { + this.modifyAckDeadline(0, callback); +}; + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +common.util.promisifyAll(Message); + +module.exports = Message; diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js new file mode 100644 index 00000000000..653b0c5ed25 --- /dev/null +++ b/packages/pubsub/src/publisher.js @@ -0,0 +1,106 @@ +/*! + * Copyright 2014 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module pubsub/publisher + */ + +'use strict'; + +var arrify = require('arrify'); +var common = require('@google-cloud/common'); +var extend = require('extend'); +var is = require('is'); + +/** + * + */ +var Batch = require('./batch.js'); + +/** + * + */ +function Publisher(topic, options) { + options = options || {}; + + this.topic = topic; + this.api = topic.api; + + var batchOptions = extend(options.batching, { + send: this.publish_.bind(this) + }); + + this.batch_ = new Batch(batchOptions); +} + +/** + * + */ +Publisher.prototype.publish = function(data, attrs, callback) { + if (!(data instanceof Buffer)) { + throw new Error('Data must be in the form of a Buffer.'); + } + + if (is.fn(attrs)) { + callback = attrs; + attrs = {}; + } + + var message = { + data: data, + attrs: attrs, + size: data.length + }; + + this.batch_.add(message, callback); +}; + +/** + * This should never be called directly. + */ +Publisher.prototype.publish_ = function(messages, done) { + var self = this; + + var reqOpts = { + topic: this.topic.name, + messages: messages.map(function(message) { + return { + data: message.data, + attributes: message.attrs + }; + }) + }; + + this.api.Publisher.publish(reqOpts, function(err, resp) { + if (err) { + done(err); + return; + } + + done(null, { + responses: resp.messageIds + }); + }); +}; + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +common.util.promisifyAll(Publisher); + +module.exports = Publisher; diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index 4a578191e49..05ad116b00b 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -20,9 +20,7 @@ 'use strict'; -var commonGrpc = require('@google-cloud/common-grpc'); var is = require('is'); -var util = require('util'); /** * A Snapshot object will give you access to your Cloud Pub/Sub snapshot. @@ -90,56 +88,10 @@ var util = require('util'); * }); */ function Snapshot(parent, name) { - var projectId = parent.projectId; - - if (!projectId && parent.parent) { - projectId = parent.parent.projectId; - } - - this.name = Snapshot.formatName_(projectId, name); - - var methods = { - /** - * Delete the snapshot. - * - * @param {function=} callback - The callback function. - * @param {?error} callback.err - An error returned while making this - * request. - * @param {object} callback.apiResponse - The full API response from the - * service. - * - * @example - * snapshot.delete(function(err, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * snapshot.delete().then(function(data) { - * var apiResponse = data[0]; - * }); - */ - delete: { - protoOpts: { - service: 'Subscriber', - method: 'deleteSnapshot' - }, - reqOpts: { - snapshot: this.name - } - } - }; - - var config = { - parent: parent, - id: this.name, - methods: methods - }; - - var isSubscription = is.fn(parent.createSnapshot); - - if (isSubscription) { - config.createMethod = parent.createSnapshot.bind(parent); + this.api = parent.api; + this.name = Snapshot.formatName_(parent.projectId, name); + if (is.fn(parent.createSnapshot)) { /** * Create a snapshot with the given name. * @@ -174,8 +126,10 @@ function Snapshot(parent, name) { * var apiResponse = data[1]; * }); */ - methods.create = true; + this.create = parent.createSnapshot.bind(parent, name); + } + if (is.fn(parent.seek)) { /** * Seeks an existing subscription to the snapshot. * @@ -202,12 +156,8 @@ function Snapshot(parent, name) { */ this.seek = parent.seek.bind(parent, name); } - - commonGrpc.ServiceObject.call(this, config); } -util.inherits(Snapshot, commonGrpc.ServiceObject); - /** * Format the name of a snapshot. A snapshot's full name is in the format of * projects/{projectId}/snapshots/{snapshotName} @@ -218,4 +168,31 @@ Snapshot.formatName_ = function(projectId, name) { return 'projects/' + projectId + '/snapshots/' + name.split('/').pop(); }; +/** + * Delete the snapshot. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response from the + * service. + * + * @example + * snapshot.delete(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * snapshot.delete().then(function(data) { + * var apiResponse = data[0]; + * }); + */ +Snapshot.prototype.delete = function(callback) { + var reqOpts = { + snapshot: this.name + }; + + this.api.Subscriber.deleteSnapshot(reqOpts, callback); +}; + module.exports = Snapshot; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 91847efe974..3e74eb7d93f 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -22,12 +22,16 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); -var commonGrpc = require('@google-cloud/common-grpc'); var events = require('events'); +var extend = require('extend'); var is = require('is'); -var modelo = require('modelo'); -var prop = require('propprop'); -var uuid = require('uuid'); +var util = require('util'); + +/** + * @type {module:pubsub/batch} + * @private + */ +var Batch = require('./batch.js'); /** * @type {module:pubsub/iam} @@ -36,34 +40,24 @@ var uuid = require('uuid'); var IAM = require('./iam.js'); /** - * @type {module:pubsub/snapshot} + * @type {module:pubsub/message} * @private */ -var Snapshot = require('./snapshot.js'); +var Message = require('./message.js'); /** - * @const {number} - The amount of time a subscription pull HTTP connection to - * Pub/Sub stays open. + * @type {module:pubsub/snapshot} * @private */ -var PUBSUB_API_TIMEOUT = 90000; +var Snapshot = require('./snapshot.js'); /*! Developer Documentation * * @param {module:pubsub} pubsub - PubSub object. * @param {object} options - Configuration object. - * @param {boolean} options.autoAck - Automatically acknowledge the message once - * it's pulled. (default: false) * @param {string} options.encoding - When pulling for messages, this type is * used when converting a message's data to a string. (default: 'utf-8') - * @param {number} options.interval - Interval in milliseconds to check for new - * messages. (default: 10) * @param {string} options.name - Name of the subscription. - * @param {number} options.maxInProgress - Maximum messages to consume - * simultaneously. - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds on an HTTP request to pull new messages to wait for a - * response before the connection is broken. (default: 90000) */ /** * A Subscription object will give you access to your Cloud Pub/Sub @@ -149,154 +143,22 @@ var PUBSUB_API_TIMEOUT = 90000; * // Remove the listener from receiving `message` events. * subscription.removeListener('message', onMessage); */ -function Subscription(pubsub, options) { - var name = options.name || Subscription.generateName_(); - - this.name = Subscription.formatName_(pubsub.projectId, name); - - var methods = { - /** - * Check if the subscription exists. - * - * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this - * request. - * @param {boolean} callback.exists - Whether the subscription exists or - * not. - * - * @example - * subscription.exists(function(err, exists) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.exists().then(function(data) { - * var exists = data[0]; - * }); - */ - exists: true, - - /** - * Get a subscription if it exists. - * - * You may optionally use this to "get or create" an object by providing an - * object with `autoCreate` set to `true`. Any extra configuration that is - * normally required for the `create` method must be contained within this - * object as well. - * - * **`autoCreate` is only available if you accessed this object - * through {module:pubsub/topic#subscription}.** - * - * @param {options=} options - Configuration object. - * @param {boolean} options.autoCreate - Automatically create the object if - * it does not exist. Default: `false` - * - * @example - * subscription.get(function(err, subscription, apiResponse) { - * // `subscription.metadata` has been populated. - * }); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.get().then(function(data) { - * var subscription = data[0]; - * var apiResponse = data[1]; - * }); - */ - get: true, - - /** - * Get the metadata for the subscription. - * - * @resource [Subscriptions: get API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/get} - * - * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this - * request. - * @param {?object} callback.metadata - Metadata of the subscription from - * the API. - * @param {object} callback.apiResponse - Raw API response. - * - * @example - * subscription.getMetadata(function(err, metadata, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.getMetadata().then(function(data) { - * var metadata = data[0]; - * var apiResponse = data[1]; - * }); - */ - getMetadata: { - protoOpts: { - service: 'Subscriber', - method: 'getSubscription' - }, - reqOpts: { - subscription: this.name - } - } - }; +function Subscription(pubsub, name, options) { + options = options || {}; - var config = { - parent: pubsub, - id: this.name, - methods: methods - }; + this.pubsub = pubsub; + this.api = pubsub.api; + this.projectId = pubsub.projectId; - if (options.topic) { - // Only a subscription with knowledge of its topic can be created. - config.createMethod = pubsub.subscribe.bind(pubsub, options.topic); - delete options.topic; - - /** - * Create a subscription. - * - * **This is only available if you accessed this object through - * {module:pubsub/topic#subscription}.** - * - * @param {object} config - See {module:pubsub#subscribe}. - * - * @example - * subscription.create(function(err, subscription, apiResponse) { - * if (!err) { - * // The subscription was created successfully. - * } - * }); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.create().then(function(data) { - * var subscription = data[0]; - * var apiResponse = data[1]; - * }); - */ - config.methods.create = true; - } + this.name = Subscription.formatName_(pubsub.projectId, name); + this.connection = null; + this.encoding = options.encoding || 'utf-8'; + this.ackDeadline = options.ackDeadline || 600; - commonGrpc.ServiceObject.call(this, config); events.EventEmitter.call(this); - this.autoAck = is.boolean(options.autoAck) ? options.autoAck : false; - this.closed = true; - this.encoding = options.encoding || 'utf-8'; - this.inProgressAckIds = {}; - this.interval = is.number(options.interval) ? options.interval : 10; - this.maxInProgress = - is.number(options.maxInProgress) ? options.maxInProgress : Infinity; - this.messageListeners = 0; - this.paused = false; - - if (is.number(options.timeout)) { - this.timeout = options.timeout; - } else { - // The default timeout used in google-cloud-node is 60s, but a pull request - // times out around 90 seconds. Allow an extra couple of seconds to give the - // API a chance to respond on its own before terminating the connection. - this.timeout = PUBSUB_API_TIMEOUT + 2000; + if (options.topic) { + this.create = pubsub.createSubscription.bind(pubsub, options.topic, name); } /** @@ -335,57 +197,15 @@ function Subscription(pubsub, options) { */ this.iam = new IAM(pubsub, this.name); + var batchOptions = extend(options.batching, { + send: this.ack_.bind(this) + }); + + this.batch_ = new Batch(batchOptions); this.listenForEvents_(); } -modelo.inherits(Subscription, commonGrpc.ServiceObject, events.EventEmitter); - -/** - * Simplify a message from an API response to have five properties: `id`, - * `ackId`, `data`, `attributes`, and `timestamp`. `data` is always converted to - * a string. - * - * @private - */ -Subscription.formatMessage_ = function(msg, enc) { - var innerMessage = msg.message; - var message = { - ackId: msg.ackId - }; - - if (innerMessage) { - message.id = innerMessage.messageId; - - if (innerMessage.data) { - if (enc === 'base64') { - // Prevent decoding and then re-encoding to base64. - message.data = innerMessage.data; - } else { - message.data = Buffer.from(innerMessage.data, 'base64').toString(enc); - - try { - message.data = JSON.parse(message.data); - } catch(e) {} - } - } - - if (innerMessage.attributes) { - message.attributes = innerMessage.attributes; - } - - if (innerMessage.publishTime) { - var publishTime = innerMessage.publishTime; - - if (is.defined(publishTime.seconds) && is.defined(publishTime.nanos)) { - var seconds = parseInt(publishTime.seconds, 10); - var milliseconds = parseInt(publishTime.nanos, 10) / 1e6; - message.timestamp = new Date(seconds * 1000 + milliseconds); - } - } - } - - return message; -}; +util.inherits(Subscription, events.EventEmitter); /** * Format the name of a subscription. A subscription's full name is in the @@ -403,84 +223,41 @@ Subscription.formatName_ = function(projectId, name) { }; /** - * Generate a random name to use for a name-less subscription. - * - * @private - */ -Subscription.generateName_ = function() { - return 'autogenerated-' + uuid.v4(); -}; - -/** - * Acknowledge to the backend that the message was retrieved. You must provide - * either a single ackId or an array of ackIds. - * - * @resource [Subscriptions: acknowledge API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/acknowledge} - * - * @throws {Error} If at least one ackId is not provided. - * - * @param {string|string[]} ackIds - An ackId or array of ackIds. - * @param {options=} options - Configuration object. - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds before giving up if no response is received. - * @param {function=} callback - The callback function. - * - * @example - * var ackId = 'ePHEESyhuE8e...'; - * - * subscription.ack(ackId, function(err, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.ack(ackId).then(function(data) { - * var apiResponse = data[0]; - * }); + * This should never be called directly. */ -Subscription.prototype.ack = function(ackIds, options, callback) { - var self = this; - - ackIds = arrify(ackIds); - - if (ackIds.length === 0) { - throw new Error([ - 'At least one ID must be specified before it can be acknowledged.' - ].join('')); - } +Subscription.prototype.ack_ = function(messages, callback) { + var ackIds = messages.reduce(function(ackIds, message) { + return ackIds.concat(message); + }, []); - if (is.fn(options)) { - callback = options; - options = {}; - } + if (this.connection) { + this.connection.write({ ackIds: ackIds }); - options = options || {}; - callback = callback || common.util.noop; - - var protoOpts = { - service: 'Subscriber', - method: 'acknowledge' - }; - - if (options && is.number(options.timeout)) { - protoOpts.timeout = options.timeout; + if (is.fn(callback)) { + setImmediate(callback); + } + return; } - var reqOpts = { - subscription: this.name, - ackIds: ackIds - }; - - this.parent.request(protoOpts, reqOpts, function(err, resp) { - if (!err) { - ackIds.forEach(function(ackId) { - delete self.inProgressAckIds[ackId]; - }); + this.api.Subscriber.acknowledge({ + ackIds: ackIds, + subscription: this.name + }, callback); +}; - self.refreshPausedStatus_(); - } +/** + * + */ +Subscription.prototype.ack = function(ackIds, callback) { + this.batch_.add(arrify(ackIds), callback); +}; - callback(err, resp); - }); +/** + * @private + */ +Subscription.prototype.closeConnection_ = function() { + this.connection.end(); + this.connection = null; }; /** @@ -518,59 +295,24 @@ Subscription.prototype.createSnapshot = function(name, callback) { throw new Error('A name is required to create a snapshot.'); } - var protoOpts = { - service: 'Subscriber', - method: 'createSnapshot' - }; + var snapshot = self.snapshot(name); var reqOpts = { - name: Snapshot.formatName_(this.parent.projectId, name), + name: snapshot.name, subscription: this.name }; - this.parent.request(protoOpts, reqOpts, function(err, resp) { + this.api.Subscriber.createSnapshot(reqOpts, function(err, resp) { if (err) { callback(err, null, resp); return; } - var snapshot = self.snapshot(name); snapshot.metadata = resp; - callback(null, snapshot, resp); }); }; -/** - * Add functionality on top of a message returned from the API, including the - * ability to `ack` and `skip` the message. - * - * This also records the message as being "in progress". See - * {module:subscription#refreshPausedStatus_}. - * - * @private - * - * @param {object} message - A message object. - * @return {object} message - The original message after being decorated. - * @param {function} message.ack - Ack the message. - * @param {function} message.skip - Increate the number of available messages to - * simultaneously receive. - */ -Subscription.prototype.decorateMessage_ = function(message) { - var self = this; - - this.inProgressAckIds[message.ackId] = true; - - message.ack = self.ack.bind(self, message.ackId); - - message.skip = function() { - delete self.inProgressAckIds[message.ackId]; - self.refreshPausedStatus_(); - }; - - return message; -}; - /** * Delete the subscription. Pull requests from the current subscription will be * errored once unsubscription is complete. @@ -595,30 +337,116 @@ Subscription.prototype.decorateMessage_ = function(message) { Subscription.prototype.delete = function(callback) { var self = this; - callback = callback || common.util.noop; - - var protoOpts = { - service: 'Subscriber', - method: 'deleteSubscription' + var reqOpts = { + subscription: this.name }; + this.api.Subscriber.deleteSubscription(reqOpts, function(err, resp) { + if (!err) { + self.closed = true; + self.removeAllListeners(); + } + + callback(err, resp); + }); +}; + +/** + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - Raw API response. + */ +Subscription.prototype.getMetadata = function(callback) { var reqOpts = { subscription: this.name }; - this.parent.request(protoOpts, reqOpts, function(err, resp) { - if (err) { - callback(err, resp); - return; - } + this.api.Subscriber.getSubscription(reqOpts, callback); +}; + +/** + * Begin listening for events on the subscription. This method keeps track of + * how many message listeners are assigned, and then removed, making sure + * polling is handled automatically. + * + * As long as there is one active message listener, the connection is open. As + * soon as there are no more message listeners, the connection is closed. + * + * @private + * + * @example + * subscription.listenForEvents_(); + */ +Subscription.prototype.listenForEvents_ = function() { + var self = this; + + this.on('newListener', function(event) { + if (event === 'message') { + self.messageListeners++; - self.closed = true; - self.removeAllListeners(); + if (!self.connection) { + self.openConnection_(); + } + } + }); - callback(null, resp); + this.on('removeListener', function(event) { + if (event === 'message' && --self.messageListeners === 0) { + self.closeConnection_(); + } }); }; +/** + * @param {object} config - The push config. + * @param {string} config.pushEndpoint + * @param {object} config.attributes + */ +Subscription.prototype.modifyPushConfig = function(config, callback) { + var reqOpts = { + subscription: this.name, + pushConfig: config + }; + + this.api.Subscriber.modifyPushConfig(reqOpts, callback); +}; + +/** + * @private + */ +Subscription.prototype.openConnection_ = function() { + var self = this; + + var reqOpts = { + subscription: this.name, + streamAckDeadlineSeconds: this.ackDeadline + }; + + var grpcOpts = { + timeout: Number.MAX_NUMBER + }; + + this.connection = this.api.Subscriber.streamingPull(grpcOpts) + .on('error', function(err) { + var retryCodes = [2, 4, 14]; + + if (retryCodes.indexOf(err.code) > -1) { + self.openConnection_(); + return; + } + + self.emit('error', err); + }) + .on('data', function(data) { + data.receivedMessages.forEach(function(message) { + self.emit('message', new Message(self, message)); + }); + }); + + this.connection.write(reqOpts); +}; + /** * Pull messages from the subscribed topic. If messages were found, your * callback is executed with an array of message objects. @@ -633,7 +461,7 @@ Subscription.prototype.delete = function(callback) { * @resource [Subscriptions: pull API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull} * * @param {object=} options - Configuration object. - * @param {number} options.maxResults - Limit the amount of messages pulled. + * @param {number} options.maxMessages - Limit the amount of messages pulled. * @param {boolean} options.returnImmediately - If set, the system will respond * immediately. Otherwise, wait until new messages are available. Returns if * timeout is reached. @@ -663,7 +491,7 @@ Subscription.prototype.delete = function(callback) { * // Pull a single message. * //- * var opts = { - * maxResults: 1 + * maxMessages: 1 * }; * * subscription.pull(opts, function(err, messages, apiResponse) {}); @@ -678,63 +506,34 @@ Subscription.prototype.delete = function(callback) { */ Subscription.prototype.pull = function(options, callback) { var self = this; - var MAX_EVENTS_LIMIT = 1000; - if (!callback) { + if (is.fn(options)) { callback = options; options = {}; } - if (!is.number(options.maxResults)) { - options.maxResults = MAX_EVENTS_LIMIT; - } - - var protoOpts = { - service: 'Subscriber', - method: 'pull', - timeout: this.timeout - }; - - var reqOpts = { + var reqOpts = extend({ subscription: this.name, - returnImmediately: !!options.returnImmediately, - maxMessages: options.maxResults - }; - - this.activeRequest_ = this.parent.request(protoOpts, reqOpts, function(err) { - self.activeRequest_ = null; - - var resp = arguments[1]; + }, options); + this.api.Subscriber.pull(reqOpts, function(err, resp) { if (err) { - if (err.code === 504) { - // Simulate a server timeout where no messages were received. - resp = { - receivedMessages: [] - }; - } else { + if (err.code !== 504) { callback(err, null, resp); return; } - } - - var messages = arrify(resp.receivedMessages) - .map(function(msg) { - return Subscription.formatMessage_(msg, self.encoding); - }) - .map(self.decorateMessage_.bind(self)); - self.refreshPausedStatus_(); + // Simulate a server timeout where no messages were received. + resp = { + receivedMessages: [] + }; + } - if (self.autoAck && messages.length !== 0) { - var ackIds = messages.map(prop('ackId')); + var messages = arrify(resp.receivedMessages).map(function(message) { + return new Message(self, message); + }); - self.ack(ackIds, function(err) { - callback(err, messages, resp); - }); - } else { - callback(null, messages, resp); - } + callback(null, messages, resp); }); }; @@ -766,11 +565,6 @@ Subscription.prototype.pull = function(options, callback) { * subscription.seek(date, callback); */ Subscription.prototype.seek = function(snapshot, callback) { - var protoOpts = { - service: 'Subscriber', - method: 'seek' - }; - var reqOpts = { subscription: this.name }; @@ -786,55 +580,19 @@ Subscription.prototype.seek = function(snapshot, callback) { throw new Error('Either a snapshot name or Date is needed to seek to.'); } - this.parent.request(protoOpts, reqOpts, callback); + this.api.Subscriber.seek(reqOpts, callback); }; /** - * Modify the ack deadline for a specific message. This method is useful to - * indicate that more time is needed to process a message by the subscriber, or - * to make the message available for redelivery if the processing was - * interrupted. - * - * @resource [Subscriptions: modifyAckDeadline API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/modifyAckDeadline} - * - * @param {object} options - The configuration object. - * @param {string|string[]} options.ackIds - The ack id(s) to change. - * @param {number} options.seconds - Number of seconds after call is made to - * set the deadline of the ack. - * @param {Function=} callback - The callback function. - * - * @example - * var options = { - * ackIds: ['abc'], - * seconds: 10 // Expire in 10 seconds from call. - * }; - * - * subscription.setAckDeadline(options, function(err, apiResponse) {}); * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.setAckDeadline(options).then(function(data) { - * var apiResponse = data[0]; - * }); */ -Subscription.prototype.setAckDeadline = function(options, callback) { - callback = callback || common.util.noop; - - var protoOpts = { - service: 'Subscriber', - method: 'modifyAckDeadline' - }; - +Subscription.prototype.setMetadata = function(metadata, callback) { var reqOpts = { subscription: this.name, - ackIds: arrify(options.ackIds), - ackDeadlineSeconds: options.seconds + updateMask: metadata }; - this.parent.request(protoOpts, reqOpts, function(err, resp) { - callback(err, resp); - }); + this.api.Subscriber.updateSubscription(reqOpts, callback); }; /** @@ -850,114 +608,7 @@ Subscription.prototype.setAckDeadline = function(options, callback) { * var snapshot = subscription.snapshot('my-snapshot'); */ Subscription.prototype.snapshot = function(name) { - return this.parent.snapshot.call(this, name); -}; - -/** - * Begin listening for events on the subscription. This method keeps track of - * how many message listeners are assigned, and then removed, making sure - * polling is handled automatically. - * - * As long as there is one active message listener, the connection is open. As - * soon as there are no more message listeners, the connection is closed. - * - * @private - * - * @example - * subscription.listenForEvents_(); - */ -Subscription.prototype.listenForEvents_ = function() { - var self = this; - - this.on('newListener', function(event) { - if (event === 'message') { - self.messageListeners++; - if (self.closed) { - self.closed = false; - self.startPulling_(); - } - } - }); - - this.on('removeListener', function(event) { - if (event === 'message' && --self.messageListeners === 0) { - self.closed = true; - - if (self.activeRequest_ && self.activeRequest_.abort) { - self.activeRequest_.abort(); - } - } - }); -}; - -/** - * Update the status of `maxInProgress`. Å subscription becomes "paused" (not - * pulling) when the number of messages that have yet to be ack'd or skipped - * exceeds the user's specified `maxInProgress` value. - * - * This will start pulling when that event reverses: we were paused, but one or - * more messages were just ack'd or skipped, freeing up room for more messages - * to be consumed. - * - * @private - */ -Subscription.prototype.refreshPausedStatus_ = function() { - var isCurrentlyPaused = this.paused; - var inProgress = Object.keys(this.inProgressAckIds).length; - - this.paused = inProgress >= this.maxInProgress; - - if (isCurrentlyPaused && !this.paused && this.messageListeners > 0) { - this.startPulling_(); - } -}; - -/** - * Poll the backend for new messages. This runs a loop to ping the API at the - * provided interval from the subscription's instantiation. If one wasn't - * provided, the default value is 10 milliseconds. - * - * If messages are received, they are emitted on the `message` event. - * - * Note: This method is automatically called once a message event handler is - * assigned to the description. - * - * To stop pulling, see {module:pubsub/subscription#close}. - * - * @private - * - * @example - * subscription.startPulling_(); - */ -Subscription.prototype.startPulling_ = function() { - var self = this; - - if (this.closed || this.paused) { - return; - } - - var maxResults; - - if (this.maxInProgress < Infinity) { - maxResults = this.maxInProgress - Object.keys(this.inProgressAckIds).length; - } - - this.pull({ - returnImmediately: false, - maxResults: maxResults - }, function(err, messages, apiResponse) { - if (err) { - self.emit('error', err, apiResponse); - } - - if (messages) { - messages.forEach(function(message) { - self.emit('message', message, apiResponse); - }); - } - - setTimeout(self.startPulling_.bind(self), self.interval); - }); + return this.pubsub.snapshot.call(this, name); }; /*! Developer Documentation diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index 812a6ce92bf..e857bc8dd91 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -20,12 +20,8 @@ 'use strict'; -var arrify = require('arrify'); var common = require('@google-cloud/common'); -var commonGrpc = require('@google-cloud/common-grpc'); -var extend = require('extend'); var is = require('is'); -var util = require('util'); /** * @type {module:pubsub/iam} @@ -33,6 +29,12 @@ var util = require('util'); */ var IAM = require('./iam.js'); +/** + * @type {module:pubsub/publisher} + * @private + */ +var Publisher = require('./publisher.js'); + /*! Developer Documentation * * @param {module:pubsub} pubsub - PubSub object. @@ -47,145 +49,11 @@ var IAM = require('./iam.js'); * @example * var topic = pubsub.topic('my-topic'); */ -function Topic(pubsub, name) { +function Topic(pubsub, name, options) { this.name = Topic.formatName_(pubsub.projectId, name); - - var methods = { - /** - * Create a topic. - * - * @param {object=} config - See {module:pubsub#createTopic}. - * - * @example - * topic.create(function(err, topic, apiResponse) { - * if (!err) { - * // The topic was created successfully. - * } - * }); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.create().then(function(data) { - * var topic = data[0]; - * var apiResponse = data[1]; - * }); - */ - create: true, - - /** - * Delete the topic. This will not delete subscriptions to this topic. - * - * @resource [Topics: delete API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/delete} - * - * @param {function=} callback - The callback function. - * - * @example - * topic.delete(function(err, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.delete().then(function(data) { - * var apiResponse = data[0]; - * }); - */ - delete: { - protoOpts: { - service: 'Publisher', - method: 'deleteTopic' - }, - reqOpts: { - topic: this.name - } - }, - - /** - * Check if the topic exists. - * - * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this - * request. - * @param {boolean} callback.exists - Whether the topic exists or not. - * - * @example - * topic.exists(function(err, exists) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.exists().then(function(data) { - * var exists = data[0]; - * }); - */ - exists: true, - - /** - * Get a topic if it exists. - * - * You may optionally use this to "get or create" an object by providing an - * object with `autoCreate` set to `true`. Any extra configuration that is - * normally required for the `create` method must be contained within this - * object as well. - * - * @param {options=} options - Configuration object. - * @param {boolean} options.autoCreate - Automatically create the object if - * it does not exist. Default: `false` - * - * @example - * topic.get(function(err, topic, apiResponse) { - * // `topic.metadata` has been populated. - * }); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.get().then(function(data) { - * var topic = data[0]; - * var apiResponse = data[1]; - * }); - */ - get: true, - - /** - * Get the official representation of this topic from the API. - * - * @resource [Topics: get API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/get} - * - * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this - * request. - * @param {object} callback.metadata - The metadata of the Topic. - * @param {object} callback.apiResponse - The full API response. - * - * @example - * topic.getMetadata(function(err, metadata, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.getMetadata().then(function(data) { - * var metadata = data[0]; - * var apiResponse = data[1]; - * }); - */ - getMetadata: { - protoOpts: { - service: 'Publisher', - method: 'getTopic' - }, - reqOpts: { - topic: this.name - } - } - }; - - commonGrpc.ServiceObject.call(this, { - parent: pubsub, - id: this.name, - createMethod: pubsub.createTopic.bind(pubsub), - methods: methods - }); + this.pubsub = pubsub; + this.projectId = pubsub.projectId; + this.api = pubsub.api; /** * [IAM (Identity and Access Management)](https://cloud.google.com/pubsub/access_control) @@ -224,25 +92,6 @@ function Topic(pubsub, name) { this.iam = new IAM(pubsub, this.name); } -util.inherits(Topic, commonGrpc.ServiceObject); - -/** - * Format a message object as the upstream API expects it. - * - * @private - * - * @return {object} - */ -Topic.formatMessage_ = function(message) { - if (!(message.data instanceof Buffer)) { - message.data = new Buffer(JSON.stringify(message.data)); - } - - message.data = message.data.toString('base64'); - - return message; -}; - /** * Format the name of a topic. A Topic's full name is in the format of * 'projects/{projectId}/topics/{topicName}'. @@ -259,6 +108,150 @@ Topic.formatName_ = function(projectId, name) { return 'projects/' + projectId + '/topics/' + name; }; +/** + * Create a topic. + * + * @param {object=} config - See {module:pubsub#createTopic}. + * + * @example + * topic.create(function(err, topic, apiResponse) { + * if (!err) { + * // The topic was created successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.create().then(function(data) { + * var topic = data[0]; + * var apiResponse = data[1]; + * }); + */ +Topic.prototype.create = function(callback) { + this.pubsub.createTopic(this.name, callback); +}; + +/** + * Create a subscription to this topic. + * + * All generated subscription names share a common prefix, `autogenerated-`. + * + * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} + * + * @param {string=} subName - The name of the subscription. If a name is not + * provided, a random subscription name will be generated and created. + * @param {object=} options - See a + * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + * @param {number} options.ackDeadlineSeconds - The maximum time after + * receiving a message that you must ack a message before it is redelivered. + * @param {boolean=} options.autoAck - Automatically acknowledge the message + * once it's pulled. (default: false) + * @param {string} options.encoding - When pulling for messages, this type is + * used when converting a message's data to a string. (default: 'utf-8') + * @param {number} options.interval - Interval in milliseconds to check for new + * messages. (default: 10) + * @param {number} options.maxInProgress - Maximum messages to consume + * simultaneously. + * @param {number|date} options.messageRetentionDuration - Set this to override + * the default duration of 7 days. This value is expected in seconds. + * Acceptable values are in the range of 10 minutes and 7 days. + * @param {string} options.pushEndpoint - A URL to a custom endpoint that + * messages should be pushed to. + * @param {boolean} options.retainAckedMessages - If set, acked messages are + * retained in the subscription's backlog for 7 days (unless overriden by + * `options.messageRetentionDuration`). Default: `false` + * @param {number} options.timeout - Set a maximum amount of time in + * milliseconds on an HTTP request to pull new messages to wait for a + * response before the connection is broken. + * @param {function} callback - The callback function. + * + * @example + * var callback = function(err, subscription, apiResponse) {}; + * + * // Without specifying any options. + * topic.createSubscription('newMessages', callback); + * + * //- + * // Omit the name to have one generated automatically. All generated names + * // share a common prefix, `autogenerated-`. + * //- + * topic.createSubscription(function(err, subscription, apiResponse) { + * // subscription.name = The generated name. + * }); + * + * // With options. + * topic.createSubscription('newMessages', { + * ackDeadlineSeconds: 90 + * }, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.createSubscription('newMessages').then(function(data) { + * var subscription = data[0]; + * var apiResponse = data[1]; + * }); + */ +Topic.prototype.createSubscription = function(name, options, callback) { + this.pubsub.createSubscription(this, name, options, callback); +}; + +/** + * Delete the topic. This will not delete subscriptions to this topic. + * + * @resource [Topics: delete API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/delete} + * + * @param {function=} callback - The callback function. + * + * @example + * topic.delete(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.delete().then(function(data) { + * var apiResponse = data[0]; + * }); + */ +Topic.prototype.delete = function(callback) { + var reqOpts = { + topic: this.name + }; + + this.api.Publisher.deleteTopic(reqOpts, callback); +}; + +/** + * Get the official representation of this topic from the API. + * + * @resource [Topics: get API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/get} + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.metadata - The metadata of the Topic. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * topic.getMetadata(function(err, metadata, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.getMetadata().then(function(data) { + * var metadata = data[0]; + * var apiResponse = data[1]; + * }); + */ +Topic.prototype.getMetadata = function(callback) { + var reqOpts = { + topic: this.name + }; + + this.api.Publisher.getTopic(reqOpts, callback); +}; + /** * Get a list of the subscriptions registered to this topic. You may optionally * provide a query object as the first argument to customize the response. @@ -318,7 +311,7 @@ Topic.prototype.getSubscriptions = function(options, callback) { options = options || {}; options.topic = this; - return this.parent.getSubscriptions(options, callback); + return this.pubsub.getSubscriptions(options, callback); }; /** @@ -352,201 +345,14 @@ Topic.prototype.getSubscriptionsStream = function(options) { options = options || {}; options.topic = this; - return this.parent.getSubscriptionsStream(options); -}; - -/** - * Publish the provided message or array of messages. On success, an array of - * messageIds is returned in the response. - * - * @resource [Topics: publish API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish} - * - * @throws {Error} If no message is provided. - * - * @param {*|*[]} message - The message(s) to publish. If you need to - * provide attributes for the message, you must enable `options.raw`, then - * box your message in to an object with a `data` and `attributes` property. - * `data` will be the raw message value you want to publish, and - * `attributes` is a key/value pair of attributes to apply to the message. - * All messages not provided as `Buffer` will be published in JSON format. - * If your receiving end uses another library, make sure it parses the - * message properly. - * @param {object=} options - Configuration object. - * @param {boolean} options.raw - Enable if you require setting attributes on - * your messages. - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds before giving up if no response is received. - * @param {function=} callback - The callback function. - * - * @example - * topic.publish('Hello, world!', function(err, messageIds, apiResponse) {}); - * - * //- - * // You can also publish a JSON object. - * //- - * var registerMessage = { - * userId: 3, - * name: 'Stephen', - * event: 'new user' - * }; - * - * topic.publish(registerMessage, function(err, messageIds, apiResponse) {}); - * - * //- - * // You can publish a batch of messages at once by supplying an array. - * //- - * var purchaseMessage = { - * data: { - * userId: 3, - * product: 'computer', - * event: 'purchase' - * } - * }; - * - * topic.publish([ - * registerMessage, - * purchaseMessage - * ], function(err, messageIds, apiResponse) {}); - * - * //- - * // Set attributes with your message. - * //- - * var message = { - * data: { - * userId: 3, - * product: 'book', - * event: 'rent' - * }, - * attributes: { - * key: 'value', - * hello: 'world' - * } - * }; - * - * var options = { - * raw: true - * }; - * - * topic.publish(message, options, function(err, messageIds, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.publish(message).then(function(data) { - * var messageIds = data[0]; - * var apiResponse = data[1]; - * }); - * - */ -Topic.prototype.publish = function(messages, options, callback) { - messages = arrify(messages); - - if (is.fn(options)) { - callback = options; - options = {}; - } - - options = options || {}; - callback = callback || common.util.noop; - - if (messages.length === 0) { - throw new Error('Cannot publish without a message.'); - } - - var protoOpts = { - service: 'Publisher', - method: 'publish', - }; - - if (is.number(options.timeout)) { - protoOpts.timeout = options.timeout; - } - - var reqOpts = { - topic: this.name, - messages: messages - .map(function(message) { - if (is.object(message)) { - message = extend(true, {}, message); - } - return options.raw ? message : { data: message }; - }) - .map(Topic.formatMessage_) - }; - - this.parent.request(protoOpts, reqOpts, function(err, result) { - if (err) { - callback(err, null, result); - return; - } - - callback(null, arrify(result.messageIds), result); - }); + return this.pubsub.getSubscriptionsStream(options); }; /** - * Create a subscription to this topic. - * - * All generated subscription names share a common prefix, `autogenerated-`. - * - * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} - * - * @param {string=} subName - The name of the subscription. If a name is not - * provided, a random subscription name will be generated and created. - * @param {object=} options - See a - * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadlineSeconds - The maximum time after - * receiving a message that you must ack a message before it is redelivered. - * @param {boolean=} options.autoAck - Automatically acknowledge the message - * once it's pulled. (default: false) - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') - * @param {number} options.interval - Interval in milliseconds to check for new - * messages. (default: 10) - * @param {number} options.maxInProgress - Maximum messages to consume - * simultaneously. - * @param {number|date} options.messageRetentionDuration - Set this to override - * the default duration of 7 days. This value is expected in seconds. - * Acceptable values are in the range of 10 minutes and 7 days. - * @param {string} options.pushEndpoint - A URL to a custom endpoint that - * messages should be pushed to. - * @param {boolean} options.retainAckedMessages - If set, acked messages are - * retained in the subscription's backlog for 7 days (unless overriden by - * `options.messageRetentionDuration`). Default: `false` - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds on an HTTP request to pull new messages to wait for a - * response before the connection is broken. - * @param {function} callback - The callback function. * - * @example - * // Without specifying any options. - * topic.subscribe('newMessages', function(err, subscription, apiResponse) {}); - * - * //- - * // Omit the name to have one generated automatically. All generated names - * // share a common prefix, `autogenerated-`. - * //- - * topic.subscribe(function(err, subscription, apiResponse) { - * // subscription.name = The generated name. - * }); - * - * // With options. - * topic.subscribe('newMessages', { - * ackDeadlineSeconds: 90, - * autoAck: true, - * interval: 30 - * }, function(err, subscription, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * topic.subscribe('newMessages').then(function(data) { - * var subscription = data[0]; - * var apiResponse = data[1]; - * }); */ -Topic.prototype.subscribe = function(subName, options, callback) { - this.parent.subscribe(this, subName, options, callback); +Topic.prototype.publisher = function(options) { + return new Publisher(this, options); }; /** @@ -556,10 +362,6 @@ Topic.prototype.subscribe = function(subName, options, callback) { * * @param {string} name - Name of the subscription. * @param {object=} options - Configuration object. - * @param {boolean=} options.autoAck - Automatically acknowledge the message - * once it's pulled. - * @param {number=} options.interval - Interval in milliseconds to check for new - * messages. * @return {module:pubsub/subscription} * * @example @@ -571,15 +373,15 @@ Topic.prototype.subscribe = function(subName, options, callback) { * // message.id = ID of the message. * // message.ackId = ID used to acknowledge the message receival. * // message.data = Contents of the message. - * // message.attributes = Attributes of the message. - * // message.timestamp = Timestamp when Pub/Sub received the message. + * // message.attrs = Attributes of the message. + * // message.publishTime = Timestamp when Pub/Sub received the message. * }); */ Topic.prototype.subscription = function(name, options) { options = options || {}; options.topic = this; - return this.parent.subscription(name, options); + return this.pubsub.subscription(name, options); }; /*! Developer Documentation @@ -588,7 +390,10 @@ Topic.prototype.subscription = function(name, options) { * that a callback is omitted. */ common.util.promisifyAll(Topic, { - exclude: ['subscription'] + exclude: [ + 'publisher', + 'subscription' + ] }); module.exports = Topic; diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 01e0966516c..357aacb8621 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -64,6 +64,7 @@ describe('pubsub', function() { options = options || {}; var topic = pubsub.topic(generateTopicName()); + var publisher = topic.publisher(); var subscription = topic.subscription(generateSubName()); async.series([ @@ -71,7 +72,7 @@ describe('pubsub', function() { subscription.create.bind(subscription), function(callback) { async.times(6, function(_, callback) { - topic.publish(message, options, callback); + publisher.publish(new Buffer(message), options, callback); }, callback); } ], function(err) { @@ -82,7 +83,7 @@ describe('pubsub', function() { subscription.pull({ returnImmediately: true, - maxResults: 1 + maxMessages: 1 }, function(err, messages) { if (err) { callback(err); @@ -163,26 +164,27 @@ describe('pubsub', function() { it('should publish a message', function(done) { var topic = pubsub.topic(TOPIC_NAMES[0]); - topic.publish('message from me', function(err, messageIds) { + var publisher = topic.publisher(); + var message = new Buffer('message from me'); + + publisher.publish(message, function(err, messageId) { assert.ifError(err); - assert.strictEqual(messageIds.length, 1); + assert.strictEqual(typeof messageId, 'string'); done(); }); }); it('should publish a message with attributes', function(done) { - var rawMessage = { - data: 'raw message data', - attributes: { - customAttribute: 'value' - } + var data = new Buffer('raw message data'); + var attrs = { + customAttribute: 'value' }; - publishPop(rawMessage, { raw: true }, function(err, message) { + publishPop(data, attrs, function(err, message) { assert.ifError(err); - assert.strictEqual(message.data, rawMessage.data); - assert.deepEqual(message.attributes, rawMessage.attributes); + assert.deepEqual(message.data, data); + assert.deepEqual(message.attrs, attrs); done(); }); @@ -201,6 +203,7 @@ describe('pubsub', function() { describe('Subscription', function() { var TOPIC_NAME = generateTopicName(); var topic = pubsub.topic(TOPIC_NAME); + var publisher = topic.publisher(); var SUB_NAMES = [ generateSubName(), @@ -227,7 +230,7 @@ describe('pubsub', function() { } async.times(10, function(_, next) { - topic.publish('hello', next); + publisher.publish(new Buffer('hello'), next); }, function(err) { if (err) { done(err); @@ -299,24 +302,18 @@ describe('pubsub', function() { it('should allow creation and deletion of a subscription', function(done) { var subName = generateSubName(); - topic.subscribe(subName, function(err, sub) { + topic.createSubscription(subName, function(err, sub) { assert.ifError(err); assert(sub instanceof Subscription); sub.delete(done); }); }); - it('should create a subscription with a generated name', function(done) { - topic.subscribe(function(err, sub) { - assert.ifError(err); - sub.delete(done); - }); - }); - it('should create a subscription with message retention', function(done) { + var subName = generateSubName(); var threeDaysInSeconds = 3 * 24 * 60 * 60; - topic.subscribe({ + topic.createSubscription(subName, { messageRetentionDuration: threeDaysInSeconds }, function(err, sub) { assert.ifError(err); @@ -340,14 +337,14 @@ describe('pubsub', function() { }); it('should re-use an existing subscription', function(done) { - pubsub.subscribe(topic, SUB_NAMES[0], done); + pubsub.createSubscription(topic, SUB_NAMES[0], done); }); it('should error when using a non-existent subscription', function(done) { var subscription = topic.subscription(generateSubName()); subscription.pull(function(err) { - assert.equal(err.code, 404); + assert.equal(err.code, 5); done(); }); }); @@ -357,13 +354,14 @@ describe('pubsub', function() { subscription.pull({ returnImmediately: true, - maxResults: 1 + maxMessages: 1 }, function(err, msgs) { assert.ifError(err); - assert.strictEqual(msgs.length, 1); - subscription.ack(msgs[0].ackId, done); + var message = msgs[0]; + + message.ack(done); }); }); @@ -372,18 +370,15 @@ describe('pubsub', function() { subscription.pull({ returnImmediately: true, - maxResults: 1 + maxMessages: 1 }, function(err, msgs) { assert.ifError(err); assert.strictEqual(msgs.length, 1); - var options = { - ackIds: [msgs[0].ackId], - seconds: 10 - }; + var message = msgs[0]; - subscription.setAckDeadline(options, done); + message.modifyAckDeadline(10000, done); }); }); @@ -392,63 +387,29 @@ describe('pubsub', function() { subscription.pull({ returnImmediately: true, - maxResults: 1 + maxMessages: 1 }, function(err, msgs) { assert.ifError(err); assert.strictEqual(msgs.length, 1); assert.equal(msgs[0].data, 'hello'); - subscription.ack(msgs[0].ackId, done); + + msgs[0].ack(done); }); }); it('should receive the chosen amount of results', function(done) { var subscription = topic.subscription(SUB_NAMES[1]); - var opts = { returnImmediately: true, maxResults: 3 }; - - subscription.pull(opts, function(err, messages) { - assert.ifError(err); - - assert.equal(messages.length, opts.maxResults); - - var ackIds = messages.map(function(message) { - return message.ackId; - }); + var maxMessages = 3; - subscription.ack(ackIds, done); - }); - }); - - it('should allow a custom timeout', function(done) { - var timeout = 5000; - - // We need to use a topic without any pending messages to allow the - // connection to stay open. - var topic = pubsub.topic(generateTopicName()); - var subscription = topic.subscription(generateSubName(), { - timeout: timeout - }); - - async.series([ - topic.create.bind(topic), - subscription.create.bind(subscription), - ], function(err) { + subscription.pull({ + returnImmediately: true, + maxMessages: maxMessages + }, function(err, messages) { assert.ifError(err); - var times = [Date.now()]; - - subscription.pull({ - returnImmediately: false - }, function(err) { - assert.ifError(err); - - times.push(Date.now()); - var runTime = times.pop() - times.pop(); + assert.equal(messages.length, maxMessages); - assert(runTime >= timeout - 1000); - assert(runTime <= timeout + 1000); - - done(); - }); + done(); }); }); }); @@ -506,11 +467,13 @@ describe('pubsub', function() { var SNAPSHOT_NAME = generateSnapshotName(); var topic; + var publisher; var subscription; var snapshot; before(function(done) { topic = pubsub.topic(TOPIC_NAMES[0]); + publisher = topic.publisher(); subscription = topic.subscription(generateSubName()); snapshot = subscription.snapshot(SNAPSHOT_NAME); subscription.create(done); @@ -557,10 +520,10 @@ describe('pubsub', function() { var messageId; beforeEach(function() { - subscription = topic.subscription(); + subscription = topic.subscription(generateSubName()); return subscription.create().then(function() { - return topic.publish('Hello, world!'); + return publisher.publish(new Buffer('Hello, world!')); }).then(function(data) { messageId = data[0][0]; }); From e515dc894661f6ddf4a51621d2b7d130bcd08f08 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 20 Jun 2017 14:59:28 -0400 Subject: [PATCH 02/67] funnel requests for projectId interpolation --- packages/pubsub/package.json | 2 + packages/pubsub/src/index.js | 152 +++++++++++++++++++++++----- packages/pubsub/src/message.js | 10 +- packages/pubsub/src/publisher.js | 6 +- packages/pubsub/src/snapshot.js | 7 +- packages/pubsub/src/subscription.js | 134 +++++++++++++++++++----- packages/pubsub/src/topic.js | 62 ++++++++++-- 7 files changed, 308 insertions(+), 65 deletions(-) diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index 8bde63a6141..35453e8c38a 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -60,6 +60,8 @@ "is": "^3.0.1", "modelo": "^4.2.0", "propprop": "^0.3.1", + "stream-events": "^1.0.2", + "through2": "^2.0.3", "uuid": "^3.0.1" }, "devDependencies": { diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index 4d5c6fe265a..5e77848b29e 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -22,9 +22,10 @@ var common = require('@google-cloud/common'); var extend = require('extend'); +var googleAuth = require('google-auto-auth'); var is = require('is'); -var path = require('path'); -var util = require('util'); +var streamEvents = require('stream-events'); +var through = require('through2'); var v1 = require('./v1'); @@ -68,6 +69,10 @@ function PubSub(options) { return new PubSub(options); } + this.options = extend({ + scopes: v1.ALL_SCOPES + }, options); + this.defaultBaseUrl_ = 'pubsub.googleapis.com'; if (options.servicePath) { @@ -78,13 +83,9 @@ function PubSub(options) { } } - this.api = { - Publisher: v1(options).publisherClient(options), - Subscriber: v1(options).subscriberClient(options) - }; - - this.options = options; - this.projectId = options.projectId; + this.api = {}; + this.auth = googleAuth(this.options); + this.projectId = this.options.projectId || '{{projectId}}'; } /** @@ -179,6 +180,8 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { name: subscription.name }, options); + delete reqOpts.gaxOpts; + if (reqOpts.messageRetentionDuration) { reqOpts.retainAckedMessages = true; @@ -194,7 +197,12 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { }; } - this.api.Subscriber.createSubscription(reqOpts, function(err, resp) { + this.request({ + client: 'subscriberClient', + method: 'createSubscription', + reqOpts: reqOpts, + gaxOpts: options.gaxOpts + }, function(err, resp) { if (err && err.code !== 6) { callback(err, null, resp); return; @@ -232,14 +240,24 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { * var apiResponse = data[1]; * }); */ -PubSub.prototype.createTopic = function(name, callback) { +PubSub.prototype.createTopic = function(name, gaxOpts, callback) { var topic = this.topic(name); var reqOpts = { name: topic.name }; - this.api.Publisher.createTopic(reqOpts, function(err, resp) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + + this.request({ + client: 'publisherClient', + method: 'createTopic', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, function(err, resp) { if (err) { callback(err, null, resp); return; @@ -306,7 +324,14 @@ PubSub.prototype.getSnapshots = function(options, callback) { project: 'projects/' + this.projectId }, options); - this.api.Subscriber.listSnapshots(reqOpts, function() { + delete reqOpts.gaxOpts; + + this.request({ + client: 'subscriberClient', + method: 'listSnapshots', + reqOpts: reqOpts, + gaxOpts: options.gaxOpts + }, function() { var snapshots = arguments[1]; if (snapshots) { @@ -416,21 +441,28 @@ PubSub.prototype.getSubscriptions = function(options, callback) { options = {}; } - var reqOpts = extend({}, options); - var method; + var topic = options.topic; - if (options.topic) { - if (options.topic instanceof Topic) { - reqOpts.topic = options.topic.name; + if (topic) { + if (!(topic instanceof Topic)) { + topic = this.topic(topic); } - method = this.api.Publisher.listTopicSubscriptions.bind(this.api.Publisher); - } else { - reqOpts.project = 'projects/' + this.projectId; - method = this.api.Subscriber.listSubscriptions.bind(this.api.Subscriber); + return topic.getSubscriptions(options, callback); } - method(reqOpts, function() { + + var reqOpts = extend({}, options); + + reqOpts.project = 'projects/' + this.projectId; + delete reqOpts.gaxOpts; + + this.request({ + client: 'subscriberClient', + method: 'listSubscriptions', + reqOpts: reqOpts, + gaxOpts: options.gaxOpts + }, function() { var subscriptions = arguments[1]; if (subscriptions) { @@ -549,11 +581,14 @@ PubSub.prototype.getTopics = function(options, callback) { project: 'projects/' + this.projectId }, options); - var options = { - autoPaginate: !is.number(reqOpts.pageSize) - }; + delete reqOpts.gaxOpts - this.api.Publisher.listTopics(reqOpts, options, function() { + this.request({ + client: 'publisherClient', + method: 'listTopics', + reqOpts: reqOpts, + gaxOpts: options.gaxOpts + }, function() { var topics = arguments[1]; if (topics) { @@ -597,6 +632,68 @@ PubSub.prototype.getTopics = function(options, callback) { */ PubSub.prototype.getTopicsStream = common.paginator.streamify('getTopics'); +/** + * Funnel all API requests through this method, to be sure we have a project ID. + * + * @param {object} config - Configuration object. + * @param {object} config.gaxOpts - GAX options. + * @param {function} config.method - The gax method to call. + * @param {object} config.reqOpts - Request options. + * @param {function=} callback - The callback function. + */ +PubSub.prototype.request = function(config, callback) { + var self = this; + + if (config.returnFn) { + prepareGaxRequest(callback); + } else { + makeRequestCallback(); + } + + function prepareGaxRequest(callback) { + self.auth.getProjectId(function(err, projectId) { + if (err) { + callback(err); + return; + } + + var gaxClient = self.api[config.client]; + + if (!gaxClient) { + // Lazily instantiate client. + gaxClient = v1(self.options)[config.client](self.options); + self.api[config.client] = gaxClient; + } + + var reqOpts = extend(true, {}, config.reqOpts); + reqOpts = common.util.replaceProjectIdToken(reqOpts, projectId); + + var requestFn = gaxClient[config.method].bind( + gaxClient, + reqOpts, + config.gaxOpts + ); + + callback(null, requestFn); + }); + } + + function makeRequestCallback() { + if (global.GCLOUD_SANDBOX_ENV) { + return; + } + + prepareGaxRequest(function(err, requestFn) { + if (err) { + callback(err); + return; + } + + requestFn(callback); + }); + } +}; + /** * Create a Snapshot object. See {module:pubsub/subscription#createSnapshot} to * create a snapshot. @@ -692,6 +789,7 @@ common.paginator.extend(PubSub, [ */ common.util.promisifyAll(PubSub, { exclude: [ + 'request', 'snapshot', 'subscription', 'topic' diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index 93bfff726ab..7c63a6aa06f 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -59,6 +59,10 @@ Message.prototype.modifyAckDeadline = function(milliseconds, callback) { modifyDeadlineAckIds: [this.ackId], modifyDeadlineSeconds: [seconds] }); + + if (is.fn(callback)) { + setImmediate(callback); + } return; } @@ -70,7 +74,11 @@ Message.prototype.modifyAckDeadline = function(milliseconds, callback) { ackDeadlineSeconds: seconds }; - this.api.Subscriber.modifyAckDeadline(reqOpts, callback); + this.subscription.request({ + client: 'subscriberClient', + method: 'modifyAckDeadline', + reqOpts: reqOpts + }, callback) }; /** diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 653b0c5ed25..be3e9d31068 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -84,7 +84,11 @@ Publisher.prototype.publish_ = function(messages, done) { }) }; - this.api.Publisher.publish(reqOpts, function(err, resp) { + this.topic.request({ + client: 'publisherClient', + method: 'publish', + reqOpts: reqOpts + }, function(err, resp) { if (err) { done(err); return; diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index 05ad116b00b..58c46a5fd72 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -88,6 +88,7 @@ var is = require('is'); * }); */ function Snapshot(parent, name) { + this.parent = parent; this.api = parent.api; this.name = Snapshot.formatName_(parent.projectId, name); @@ -192,7 +193,11 @@ Snapshot.prototype.delete = function(callback) { snapshot: this.name }; - this.api.Subscriber.deleteSnapshot(reqOpts, callback); + this.parent.request({ + client: 'subscriberClient', + method: 'deleteSnapshot', + reqOpts: reqOpts + }, callback); }; module.exports = Snapshot; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 3e74eb7d93f..4cec43fab06 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -147,7 +147,7 @@ function Subscription(pubsub, name, options) { options = options || {}; this.pubsub = pubsub; - this.api = pubsub.api; + this.request = pubsub.request.bind(pubsub); this.projectId = pubsub.projectId; this.name = Subscription.formatName_(pubsub.projectId, name); @@ -239,9 +239,15 @@ Subscription.prototype.ack_ = function(messages, callback) { return; } - this.api.Subscriber.acknowledge({ + var reqOpts = { ackIds: ackIds, subscription: this.name + }; + + this.request({ + client: 'subscriberClient', + method: 'acknowledge', + reqOpts: reqOpts }, callback); }; @@ -288,13 +294,18 @@ Subscription.prototype.closeConnection_ = function() { * var apiResponse = data[1]; * }); */ -Subscription.prototype.createSnapshot = function(name, callback) { +Subscription.prototype.createSnapshot = function(name, gaxOpts, callback) { var self = this; if (!is.string(name)) { throw new Error('A name is required to create a snapshot.'); } + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var snapshot = self.snapshot(name); var reqOpts = { @@ -302,7 +313,12 @@ Subscription.prototype.createSnapshot = function(name, callback) { subscription: this.name }; - this.api.Subscriber.createSnapshot(reqOpts, function(err, resp) { + this.request({ + client: 'subscriberClient', + method: 'createSnapshot', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, function(err, resp) { if (err) { callback(err, null, resp); return; @@ -334,17 +350,26 @@ Subscription.prototype.createSnapshot = function(name, callback) { * var apiResponse = data[0]; * }); */ -Subscription.prototype.delete = function(callback) { +Subscription.prototype.delete = function(gaxOpts, callback) { var self = this; + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { subscription: this.name }; - this.api.Subscriber.deleteSubscription(reqOpts, function(err, resp) { + this.request({ + client: 'subscriberClient', + method: 'deleteSubscription', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, function(err, resp) { if (!err) { - self.closed = true; - self.removeAllListeners(); + self.closeConnection_(); } callback(err, resp); @@ -357,12 +382,22 @@ Subscription.prototype.delete = function(callback) { * request. * @param {object} callback.apiResponse - Raw API response. */ -Subscription.prototype.getMetadata = function(callback) { +Subscription.prototype.getMetadata = function(gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { subscription: this.name }; - this.api.Subscriber.getSubscription(reqOpts, callback); + this.request({ + client: 'subscriberClient', + method: 'getSubscription', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback); }; /** @@ -403,13 +438,23 @@ Subscription.prototype.listenForEvents_ = function() { * @param {string} config.pushEndpoint * @param {object} config.attributes */ -Subscription.prototype.modifyPushConfig = function(config, callback) { +Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { subscription: this.name, pushConfig: config }; - this.api.Subscriber.modifyPushConfig(reqOpts, callback); + this.request({ + client: 'subscriberClient', + method: 'modifyPushConfig', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback); }; /** @@ -423,12 +468,19 @@ Subscription.prototype.openConnection_ = function() { streamAckDeadlineSeconds: this.ackDeadline }; - var grpcOpts = { - timeout: Number.MAX_NUMBER - }; + this.request({ + client: 'subscriberClient', + method: 'streamingPull', + returnFn: true + }, function(err, requestFn) { + if (err) { + self.emit('error', err); + return; + } - this.connection = this.api.Subscriber.streamingPull(grpcOpts) - .on('error', function(err) { + self.connection = requestFn(); + + self.connection.on('error', function(err) { var retryCodes = [2, 4, 14]; if (retryCodes.indexOf(err.code) > -1) { @@ -437,14 +489,16 @@ Subscription.prototype.openConnection_ = function() { } self.emit('error', err); - }) - .on('data', function(data) { + }); + + self.connection.on('data', function(data) { data.receivedMessages.forEach(function(message) { self.emit('message', new Message(self, message)); }); }); - this.connection.write(reqOpts); + self.connection.write(reqOpts); + }); }; /** @@ -515,10 +569,16 @@ Subscription.prototype.pull = function(options, callback) { var reqOpts = extend({ subscription: this.name, }, options); - - this.api.Subscriber.pull(reqOpts, function(err, resp) { + delete reqOpts.gaxOpts; + + this.request({ + client: 'subscriberClient', + method: 'pull', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, function(err, resp) { if (err) { - if (err.code !== 504) { + if (err.code !== 4) { callback(err, null, resp); return; } @@ -564,7 +624,12 @@ Subscription.prototype.pull = function(options, callback) { * * subscription.seek(date, callback); */ -Subscription.prototype.seek = function(snapshot, callback) { +Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { subscription: this.name }; @@ -580,19 +645,34 @@ Subscription.prototype.seek = function(snapshot, callback) { throw new Error('Either a snapshot name or Date is needed to seek to.'); } - this.api.Subscriber.seek(reqOpts, callback); + this.request({ + client: 'subscriberClient', + method: 'seek', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback) }; /** * */ -Subscription.prototype.setMetadata = function(metadata, callback) { +Subscription.prototype.setMetadata = function(metadata, gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { subscription: this.name, updateMask: metadata }; - this.api.Subscriber.updateSubscription(reqOpts, callback); + this.request({ + client: 'subscriberClient', + method: 'updateSubscription', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback); }; /** diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index e857bc8dd91..e9475e52893 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -53,7 +53,7 @@ function Topic(pubsub, name, options) { this.name = Topic.formatName_(pubsub.projectId, name); this.pubsub = pubsub; this.projectId = pubsub.projectId; - this.api = pubsub.api; + this.request = pubsub.request.bind(pubsub); /** * [IAM (Identity and Access Management)](https://cloud.google.com/pubsub/access_control) @@ -214,12 +214,22 @@ Topic.prototype.createSubscription = function(name, options, callback) { * var apiResponse = data[0]; * }); */ -Topic.prototype.delete = function(callback) { +Topic.prototype.delete = function(gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { topic: this.name }; - this.api.Publisher.deleteTopic(reqOpts, callback); + this.request({ + client: 'publisherClient', + method: 'deleteTopic', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback); }; /** @@ -244,12 +254,22 @@ Topic.prototype.delete = function(callback) { * var apiResponse = data[1]; * }); */ -Topic.prototype.getMetadata = function(callback) { +Topic.prototype.getMetadata = function(gaxOpts, callback) { + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + var reqOpts = { topic: this.name }; - this.api.Publisher.getTopic(reqOpts, callback); + this.request({ + client: 'publisherClient', + method: 'getTopic', + reqOpts: reqOpts, + gaxOpts: gaxOpts + }, callback); }; /** @@ -303,15 +323,41 @@ Topic.prototype.getMetadata = function(callback) { * }); */ Topic.prototype.getSubscriptions = function(options, callback) { + var self = this; + if (is.fn(options)) { callback = options; options = {}; } - options = options || {}; - options.topic = this; + var reqOpts = extend({}, options); + delete reqOpts.gaxOpts; + + this.request({ + client: 'publisherClient', + method: 'listTopicSubscriptions', + reqOpts: reqOpts, + gaxOpts: options.gaxOpts + }, function() { + var subscriptions = arguments[1]; + + if (subscriptions) { + arguments[1] = subscriptions.map(function(sub) { + // Depending on if we're using a subscriptions.list or + // topics.subscriptions.list API endpoint, we will get back a + // Subscription resource or just the name of the subscription. + var subscriptionInstance = self.subscription(sub.name || sub); + + if (sub.name) { + subscriptionInstance.metadata = sub; + } + + return subscriptionInstance; + }); + } - return this.pubsub.getSubscriptions(options, callback); + callback.apply(null, arguments); + }); }; /** From 3fbdd2f87ddbd631b350215b840d02ecc75563a1 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 13:13:18 -0400 Subject: [PATCH 03/67] change Error to TypeError --- packages/pubsub/src/publisher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index be3e9d31068..dd68d91c855 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -51,7 +51,7 @@ function Publisher(topic, options) { */ Publisher.prototype.publish = function(data, attrs, callback) { if (!(data instanceof Buffer)) { - throw new Error('Data must be in the form of a Buffer.'); + throw new TypeError('Data must be in the form of a Buffer.'); } if (is.fn(attrs)) { From db200cc26f877bbb39b2c8396a7ec0907de83a3d Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 13:14:03 -0400 Subject: [PATCH 04/67] update license year --- packages/pubsub/src/batch.js | 2 +- packages/pubsub/src/iam.js | 2 +- packages/pubsub/src/message.js | 3 ++- packages/pubsub/src/publisher.js | 2 +- packages/pubsub/src/snapshot.js | 2 +- packages/pubsub/src/subscription.js | 2 +- packages/pubsub/src/topic.js | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/pubsub/src/batch.js b/packages/pubsub/src/batch.js index 1c2a0698802..efa1e745952 100644 --- a/packages/pubsub/src/batch.js +++ b/packages/pubsub/src/batch.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/iam.js b/packages/pubsub/src/iam.js index fe6e3b997ac..21c6ab012ed 100644 --- a/packages/pubsub/src/iam.js +++ b/packages/pubsub/src/iam.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index 7c63a6aa06f..b493767434e 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,6 +63,7 @@ Message.prototype.modifyAckDeadline = function(milliseconds, callback) { if (is.fn(callback)) { setImmediate(callback); } + return; } diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index dd68d91c855..c00f74db97f 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index 58c46a5fd72..5858b256265 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 4cec43fab06..8621b5eb784 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index e9475e52893..f6ec8d1211e 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -1,5 +1,5 @@ /*! - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2017 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 1cca96027cc82f8fcfe622350abe2083442717eb Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 13:32:53 -0400 Subject: [PATCH 05/67] refactor batching --- packages/pubsub/src/publisher.js | 8 +-- packages/pubsub/src/{batch.js => queue.js} | 82 ++++++++-------------- packages/pubsub/src/subscription.js | 18 ++--- 3 files changed, 43 insertions(+), 65 deletions(-) rename packages/pubsub/src/{batch.js => queue.js} (70%) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index c00f74db97f..977f45e1986 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -28,7 +28,7 @@ var is = require('is'); /** * */ -var Batch = require('./batch.js'); +var Queue = require('./queue.js'); /** * @@ -39,11 +39,11 @@ function Publisher(topic, options) { this.topic = topic; this.api = topic.api; - var batchOptions = extend(options.batching, { + var queueOptions = extend(options.batching, { send: this.publish_.bind(this) }); - this.batch_ = new Batch(batchOptions); + this.queue_ = new Queue(queueOptions); } /** @@ -65,7 +65,7 @@ Publisher.prototype.publish = function(data, attrs, callback) { size: data.length }; - this.batch_.add(message, callback); + this.queue_.add(message, callback); }; /** diff --git a/packages/pubsub/src/batch.js b/packages/pubsub/src/queue.js similarity index 70% rename from packages/pubsub/src/batch.js rename to packages/pubsub/src/queue.js index efa1e745952..8d2da2e79c0 100644 --- a/packages/pubsub/src/batch.js +++ b/packages/pubsub/src/queue.js @@ -15,7 +15,7 @@ */ /*! - * @module pubsub/batch + * @module pubsub/queue */ 'use strict'; @@ -23,13 +23,13 @@ var prop = require('propprop'); /** - * Semi-generic Batch object used to queue up multiple requests while limiting + * Semi-generic Queue object used to queue up multiple requests while limiting * the number of concurrent requests. * * @constructor - * @alias module:pubsub/batch + * @alias module:pubsub/queue * - * @param {object} options - Batch configuration settings. + * @param {object} options - Queue configuration settings. * @param {number} options.maxBytes - Max size of data to be sent per request. * @param {number} options.maxMessages - Max number of messages to be sent per * request. @@ -38,10 +38,10 @@ var prop = require('propprop'); * @param {number} options.maxRequests - Max number of concurrent requests * allowed. * @param {function(messages, callback)} options.sendHandler - Function to be - * called with message batch. + * called with message queue. * * @example - * var batch = new Batch({ + * var queue = new Queue({ * maxBytes: Math.pow(1024, 2) * 5, * maxMessages: 1000, * maxMilliseconds: 1000, @@ -70,11 +70,10 @@ var prop = require('propprop'); * } * }); */ -function Batch(options) { +function Queue(options) { this.maxBytes = options.maxBytes || Math.pow(1024, 2) * 5; this.maxMessages = options.maxMessages || 1000; this.maxMilliseconds = options.maxMilliseconds || 1000; - this.maxRequests = options.maxRequests || 5; this.sendHandler = options.send; this.inventory = { @@ -87,7 +86,7 @@ function Batch(options) { var size = 0; this.queued.forEach(function(message) { - size += message.size; + size += message.data.size; }); return size; @@ -95,7 +94,6 @@ function Batch(options) { }); this.intervalHandle_ = null; - this.activeRequests_ = 0; } /** @@ -105,32 +103,31 @@ function Batch(options) { * @param {function} callback - Callback to be ran once message is sent. * * @example - * batch.add(1234, function(err, resp) {}); + * queue.add(1234, function(err, resp) {}); */ -Batch.prototype.add = function(data, callback) { - this.inventory.queued.add({ - data: data, - callback: callback - }); - - var reachedMaxMessages = this.inventory.queued.size >= this.maxMessages; - var reachedMaxBytes = this.inventory.queueBytes >= this.maxBytes; - var reachedMaxRequests = this.activeRequests_ >= this.maxRequests; +Queue.prototype.add = function(data, callback) { + var hasMaxMessages = (this.inventory.queued.size + 1) >= this.maxMessages; + var hasMaxBytes = (this.inventory.queueBytes + data.size) >= this.maxBytes; - if ((reachedMaxMessages || reachedMaxBytes) && !reachedMaxRequests) { + if (hasMaxMessages || hasMaxBytes) { this.send(this.getNextMessageBatch()); } else if (!this.intervalHandle_) { - this.beginSending(); + this.startSendInterval(); } + + this.inventory.queued.add({ + data: data, + callback: callback + }); }; /** * Starts the send interval. * * @example - * batch.beginSending(); + * queue.startSendInterval(); */ -Batch.prototype.beginSending = function() { +Queue.prototype.startSendInterval = function() { if (this.intervalHandle_) { return; } @@ -148,33 +145,18 @@ Batch.prototype.beginSending = function() { }; /** - * Prepares next batch of messages to be sent. + * Prepares next queue of messages to be sent. * * @return array * * @example - * var messages = batch.getNextMessageBatch(); - * batch.send(messages); + * var messages = queue.getNextMessageBatch(); + * queue.send(messages); */ -Batch.prototype.getNextMessageBatch = function() { - var size = 0; - var messages = []; - - var queued = this.inventory.queued[Symbol.iterator](); - var message; - - while (message = queued.next().value) { - var isOverSizeLimit = (size + message.size) > this.maxBytes; - var isOverMessageLimit = (messages.length + 1) > this.maxMessages; - - if (isOverSizeLimit || isOverMessageLimit) { - break; - } - - this.inventory.queued.delete(message); - messages.push(message); - } +Queue.prototype.getNextMessageBatch = function() { + var messages = Array.from(this.inventory.queued); + this.inventory.queued.clear(); return messages; }; @@ -189,9 +171,9 @@ Batch.prototype.getNextMessageBatch = function() { * callback: function(err, resp) {} * }]; * - * batch.send(messages); + * queue.send(messages); */ -Batch.prototype.send = function(messages) { +Queue.prototype.send = function(messages) { if (this.activeRequests_ >= this.maxRequests) { this.scheduleSend(); return; @@ -203,11 +185,7 @@ Batch.prototype.send = function(messages) { self.inventory.inFlight.add(message); }); - this.activeRequests_ += 1; - this.sendHandler(messages.map(prop('data')), function(err, resp) { - self.activeRequests_ -= 1; - messages.forEach(function(message, i) { var response = resp && resp.responses ? resp.responses[i] : resp; @@ -217,4 +195,4 @@ Batch.prototype.send = function(messages) { }); }; -module.exports = Batch; +module.exports = Queue; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 8621b5eb784..a29cda37635 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -27,12 +27,6 @@ var extend = require('extend'); var is = require('is'); var util = require('util'); -/** - * @type {module:pubsub/batch} - * @private - */ -var Batch = require('./batch.js'); - /** * @type {module:pubsub/iam} * @private @@ -45,6 +39,12 @@ var IAM = require('./iam.js'); */ var Message = require('./message.js'); +/** + * @type {module:pubsub/queue} + * @private + */ +var Queue = require('./queue.js'); + /** * @type {module:pubsub/snapshot} * @private @@ -197,11 +197,11 @@ function Subscription(pubsub, name, options) { */ this.iam = new IAM(pubsub, this.name); - var batchOptions = extend(options.batching, { + var queueOptions = extend(options.batching, { send: this.ack_.bind(this) }); - this.batch_ = new Batch(batchOptions); + this.queue_ = new Queue(queueOptions); this.listenForEvents_(); } @@ -255,7 +255,7 @@ Subscription.prototype.ack_ = function(messages, callback) { * */ Subscription.prototype.ack = function(ackIds, callback) { - this.batch_.add(arrify(ackIds), callback); + this.queue_.add(arrify(ackIds), callback); }; /** From aec964039ffe6bd063120c37245b0a5ea2efcbd4 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 15:00:00 -0400 Subject: [PATCH 06/67] added batching for modifyAckDeadline --- packages/pubsub/src/message.js | 51 ++++--------------- packages/pubsub/src/queue.js | 6 ++- packages/pubsub/src/subscription.js | 77 +++++++++++++++++++++++------ 3 files changed, 76 insertions(+), 58 deletions(-) diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index b493767434e..f5bea209ff6 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -20,8 +20,6 @@ 'use strict'; -var common = require('@google-cloud/common'); - /** * */ @@ -44,56 +42,25 @@ function Message(subscription, resp) { /** * */ -Message.prototype.ack = function(callback) { - this.subscription.ack(this.ackId, callback) +Message.prototype.ack = function() { + this.subscription.ackQueue_.add([this.ackId]); }; /** * */ -Message.prototype.modifyAckDeadline = function(milliseconds, callback) { - var seconds = milliseconds / 1000; - - if (this.subscription.connection) { - this.subscription.connection.write({ - modifyDeadlineAckIds: [this.ackId], - modifyDeadlineSeconds: [seconds] - }); - - if (is.fn(callback)) { - setImmediate(callback); - } - - return; - } - - callback = callback || common.util.noop; - - reqOpts = { - subscription: this.subscription.name, - ackIds: [this.ackId], - ackDeadlineSeconds: seconds - }; - - this.subscription.request({ - client: 'subscriberClient', - method: 'modifyAckDeadline', - reqOpts: reqOpts - }, callback) +Message.prototype.modifyAckDeadline = function(milliseconds) { + this.subscription.modifyQueue_.add({ + ackId: this.ackId, + deadline: milliseconds / 1000 + }); }; /** * */ -Message.prototype.nack = function(callback) { - this.modifyAckDeadline(0, callback); +Message.prototype.nack = function() { + this.modifyAckDeadline(0); }; -/*! Developer Documentation - * - * All async methods (except for streams) will return a Promise in the event - * that a callback is omitted. - */ -common.util.promisifyAll(Message); - module.exports = Message; diff --git a/packages/pubsub/src/queue.js b/packages/pubsub/src/queue.js index 8d2da2e79c0..13a1a0e77d9 100644 --- a/packages/pubsub/src/queue.js +++ b/packages/pubsub/src/queue.js @@ -20,6 +20,7 @@ 'use strict'; +var common = require('@google-cloud/common'); var prop = require('propprop'); /** @@ -117,7 +118,7 @@ Queue.prototype.add = function(data, callback) { this.inventory.queued.add({ data: data, - callback: callback + callback: callback || common.util.noop }); }; @@ -187,9 +188,10 @@ Queue.prototype.send = function(messages) { this.sendHandler(messages.map(prop('data')), function(err, resp) { messages.forEach(function(message, i) { + var error = Array.isArray(err) ? err[i] : err; var response = resp && resp.responses ? resp.responses[i] : resp; - message.callback(err, response); + message.callback(error, response); self.inventory.inFlight.delete(message); }); }); diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index a29cda37635..93ddf499eb0 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -197,11 +197,15 @@ function Subscription(pubsub, name, options) { */ this.iam = new IAM(pubsub, this.name); - var queueOptions = extend(options.batching, { + this.ackQueue_ = new Queue(extend({ send: this.ack_.bind(this) - }); + }, options.batching)); + + this.modifyQueue_ = new Queue(extend({ + send: this.modifyAckDeadline_.bind(this) + }, options.batching)); + - this.queue_ = new Queue(queueOptions); this.listenForEvents_(); } @@ -232,10 +236,7 @@ Subscription.prototype.ack_ = function(messages, callback) { if (this.connection) { this.connection.write({ ackIds: ackIds }); - - if (is.fn(callback)) { - setImmediate(callback); - } + setImmediate(callback); return; } @@ -251,13 +252,6 @@ Subscription.prototype.ack_ = function(messages, callback) { }, callback); }; -/** - * - */ -Subscription.prototype.ack = function(ackIds, callback) { - this.queue_.add(arrify(ackIds), callback); -}; - /** * @private */ @@ -433,6 +427,61 @@ Subscription.prototype.listenForEvents_ = function() { }); }; +/** + * + */ +Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { + var self = this; + var reqOpts; + + if (this.connection) { + reqOpts = messages.reduce(function(reqOpts, message) { + message.modifyDeadlineAckIds.push(message.ackId); + message.modifyDeadlineSeconds.push(message.deadline); + return reqOpts; + }, { + modifyDeadlineAckIds: [], + modifyDeadlineSeconds: [] + }); + + this.connection.write(reqOpts); + setImmediate(callback); + return; + } + + var requests = groupByDeadline(messages).map(function(reqOpts) { + return self.request({ + client: 'subscriberClient', + method: 'modifyAckDeadline', + reqOpts: reqOpts + }).catch(function(err) { + self.emit('error', err); + }); + }); + + Promise.all(requests) + .then(callback.bind(null, null), callback); + + function groupByDeadline(messages) { + var requests = new Map(); + + messages.forEach(function(message) { + if (!requests.has(message.deadline)) { + requests.set(message.deadline, { + subscription: self.name, + ackIds: [], + ackDeadlineSeconds: message.deadline + }); + } + + var request = requests.get(message.deadline); + request.ackIds.push(message.ackId); + }); + + return Array.from(requests.values()); + } +}; + /** * @param {object} config - The push config. * @param {string} config.pushEndpoint From 69841263ccf81087a6d61730b5437a07168154b8 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 17:16:31 -0400 Subject: [PATCH 07/67] implement message flow control --- packages/pubsub/src/message.js | 9 +++--- packages/pubsub/src/subscription.js | 49 +++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index f5bea209ff6..1ba40b2e590 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -32,6 +32,7 @@ function Message(subscription, resp) { this.id = resp.message.messageId; this.data = resp.message.data; this.attrs = resp.message.attributes; + this.deadline = null; var pt = resp.message.publishTime; var milliseconds = parseInt(pt.nanos, 10) / 1e6; @@ -43,17 +44,15 @@ function Message(subscription, resp) { * */ Message.prototype.ack = function() { - this.subscription.ackQueue_.add([this.ackId]); + this.subscription.ackQueue_.add(this); }; /** * */ Message.prototype.modifyAckDeadline = function(milliseconds) { - this.subscription.modifyQueue_.add({ - ackId: this.ackId, - deadline: milliseconds / 1000 - }); + this.deadline = milliseconds / 1000; + this.subscription.modifyQueue_.add(this); }; /** diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 93ddf499eb0..bf4d19071aa 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -25,6 +25,7 @@ var common = require('@google-cloud/common'); var events = require('events'); var extend = require('extend'); var is = require('is'); +var os = require('os'); var util = require('util'); /** @@ -154,6 +155,12 @@ function Subscription(pubsub, name, options) { this.connection = null; this.encoding = options.encoding || 'utf-8'; this.ackDeadline = options.ackDeadline || 600; + this.messages_ = []; + + this.flowControl = extend({ + highWaterMark: 16, + maxMessages: Infinity + }, options.flowControl); events.EventEmitter.call(this); @@ -230,13 +237,15 @@ Subscription.formatName_ = function(projectId, name) { * This should never be called directly. */ Subscription.prototype.ack_ = function(messages, callback) { + var self = this; var ackIds = messages.reduce(function(ackIds, message) { - return ackIds.concat(message); + ackIds.push(message.ackId); + return ackIds; }, []); if (this.connection) { this.connection.write({ ackIds: ackIds }); - setImmediate(callback); + setImmediate(onComplete); return; } @@ -249,7 +258,19 @@ Subscription.prototype.ack_ = function(messages, callback) { client: 'subscriberClient', method: 'acknowledge', reqOpts: reqOpts - }, callback); + }, onComplete); + + function onComplete(err, resp) { + messages.forEach(function(message) { + self.messages_.delete(message); + }); + + if (self.connection.isPaused()) { + self.connection.resume(); + } + + callback(err, resp); + } }; /** @@ -427,6 +448,15 @@ Subscription.prototype.listenForEvents_ = function() { }); }; +/** + * + */ +Subscription.prototype.message_ = function(data) { + var message = new Message(self, data); + this.messages_.push(message); + return message; +}; + /** * */ @@ -520,7 +550,10 @@ Subscription.prototype.openConnection_ = function() { this.request({ client: 'subscriberClient', method: 'streamingPull', - returnFn: true + returnFn: true, + reqOpts: { + highWaterMark: self.flowControl.highWaterMark + } }, function(err, requestFn) { if (err) { self.emit('error', err); @@ -541,9 +574,13 @@ Subscription.prototype.openConnection_ = function() { }); self.connection.on('data', function(data) { - data.receivedMessages.forEach(function(message) { - self.emit('message', new Message(self, message)); + arrify(data.receivedMessages).forEach(function(message) { + self.emit('message', self.message_(message))); }); + + if (self.messages_.length >= self.flowControl.maxMessages) { + self.connection.pause(); + } }); self.connection.write(reqOpts); From cad7d07f136d16654c0cc126e72c83923acd1560 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 17:17:05 -0400 Subject: [PATCH 08/67] remove all message listeners upon sub delete --- packages/pubsub/src/subscription.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index bf4d19071aa..44afe630103 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -384,7 +384,7 @@ Subscription.prototype.delete = function(gaxOpts, callback) { gaxOpts: gaxOpts }, function(err, resp) { if (!err) { - self.closeConnection_(); + self.removeAllListeners('message'); } callback(err, resp); From 73eec6d9771c8d873092f84b14813183a713fb15 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 21 Jun 2017 17:20:32 -0400 Subject: [PATCH 09/67] add comment explaining error handling --- packages/pubsub/src/subscription.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 44afe630103..f2dce84cc09 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -563,7 +563,13 @@ Subscription.prototype.openConnection_ = function() { self.connection = requestFn(); self.connection.on('error', function(err) { - var retryCodes = [2, 4, 14]; + // in the event of a connection error, we will attempt to re-establish + // a new connection if the grpc error code is something safe to retry on + var retryCodes = [ + 2, // Unknown Error + 4, // DeadlineExceeded + 14 // Unavailable + ]; if (retryCodes.indexOf(err.code) > -1) { self.openConnection_(); From 4088c45e6199470124a008f5d0d2188e1e90e052 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 22 Jun 2017 02:02:30 -0400 Subject: [PATCH 10/67] small refactor to inventory management --- packages/pubsub/src/subscription.js | 36 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index f2dce84cc09..cef1c642d88 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -155,7 +155,7 @@ function Subscription(pubsub, name, options) { this.connection = null; this.encoding = options.encoding || 'utf-8'; this.ackDeadline = options.ackDeadline || 600; - this.messages_ = []; + this.messages_ = new Set(); this.flowControl = extend({ highWaterMark: 16, @@ -451,9 +451,9 @@ Subscription.prototype.listenForEvents_ = function() { /** * */ -Subscription.prototype.message_ = function(data) { +Subscription.prototype.createMessage_ = function(data) { var message = new Message(self, data); - this.messages_.push(message); + this.messages_.add(message); return message; }; @@ -475,7 +475,7 @@ Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { }); this.connection.write(reqOpts); - setImmediate(callback); + setImmediate(onComplete); return; } @@ -490,7 +490,7 @@ Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { }); Promise.all(requests) - .then(callback.bind(null, null), callback); + .then(onComplete.bind(null, null), onComplete); function groupByDeadline(messages) { var requests = new Map(); @@ -510,6 +510,22 @@ Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { return Array.from(requests.values()); } + + function onComplete(err, resp) { + messages.forEach(function(message) { + if (message.deadline === 0) { + self.messages_.delete(message); + } + }); + + if (self.connection.isPaused()) { + if (self.messages_.size < self.flowControl.maxMessages) { + self.connection.resume(); + } + } + + callback(err, resp); + } }; /** @@ -581,11 +597,13 @@ Subscription.prototype.openConnection_ = function() { self.connection.on('data', function(data) { arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', self.message_(message))); + self.emit('message', self.createMessage_(message))); }); - if (self.messages_.length >= self.flowControl.maxMessages) { - self.connection.pause(); + if (!self.connection.isPaused()) { + if (self.messages_.size >= self.flowControl.maxMessages) { + self.connection.pause(); + } } }); @@ -682,7 +700,7 @@ Subscription.prototype.pull = function(options, callback) { } var messages = arrify(resp.receivedMessages).map(function(message) { - return new Message(self, message); + return self.createMessage_(message); }); callback(null, messages, resp); From e0cd3446e859e253619130484a5417237244a132 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 26 Jun 2017 18:15:23 -0400 Subject: [PATCH 11/67] store deadline in milliseconds --- packages/pubsub/src/message.js | 2 +- packages/pubsub/src/subscription.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index 1ba40b2e590..30446817d2b 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -51,7 +51,7 @@ Message.prototype.ack = function() { * */ Message.prototype.modifyAckDeadline = function(milliseconds) { - this.deadline = milliseconds / 1000; + this.deadline = milliseconds; this.subscription.modifyQueue_.add(this); }; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index cef1c642d88..e789d52f666 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -466,8 +466,8 @@ Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { if (this.connection) { reqOpts = messages.reduce(function(reqOpts, message) { - message.modifyDeadlineAckIds.push(message.ackId); - message.modifyDeadlineSeconds.push(message.deadline); + reqOpts.modifyDeadlineAckIds.push(message.ackId); + reqOpts.modifyDeadlineSeconds.push(message.deadline / 1000); return reqOpts; }, { modifyDeadlineAckIds: [], @@ -500,7 +500,7 @@ Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { requests.set(message.deadline, { subscription: self.name, ackIds: [], - ackDeadlineSeconds: message.deadline + ackDeadlineSeconds: message.deadline / 1000 }); } From f53c47f5f25b1e964ae425c13263ecf70df944ba Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 26 Jun 2017 18:19:52 -0400 Subject: [PATCH 12/67] update queue settings to add caps --- packages/pubsub/src/queue.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pubsub/src/queue.js b/packages/pubsub/src/queue.js index 13a1a0e77d9..9babd657cdc 100644 --- a/packages/pubsub/src/queue.js +++ b/packages/pubsub/src/queue.js @@ -72,8 +72,10 @@ var prop = require('propprop'); * }); */ function Queue(options) { - this.maxBytes = options.maxBytes || Math.pow(1024, 2) * 5; - this.maxMessages = options.maxMessages || 1000; + var maxBytes = options.maxBytes || Math.pow(1024, 2) * 5; + + this.maxBytes = Math.min(maxBytes, Math.pow(1024, 2) * 9); + this.maxMessages = Math.min(options.maxMessages || 1000, 1000); this.maxMilliseconds = options.maxMilliseconds || 1000; this.sendHandler = options.send; From c6615c4abfafc5a06e65164db015c41891abaa8a Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 26 Jun 2017 18:22:48 -0400 Subject: [PATCH 13/67] tweak how we calculate max message cap --- packages/pubsub/src/queue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pubsub/src/queue.js b/packages/pubsub/src/queue.js index 9babd657cdc..8a097d5f933 100644 --- a/packages/pubsub/src/queue.js +++ b/packages/pubsub/src/queue.js @@ -109,7 +109,7 @@ function Queue(options) { * queue.add(1234, function(err, resp) {}); */ Queue.prototype.add = function(data, callback) { - var hasMaxMessages = (this.inventory.queued.size + 1) >= this.maxMessages; + var hasMaxMessages = this.inventory.queued.size >= this.maxMessages; var hasMaxBytes = (this.inventory.queueBytes + data.size) >= this.maxBytes; if (hasMaxMessages || hasMaxBytes) { From 7d130326a658fce7decef3956d8b61a5de20c2de Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 01:12:36 -0400 Subject: [PATCH 14/67] update system tests to use message events --- packages/pubsub/system-test/pubsub.js | 143 +++++++++++++------------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 357aacb8621..93da54e98d9 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -81,16 +81,10 @@ describe('pubsub', function() { return; } - subscription.pull({ - returnImmediately: true, - maxMessages: 1 - }, function(err, messages) { - if (err) { - callback(err); - return; - } + subscription.on('error', callback); - callback(null, messages.pop()); + subscription.once('message', function(message) { + callback(null, message); }); }); } @@ -145,11 +139,9 @@ describe('pubsub', function() { it('should allow manual paging', function(done) { pubsub.getTopics({ pageSize: TOPIC_NAMES.length - 1 - }, function(err, topics, nextQuery) { + }, function(err, topics) { assert.ifError(err); assert(topics.length, TOPIC_NAMES.length - 1); - assert(nextQuery.pageSize, TOPIC_NAMES.length - 1); - assert(!!nextQuery.pageToken, true); done(); }); }); @@ -343,74 +335,59 @@ describe('pubsub', function() { it('should error when using a non-existent subscription', function(done) { var subscription = topic.subscription(generateSubName()); - subscription.pull(function(err) { - assert.equal(err.code, 5); - done(); + subscription.on('error', function(err) { + assert.strictEqual(err.code, 5); + subscription.close(done); }); - }); - - it('should be able to pull and ack', function(done) { - var subscription = topic.subscription(SUB_NAMES[0]); - - subscription.pull({ - returnImmediately: true, - maxMessages: 1 - }, function(err, msgs) { - assert.ifError(err); - assert.strictEqual(msgs.length, 1); - var message = msgs[0]; - - message.ack(done); + subscription.on('message', function() { + done(new Error('Should not have been called.')); }); }); - it('should be able to set a new ack deadline', function(done) { - var subscription = topic.subscription(SUB_NAMES[0]); - - subscription.pull({ - returnImmediately: true, - maxMessages: 1 - }, function(err, msgs) { - assert.ifError(err); + it('should receive the published messages', function(done) { + var subscription = topic.subscription(SUB_NAMES[1]); + var messageCount = 0; - assert.strictEqual(msgs.length, 1); + subscription.on('error', done); - var message = msgs[0]; + subscription.on('message', function(message) { + assert.deepEqual(message.data, new Buffer('hello')); - message.modifyAckDeadline(10000, done); + if (++messageCount === 10) { + subscription.close(done); + } }); }); - it('should receive the published message', function(done) { + it('should ack the message', function(done) { var subscription = topic.subscription(SUB_NAMES[1]); - subscription.pull({ - returnImmediately: true, - maxMessages: 1 - }, function(err, msgs) { - assert.ifError(err); - assert.strictEqual(msgs.length, 1); - assert.equal(msgs[0].data, 'hello'); + subscription.on('error', done); + subscription.on('message', ack); - msgs[0].ack(done); - }); + function ack(message) { + // remove listener to we only ack first message + subscription.removeListener('message', ack); + + message.ack(); + setTimeout(() => subscription.close(done), 2500); + } }); - it('should receive the chosen amount of results', function(done) { + it('should nack the message', function(done) { var subscription = topic.subscription(SUB_NAMES[1]); - var maxMessages = 3; - subscription.pull({ - returnImmediately: true, - maxMessages: maxMessages - }, function(err, messages) { - assert.ifError(err); + subscription.on('error', done); + subscription.on('message', nack); - assert.equal(messages.length, maxMessages); + function nack(message) { + // remove listener to we only ack first message + subscription.removeListener('message', nack); - done(); - }); + message.nack(); + setTimeout(() => subscription.close(done), 2500); + } }); }); @@ -471,20 +448,35 @@ describe('pubsub', function() { var subscription; var snapshot; - before(function(done) { + function deleteAllSnapshots() { + return pubsub.getSnapshots().then(function(data) { + return Promise.all(data[0].map(function(snapshot) { + return snapshot.delete(); + })); + }); + } + + function wait(milliseconds) { + return function() { + return new Promise(function(resolve) { + setTimeout(resolve, milliseconds); + }); + } + } + + before(function() { topic = pubsub.topic(TOPIC_NAMES[0]); publisher = topic.publisher(); subscription = topic.subscription(generateSubName()); snapshot = subscription.snapshot(SNAPSHOT_NAME); - subscription.create(done); + + return deleteAllSnapshots() + .then(wait(2500)) + .then(subscription.create.bind(subscription)); }); after(function() { - return pubsub.getSnapshots().then(function(data) { - return Promise.all(data[0].map(function(snapshot) { - return snapshot.delete(); - })); - }); + return deleteAllSnapshots(); }); it('should create a snapshot', function(done) { @@ -530,10 +522,19 @@ describe('pubsub', function() { }); function checkMessage() { - return subscription.pull().then(function(data) { - var message = data[0][0]; - assert.strictEqual(message.id, messageId); - return message.ack(); + return new Promise(function(resolve, reject) { + function onError(err) { + subscription.removeListener('message', onMessage); + reject(err); + } + + function onMessage(message) { + subscription.removeListener('error', onError); + resolve(message); + } + + subscription.once('error', onError); + subscription.once('message', onMessage); }); } From 4a9e0e49e70bce8c351719d5ec8c257a55e312e5 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 01:13:15 -0400 Subject: [PATCH 15/67] refactor publisher to not use queue object --- packages/pubsub/src/publisher.js | 73 +++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 977f45e1986..c9c5ef3c951 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -25,25 +25,46 @@ var common = require('@google-cloud/common'); var extend = require('extend'); var is = require('is'); -/** - * - */ -var Queue = require('./queue.js'); - /** * */ function Publisher(topic, options) { - options = options || {}; + options = extend(true, { + batching: { + maxBytes: Math.pow(1024, 2) * 5, + maxMessages: 1000, + maxMilliseconds: 1000 + } + }, options); this.topic = topic; this.api = topic.api; - var queueOptions = extend(options.batching, { - send: this.publish_.bind(this) + this.inventory_ = { + queued: [] + }; + + Object.defineProperty(this.inventory_, 'queueBytes', { + get: function() { + var size = 0; + + this.queued.forEach(function(message) { + size += message.data.size; + }); + + return size; + } }); - this.queue_ = new Queue(queueOptions); + this.settings = { + batching: { + maxBytes: Math.min(options.batching.maxBytes, Math.pow(1024, 2) * 9), + maxMessages: Math.min(options.batching.maxMessages, 1000), + maxMilliseconds: options.maxMilliseconds + } + }; + + this.timeoutHandle_ = null; } /** @@ -59,21 +80,34 @@ Publisher.prototype.publish = function(data, attrs, callback) { attrs = {}; } - var message = { + var opts = this.settings.batching; + + var hasMaxMessages = this.inventory_.queued.length >= opts.maxMessages; + var hasMaxBytes = (this.inventory_.queueBytes + data.size) >= opts.maxBytes; + + if (hasMaxMessages || hasMaxBytes) { + this.publish_(); + } else if (!this.timeoutHandle_) { + this.timeoutHandle_ = setTimeout( + this.publish_.bind(this), opts.maxMilliseconds); + } + + this.inventory_.queued.push({ data: data, attrs: attrs, - size: data.length - }; - - this.queue_.add(message, callback); + callback: callback + }); }; /** * This should never be called directly. */ -Publisher.prototype.publish_ = function(messages, done) { +Publisher.prototype.publish_ = function() { var self = this; + var messages = this.inventory_.queued; + this.inventory_.queued = []; + var reqOpts = { topic: this.topic.name, messages: messages.map(function(message) { @@ -89,13 +123,10 @@ Publisher.prototype.publish_ = function(messages, done) { method: 'publish', reqOpts: reqOpts }, function(err, resp) { - if (err) { - done(err); - return; - } + var messageIds = arrify(resp && resp.messageIds); - done(null, { - responses: resp.messageIds + messages.forEach(function(message, i) { + message.callback(err, messageIds[i]); }); }); }; From 5d6ed94aab8cf862996f5a17929bdc2174b7571f Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 01:14:41 -0400 Subject: [PATCH 16/67] create new getSubscriptions streaming method for topics --- packages/pubsub/src/topic.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index f6ec8d1211e..bb16fb01e72 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -21,6 +21,7 @@ 'use strict'; var common = require('@google-cloud/common'); +var extend = require('extend'); var is = require('is'); /** @@ -330,7 +331,10 @@ Topic.prototype.getSubscriptions = function(options, callback) { options = {}; } - var reqOpts = extend({}, options); + var reqOpts = extend({ + topic: this.name + }, options); + delete reqOpts.gaxOpts; this.request({ @@ -387,12 +391,8 @@ Topic.prototype.getSubscriptions = function(options, callback) { * this.end(); * }); */ -Topic.prototype.getSubscriptionsStream = function(options) { - options = options || {}; - options.topic = this; - - return this.pubsub.getSubscriptionsStream(options); -}; +Topic.prototype.getSubscriptionsStream = + common.paginator.streamify('getSubscriptions'); /** * From 543af0f7d95257a94d0f709129477acdc2c7d8fc Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 01:15:18 -0400 Subject: [PATCH 17/67] huge refactor of subscription/message objects --- packages/pubsub/src/histogram.js | 72 +++++ packages/pubsub/src/message.js | 16 +- packages/pubsub/src/subscription.js | 463 +++++++++++++++------------- 3 files changed, 316 insertions(+), 235 deletions(-) create mode 100644 packages/pubsub/src/histogram.js diff --git a/packages/pubsub/src/histogram.js b/packages/pubsub/src/histogram.js new file mode 100644 index 00000000000..f878049d99f --- /dev/null +++ b/packages/pubsub/src/histogram.js @@ -0,0 +1,72 @@ +/*! + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module pubsub/histogram + */ + +'use strict'; + +var MIN_VALUE = 10000; +var MAX_VALUE = 600000; + +/** + * + */ +function Histogram() { + this.data = new Map(); + this.length = 0; +} + +/** + * + */ +Histogram.prototype.add = function(value) { + value = Math.max(value, MIN_VALUE); + value = Math.min(value, MAX_VALUE); + + if (!this.data.has(value)) { + this.data.set(value, 0); + } + + var count = this.data.get(value); + this.data.set(value, count + 1); + this.length += 1; +}; + +/** + * + */ +Histogram.prototype.percentile = function(percent) { + percent = Math.min(percent, 100); + + var target = this.length - (this.length * (percent / 100)); + var keys = Array.from(this.data.keys()); + var key; + + for (var i = keys.length - 1; i > -1; i--) { + key = keys[i]; + target -= this.data.get(key); + + if (target <= 0) { + return key; + } + } + + return MIN_VALUE; +}; + +module.exports = Histogram; diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index 30446817d2b..ce7a219a94b 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -25,41 +25,31 @@ */ function Message(subscription, resp) { this.subscription = subscription; - this.api = subscription.api; this.ackId = resp.ackId; - this.id = resp.message.messageId; this.data = resp.message.data; this.attrs = resp.message.attributes; - this.deadline = null; var pt = resp.message.publishTime; var milliseconds = parseInt(pt.nanos, 10) / 1e6; this.publishTime = new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds); + this.received_ = Date.now(); } /** * */ Message.prototype.ack = function() { - this.subscription.ackQueue_.add(this); -}; - -/** - * - */ -Message.prototype.modifyAckDeadline = function(milliseconds) { - this.deadline = milliseconds; - this.subscription.modifyQueue_.add(this); + this.subscription.ack_(this); }; /** * */ Message.prototype.nack = function() { - this.modifyAckDeadline(0); + this.subscription.nack_(this); }; module.exports = Message; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index e789d52f666..bdad2251004 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -26,25 +26,26 @@ var events = require('events'); var extend = require('extend'); var is = require('is'); var os = require('os'); +var prop = require('propprop'); var util = require('util'); /** - * @type {module:pubsub/iam} + * @type {module:pubsub/histogram} * @private */ -var IAM = require('./iam.js'); +var Histogram = require('./histogram.js'); /** - * @type {module:pubsub/message} + * @type {module:pubsub/iam} * @private */ -var Message = require('./message.js'); +var IAM = require('./iam.js'); /** - * @type {module:pubsub/queue} + * @type {module:pubsub/message} * @private */ -var Queue = require('./queue.js'); +var Message = require('./message.js'); /** * @type {module:pubsub/snapshot} @@ -149,19 +150,29 @@ function Subscription(pubsub, name, options) { this.pubsub = pubsub; this.request = pubsub.request.bind(pubsub); - this.projectId = pubsub.projectId; + this.histogram = new Histogram(); + this.projectId = pubsub.projectId; this.name = Subscription.formatName_(pubsub.projectId, name); + this.connection = null; - this.encoding = options.encoding || 'utf-8'; - this.ackDeadline = options.ackDeadline || 600; - this.messages_ = new Set(); + this.ackDeadline = options.ackDeadline || 10000; + + this.inventory_ = { + lease: new Set(), + ack: [], + nack: [] + }; this.flowControl = extend({ highWaterMark: 16, maxMessages: Infinity }, options.flowControl); + this.flushTimeoutHandle_ = null; + this.leaseTimeoutHandle_ = null; + this.userClosed_ = false; + events.EventEmitter.call(this); if (options.topic) { @@ -204,15 +215,6 @@ function Subscription(pubsub, name, options) { */ this.iam = new IAM(pubsub, this.name); - this.ackQueue_ = new Queue(extend({ - send: this.ack_.bind(this) - }, options.batching)); - - this.modifyQueue_ = new Queue(extend({ - send: this.modifyAckDeadline_.bind(this) - }, options.batching)); - - this.listenForEvents_(); } @@ -234,51 +236,63 @@ Subscription.formatName_ = function(projectId, name) { }; /** - * This should never be called directly. + * */ -Subscription.prototype.ack_ = function(messages, callback) { - var self = this; - var ackIds = messages.reduce(function(ackIds, message) { - ackIds.push(message.ackId); - return ackIds; - }, []); +Subscription.prototype.ack_ = function(message) { + this.breakLease_(message); + this.histogram.add(Date.now() - message.received_); if (this.connection) { - this.connection.write({ ackIds: ackIds }); - setImmediate(onComplete); + this.connection.write({ ackIds: [message.ackId] }); return; } - var reqOpts = { - ackIds: ackIds, - subscription: this.name - }; - - this.request({ - client: 'subscriberClient', - method: 'acknowledge', - reqOpts: reqOpts - }, onComplete); + this.inventory_.ack.push(message); + this.setFlushTimeout_(); +}; - function onComplete(err, resp) { - messages.forEach(function(message) { - self.messages_.delete(message); - }); +/** + * + */ +Subscription.prototype.breakLease_ = function(message) { + this.inventory_.lease.delete(message); - if (self.connection.isPaused()) { - self.connection.resume(); - } + if (this.connection && this.connection.isPaused()) { + this.connection.resume(); + } - callback(err, resp); + if (!this.inventory_.lease.size) { + clearTimeout(this.leaseTimeoutHandle_); + this.leaseTimeoutHandle_ = null; } +} + +/** + * + */ +Subscription.prototype.close = function(callback) { + this.userClosed_ = true; + + clearTimeout(this.leaseTimeoutHandle_); + this.leaseTimeoutHandle_ = null; + + clearTimeout(this.flushTimeoutHandle_); + this.flushTimeoutHandle_ = null; + + this.flushQueues_(); + this.closeConnection_(callback); }; /** - * @private + * */ -Subscription.prototype.closeConnection_ = function() { - this.connection.end(); - this.connection = null; +Subscription.prototype.closeConnection_ = function(callback) { + if (this.connection) { + this.connection.end(callback || common.util.noop); + this.connection = null; + } else if (is.fn(callback)) { + setImmediate(callback); + } }; /** @@ -385,12 +399,80 @@ Subscription.prototype.delete = function(gaxOpts, callback) { }, function(err, resp) { if (!err) { self.removeAllListeners('message'); + self.close(); } callback(err, resp); }); }; +/** + * + */ +Subscription.prototype.flushQueues_ = function() { + var acks = this.inventory_.ack; + var nacks = this.inventory_.nack; + + if (!acks.length && !nacks.length) { + return; + } + + this.inventory_.ack = []; + this.inventory_.nack = []; + + var getAckId = prop('ackId'); + + if (this.connection) { + var reqOpts = {}; + + if (acks.length) { + reqOpts.ackIds = acks.map(getAckId); + } + + if (nacks.length) { + reqOpts.modifyDeadlineAckIds = nacks.map(getAckId); + reqOpts.modifyDeadlineSeconds = Array(nacks.length).fill(0); + } + + this.connection.write(reqOpts); + return; + } + + var self = this; + + if (acks.length) { + this.request({ + client: 'subscriberClient', + method: 'acknowledge', + reqOpts: { + subscription: this.name, + ackIds: acks.map(getAckId) + } + }, function(err) { + if (err) { + self.emit('error', err); + } + }); + } + + if (nacks.length) { + this.request({ + client: 'subscriberClient', + method: 'modifyAckDeadline', + reqOpts: { + subscription: this.name, + ackIds: nacks.map(getAckId), + ackDeadlineSeconds: 0 + } + }, function(err) { + if (err) { + self.emit('error', err); + } + }); + } +}; + + /** * @param {function} callback - The callback function. * @param {?error} callback.err - An error returned while making this @@ -415,6 +497,18 @@ Subscription.prototype.getMetadata = function(gaxOpts, callback) { }, callback); }; +/** + * + */ +Subscription.prototype.leaseMessage_ = function(data) { + var message = new Message(this, data); + + this.inventory_.lease.add(message); + this.setLeaseTimeout_(); + + return message; +}; + /** * Begin listening for events on the subscription. This method keeps track of * how many message listeners are assigned, and then removed, making sure @@ -448,86 +542,6 @@ Subscription.prototype.listenForEvents_ = function() { }); }; -/** - * - */ -Subscription.prototype.createMessage_ = function(data) { - var message = new Message(self, data); - this.messages_.add(message); - return message; -}; - -/** - * - */ -Subscription.prototype.modifyAckDeadline_ = function(messages, callback) { - var self = this; - var reqOpts; - - if (this.connection) { - reqOpts = messages.reduce(function(reqOpts, message) { - reqOpts.modifyDeadlineAckIds.push(message.ackId); - reqOpts.modifyDeadlineSeconds.push(message.deadline / 1000); - return reqOpts; - }, { - modifyDeadlineAckIds: [], - modifyDeadlineSeconds: [] - }); - - this.connection.write(reqOpts); - setImmediate(onComplete); - return; - } - - var requests = groupByDeadline(messages).map(function(reqOpts) { - return self.request({ - client: 'subscriberClient', - method: 'modifyAckDeadline', - reqOpts: reqOpts - }).catch(function(err) { - self.emit('error', err); - }); - }); - - Promise.all(requests) - .then(onComplete.bind(null, null), onComplete); - - function groupByDeadline(messages) { - var requests = new Map(); - - messages.forEach(function(message) { - if (!requests.has(message.deadline)) { - requests.set(message.deadline, { - subscription: self.name, - ackIds: [], - ackDeadlineSeconds: message.deadline / 1000 - }); - } - - var request = requests.get(message.deadline); - request.ackIds.push(message.ackId); - }); - - return Array.from(requests.values()); - } - - function onComplete(err, resp) { - messages.forEach(function(message) { - if (message.deadline === 0) { - self.messages_.delete(message); - } - }); - - if (self.connection.isPaused()) { - if (self.messages_.size < self.flowControl.maxMessages) { - self.connection.resume(); - } - } - - callback(err, resp); - } -}; - /** * @param {object} config - The push config. * @param {string} config.pushEndpoint @@ -552,6 +566,24 @@ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { }, callback); }; +/** + * + */ +Subscription.prototype.nack_ = function(message) { + this.breakLease_(message); + + if (this.connection) { + this.connection.write({ + modifyDeadlineAckIds: [message.ackId], + modifyDeadlineSeconds: [0] + }); + return; + } + + this.inventory_.nack.push(message); + this.setFlushTimeout_(); +} + /** * @private */ @@ -560,7 +592,7 @@ Subscription.prototype.openConnection_ = function() { var reqOpts = { subscription: this.name, - streamAckDeadlineSeconds: this.ackDeadline + streamAckDeadlineSeconds: this.ackDeadline / 1000 }; this.request({ @@ -576,9 +608,9 @@ Subscription.prototype.openConnection_ = function() { return; } - self.connection = requestFn(); + var connection = requestFn(); - self.connection.on('error', function(err) { + connection.on('error', function(err) { // in the event of a connection error, we will attempt to re-establish // a new connection if the grpc error code is something safe to retry on var retryCodes = [ @@ -588,6 +620,7 @@ Subscription.prototype.openConnection_ = function() { ]; if (retryCodes.indexOf(err.code) > -1) { + self.closeConnection_(); self.openConnection_(); return; } @@ -595,116 +628,81 @@ Subscription.prototype.openConnection_ = function() { self.emit('error', err); }); - self.connection.on('data', function(data) { + connection.on('data', function(data) { arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', self.createMessage_(message))); + self.emit('message', self.leaseMessage_(message)); }); - if (!self.connection.isPaused()) { - if (self.messages_.size >= self.flowControl.maxMessages) { - self.connection.pause(); - } + if (self.inventory_.lease.size >= self.flowControl.maxMessages) { + connection.pause(); } }); - self.connection.write(reqOpts); + // this event is the closest thing that we have to an indication that a + // connection was opened successfully. when we know it was opened we'll + // expose it to the client. this should prevent writes from happening + // before it was opened + connection.on('metadata', function() { + self.connection = connection; + + clearTimeout(self.flushTimeoutHandle_); + self.flushTimeoutHandle_ = null; + self.flushQueues_(); + }); + + connection.on('close', function() { + if (!self.userClosed_) { + self.closeConnection_(); + self.openConnection_(); + } + }); + + connection.write(reqOpts); }); }; /** - * Pull messages from the subscribed topic. If messages were found, your - * callback is executed with an array of message objects. * - * Note that messages are pulled automatically once you register your first - * event listener to the subscription, thus the call to `pull` is handled for - * you. If you don't want to start pulling, simply don't register a - * `subscription.on('message', function() {})` event handler. - * - * @todo Should not be racing with other pull. - * - * @resource [Subscriptions: pull API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull} - * - * @param {object=} options - Configuration object. - * @param {number} options.maxMessages - Limit the amount of messages pulled. - * @param {boolean} options.returnImmediately - If set, the system will respond - * immediately. Otherwise, wait until new messages are available. Returns if - * timeout is reached. - * @param {function} callback - The callback function. - * - * @example - * //- - * // Pull all available messages. - * //- - * subscription.pull(function(err, messages) { - * // messages = [ - * // { - * // ackId: '', // ID used to acknowledge its receival. - * // id: '', // Unique message ID. - * // data: '', // Contents of the message. - * // attributes: {} // Attributes of the message. - * // - * // Helper functions: - * // ack(callback): // Ack the message. - * // skip(): // Free up 1 slot on the sub's maxInProgress value. - * // }, - * // // ... - * // ] - * }); - * - * //- - * // Pull a single message. - * //- - * var opts = { - * maxMessages: 1 - * }; - * - * subscription.pull(opts, function(err, messages, apiResponse) {}); - * - * //- - * // If the callback is omitted, we'll return a Promise. - * //- - * subscription.pull(opts).then(function(data) { - * var messages = data[0]; - * var apiResponse = data[1]; - * }); */ -Subscription.prototype.pull = function(options, callback) { +Subscription.prototype.renewLeases_ = function() { var self = this; - if (is.fn(options)) { - callback = options; - options = {}; + this.leaseTimeoutHandle_ = null; + + if (!this.inventory_.lease.size) { + return; } - var reqOpts = extend({ - subscription: this.name, - }, options); - delete reqOpts.gaxOpts; + var ackIds = Array.from(this.inventory_.lease).map(prop('ackId')); + + this.ackDeadline = this.histogram.percentile(99); + var ackDeadlineSeconds = this.ackDeadline / 1000; + + if (this.connection) { + this.connection.write({ + modifyDeadlineAckIds: ackIds, + modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) + }); + + this.setLeaseTimeout_(); + return; + } this.request({ client: 'subscriberClient', - method: 'pull', - reqOpts: reqOpts, - gaxOpts: gaxOpts - }, function(err, resp) { + method: 'modifyAckDeadline', + reqOpts: { + subscription: self.name, + ackIds: ackIds, + ackDeadlineSeconds: ackDeadlineSeconds + } + }, function(err) { if (err) { - if (err.code !== 4) { - callback(err, null, resp); - return; - } - - // Simulate a server timeout where no messages were received. - resp = { - receivedMessages: [] - }; + self.emit('error', err); } - - var messages = arrify(resp.receivedMessages).map(function(message) { - return self.createMessage_(message); - }); - - callback(null, messages, resp); }); + + this.setLeaseTimeout_(); }; /** @@ -745,7 +743,7 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { }; if (is.string(snapshot)) { - reqOpts.snapshot = Snapshot.formatName_(this.parent.projectId, snapshot); + reqOpts.snapshot = Snapshot.formatName_(this.pubsub.projectId, snapshot); } else if (is.date(snapshot)) { reqOpts.time = { seconds: Math.floor(snapshot.getTime() / 1000), @@ -763,6 +761,27 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { }, callback) }; +/** + * + */ +Subscription.prototype.setFlushTimeout_ = function() { + if (!this.flushTimeoutHandle_) { + this.flushTimeoutHandle_ = setTimeout(this.flushQueues_.bind(this), 1000); + } +}; + +/** + * + */ +Subscription.prototype.setLeaseTimeout_ = function() { + if (this.leaseTimeoutHandle_) { + return; + } + + var timeout = Math.random() * this.ackDeadline * 0.9; + this.leaseTimeoutHandle_ = setTimeout(this.renewLeases_.bind(this), timeout); +}; + /** * */ From 955d7eba2baeb27de08becdfed1ff754f2749c0e Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 01:15:35 -0400 Subject: [PATCH 18/67] remove queue object --- packages/pubsub/src/queue.js | 202 ----------------------------------- 1 file changed, 202 deletions(-) delete mode 100644 packages/pubsub/src/queue.js diff --git a/packages/pubsub/src/queue.js b/packages/pubsub/src/queue.js deleted file mode 100644 index 8a097d5f933..00000000000 --- a/packages/pubsub/src/queue.js +++ /dev/null @@ -1,202 +0,0 @@ -/*! - * Copyright 2017 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*! - * @module pubsub/queue - */ - -'use strict'; - -var common = require('@google-cloud/common'); -var prop = require('propprop'); - -/** - * Semi-generic Queue object used to queue up multiple requests while limiting - * the number of concurrent requests. - * - * @constructor - * @alias module:pubsub/queue - * - * @param {object} options - Queue configuration settings. - * @param {number} options.maxBytes - Max size of data to be sent per request. - * @param {number} options.maxMessages - Max number of messages to be sent per - * request. - * @param {number} options.maxMilliseconds - Max time to wait before sending - * data. - * @param {number} options.maxRequests - Max number of concurrent requests - * allowed. - * @param {function(messages, callback)} options.sendHandler - Function to be - * called with message queue. - * - * @example - * var queue = new Queue({ - * maxBytes: Math.pow(1024, 2) * 5, - * maxMessages: 1000, - * maxMilliseconds: 1000, - * maxRequests: 5, - * send: function(messages, done) { - * var reqOpts = { - * topic: self.topic.name, - * messages = messages.map(function(message) { - * return { - * data: message.data, - * attributes: message.attrs - * }; - * }) - * }; - * - * return self.api.Publisher.publish(reqOpts, function(err, resp) { - * if (err) { - * done(err); - * return; - * } - * - * done(null, { - * responses: resp.messageIds - * }) - * }); - * } - * }); - */ -function Queue(options) { - var maxBytes = options.maxBytes || Math.pow(1024, 2) * 5; - - this.maxBytes = Math.min(maxBytes, Math.pow(1024, 2) * 9); - this.maxMessages = Math.min(options.maxMessages || 1000, 1000); - this.maxMilliseconds = options.maxMilliseconds || 1000; - this.sendHandler = options.send; - - this.inventory = { - queued: new Set(), - inFlight: new Set() - }; - - Object.defineProperty(this.inventory, 'queueBytes', { - get: function() { - var size = 0; - - this.queued.forEach(function(message) { - size += message.data.size; - }); - - return size; - } - }); - - this.intervalHandle_ = null; -} - -/** - * Adds message to the queue. - * - * @param {*} data - The message data. - * @param {function} callback - Callback to be ran once message is sent. - * - * @example - * queue.add(1234, function(err, resp) {}); - */ -Queue.prototype.add = function(data, callback) { - var hasMaxMessages = this.inventory.queued.size >= this.maxMessages; - var hasMaxBytes = (this.inventory.queueBytes + data.size) >= this.maxBytes; - - if (hasMaxMessages || hasMaxBytes) { - this.send(this.getNextMessageBatch()); - } else if (!this.intervalHandle_) { - this.startSendInterval(); - } - - this.inventory.queued.add({ - data: data, - callback: callback || common.util.noop - }); -}; - -/** - * Starts the send interval. - * - * @example - * queue.startSendInterval(); - */ -Queue.prototype.startSendInterval = function() { - if (this.intervalHandle_) { - return; - } - - var self = this; - - this.intervalHandle_ = setInterval(function() { - self.send(self.getNextMessageBatch()); - - if (self.inventory.queued.size === 0) { - clearInterval(self.intervalHandle_); - self.intervalHandle_ = null; - } - }, this.maxMilliseconds); -}; - -/** - * Prepares next queue of messages to be sent. - * - * @return array - * - * @example - * var messages = queue.getNextMessageBatch(); - * queue.send(messages); - */ -Queue.prototype.getNextMessageBatch = function() { - var messages = Array.from(this.inventory.queued); - - this.inventory.queued.clear(); - return messages; -}; - -/** - * Sends out messages - * - * @param {array} messages - Array of messages. - * - * @example - * var messages = [{ - * data: 123, - * callback: function(err, resp) {} - * }]; - * - * queue.send(messages); - */ -Queue.prototype.send = function(messages) { - if (this.activeRequests_ >= this.maxRequests) { - this.scheduleSend(); - return; - } - - var self = this; - - messages.forEach(function(message) { - self.inventory.inFlight.add(message); - }); - - this.sendHandler(messages.map(prop('data')), function(err, resp) { - messages.forEach(function(message, i) { - var error = Array.isArray(err) ? err[i] : err; - var response = resp && resp.responses ? resp.responses[i] : resp; - - message.callback(error, response); - self.inventory.inFlight.delete(message); - }); - }); -}; - -module.exports = Queue; From 5827b04566503b4717ca3411347f852e5a530281 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 03:29:25 -0400 Subject: [PATCH 19/67] kill timeout if publish queue fills --- packages/pubsub/src/publisher.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index c9c5ef3c951..a4a65aa21a1 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -86,6 +86,8 @@ Publisher.prototype.publish = function(data, attrs, callback) { var hasMaxBytes = (this.inventory_.queueBytes + data.size) >= opts.maxBytes; if (hasMaxMessages || hasMaxBytes) { + clearTimeout(this.timeoutHandle_); + this.timeoutHandle_ = null; this.publish_(); } else if (!this.timeoutHandle_) { this.timeoutHandle_ = setTimeout( From cce7949670b9bcce0b7c7492f589eef2a868f09f Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 14:21:15 -0400 Subject: [PATCH 20/67] optimize in memory message storing --- packages/pubsub/src/publisher.js | 7 +++++-- packages/pubsub/src/subscription.js | 30 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index a4a65aa21a1..0326a7b0481 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -24,6 +24,7 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); var extend = require('extend'); var is = require('is'); +var prop = require('propprop'); /** * @@ -108,6 +109,8 @@ Publisher.prototype.publish_ = function() { var self = this; var messages = this.inventory_.queued; + var callbacks = messages.map(prop('callback')); + this.inventory_.queued = []; var reqOpts = { @@ -127,8 +130,8 @@ Publisher.prototype.publish_ = function() { }, function(err, resp) { var messageIds = arrify(resp && resp.messageIds); - messages.forEach(function(message, i) { - message.callback(err, messageIds[i]); + callbacks.forEach(function(callback, i) { + callback(err, messageIds[i]); }); }); }; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index bdad2251004..da49f7176f2 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -159,7 +159,7 @@ function Subscription(pubsub, name, options) { this.ackDeadline = options.ackDeadline || 10000; this.inventory_ = { - lease: new Set(), + lease: [], ack: [], nack: [] }; @@ -247,7 +247,7 @@ Subscription.prototype.ack_ = function(message) { return; } - this.inventory_.ack.push(message); + this.inventory_.ack.push(message.ackId); this.setFlushTimeout_(); }; @@ -255,13 +255,14 @@ Subscription.prototype.ack_ = function(message) { * */ Subscription.prototype.breakLease_ = function(message) { - this.inventory_.lease.delete(message); + var messageIndex = this.inventory_.lease.indexOf(message.ackId); + this.inventory_.lease.splice(0, messageIndex); if (this.connection && this.connection.isPaused()) { this.connection.resume(); } - if (!this.inventory_.lease.size) { + if (!this.inventory_.lease.length) { clearTimeout(this.leaseTimeoutHandle_); this.leaseTimeoutHandle_ = null; } @@ -420,17 +421,15 @@ Subscription.prototype.flushQueues_ = function() { this.inventory_.ack = []; this.inventory_.nack = []; - var getAckId = prop('ackId'); - if (this.connection) { var reqOpts = {}; if (acks.length) { - reqOpts.ackIds = acks.map(getAckId); + reqOpts.ackIds = acks; } if (nacks.length) { - reqOpts.modifyDeadlineAckIds = nacks.map(getAckId); + reqOpts.modifyDeadlineAckIds = nacks; reqOpts.modifyDeadlineSeconds = Array(nacks.length).fill(0); } @@ -446,7 +445,7 @@ Subscription.prototype.flushQueues_ = function() { method: 'acknowledge', reqOpts: { subscription: this.name, - ackIds: acks.map(getAckId) + ackIds: acks } }, function(err) { if (err) { @@ -461,7 +460,7 @@ Subscription.prototype.flushQueues_ = function() { method: 'modifyAckDeadline', reqOpts: { subscription: this.name, - ackIds: nacks.map(getAckId), + ackIds: nacks, ackDeadlineSeconds: 0 } }, function(err) { @@ -503,7 +502,7 @@ Subscription.prototype.getMetadata = function(gaxOpts, callback) { Subscription.prototype.leaseMessage_ = function(data) { var message = new Message(this, data); - this.inventory_.lease.add(message); + this.inventory_.lease.push(message.ackId); this.setLeaseTimeout_(); return message; @@ -580,7 +579,7 @@ Subscription.prototype.nack_ = function(message) { return; } - this.inventory_.nack.push(message); + this.inventory_.nack.push(message.ackId); this.setFlushTimeout_(); } @@ -633,7 +632,7 @@ Subscription.prototype.openConnection_ = function() { self.emit('message', self.leaseMessage_(message)); }); - if (self.inventory_.lease.size >= self.flowControl.maxMessages) { + if (self.inventory_.lease.length >= self.flowControl.maxMessages) { connection.pause(); } }); @@ -669,12 +668,11 @@ Subscription.prototype.renewLeases_ = function() { this.leaseTimeoutHandle_ = null; - if (!this.inventory_.lease.size) { + if (!this.inventory_.lease.length) { return; } - var ackIds = Array.from(this.inventory_.lease).map(prop('ackId')); - + var ackIds = this.inventory_.lease; this.ackDeadline = this.histogram.percentile(99); var ackDeadlineSeconds = this.ackDeadline / 1000; From cdd9803d41baa3566535d08a2c807f25546c2aab Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 27 Jun 2017 14:22:41 -0400 Subject: [PATCH 21/67] let gax handle errors --- packages/pubsub/src/subscription.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index da49f7176f2..bbf22da8129 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -610,20 +610,6 @@ Subscription.prototype.openConnection_ = function() { var connection = requestFn(); connection.on('error', function(err) { - // in the event of a connection error, we will attempt to re-establish - // a new connection if the grpc error code is something safe to retry on - var retryCodes = [ - 2, // Unknown Error - 4, // DeadlineExceeded - 14 // Unavailable - ]; - - if (retryCodes.indexOf(err.code) > -1) { - self.closeConnection_(); - self.openConnection_(); - return; - } - self.emit('error', err); }); From 91485d89d81a2f177b2780646ea3839d4db4c4a7 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 10 Jul 2017 15:06:11 -0400 Subject: [PATCH 22/67] use maxBytes option to do inventory management --- packages/pubsub/src/subscription.js | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index bbf22da8129..fde1231137b 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -161,11 +161,12 @@ function Subscription(pubsub, name, options) { this.inventory_ = { lease: [], ack: [], - nack: [] + nack: [], + bytes: 0 }; this.flowControl = extend({ - highWaterMark: 16, + maxBytes: os.freemem() * 0.2, maxMessages: Infinity }, options.flowControl); @@ -256,10 +257,14 @@ Subscription.prototype.ack_ = function(message) { */ Subscription.prototype.breakLease_ = function(message) { var messageIndex = this.inventory_.lease.indexOf(message.ackId); + this.inventory_.lease.splice(0, messageIndex); + this.inventory_.bytes -= message.data.length; - if (this.connection && this.connection.isPaused()) { - this.connection.resume(); + if (this.connection) { + if (this.connection.isPaused() && !this.hasMaxMessages_()) { + this.connection.resume(); + } } if (!this.inventory_.lease.length) { @@ -496,6 +501,14 @@ Subscription.prototype.getMetadata = function(gaxOpts, callback) { }, callback); }; +/** + * + */ +Subscription.prototype.hasMaxMessages_ = function() { + return this.inventory_.lease.length >= this.flowControl.maxMessages || + this.inventory_.bytes >= this.flowControl.maxBytes; +}; + /** * */ @@ -503,6 +516,7 @@ Subscription.prototype.leaseMessage_ = function(data) { var message = new Message(this, data); this.inventory_.lease.push(message.ackId); + this.inventory_.bytes += message.data.length; this.setLeaseTimeout_(); return message; @@ -597,10 +611,7 @@ Subscription.prototype.openConnection_ = function() { this.request({ client: 'subscriberClient', method: 'streamingPull', - returnFn: true, - reqOpts: { - highWaterMark: self.flowControl.highWaterMark - } + returnFn: true }, function(err, requestFn) { if (err) { self.emit('error', err); @@ -618,7 +629,7 @@ Subscription.prototype.openConnection_ = function() { self.emit('message', self.leaseMessage_(message)); }); - if (self.inventory_.lease.length >= self.flowControl.maxMessages) { + if (self.hasMaxMessages_()) { connection.pause(); } }); From d605fe2e699c5db592743fadef35595a723260fe Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 10 Jul 2017 18:51:39 -0400 Subject: [PATCH 23/67] add connection pooling --- packages/pubsub/package.json | 1 + packages/pubsub/src/connection-pool.js | 190 +++++++++++++++++++++++++ packages/pubsub/src/message.js | 3 +- packages/pubsub/src/subscription.js | 190 +++++++++++++------------ 4 files changed, 291 insertions(+), 93 deletions(-) create mode 100644 packages/pubsub/src/connection-pool.js diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index 35453e8c38a..0d1233acf3c 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -54,6 +54,7 @@ "@google-cloud/common": "^0.13.0", "@google-cloud/common-grpc": "^0.4.0", "arrify": "^1.0.0", + "async-each": "^1.0.1", "extend": "^3.0.0", "google-gax": "^0.13.0", "google-proto-files": "^0.12.0", diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js new file mode 100644 index 00000000000..75a621cef9e --- /dev/null +++ b/packages/pubsub/src/connection-pool.js @@ -0,0 +1,190 @@ +/*! + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module pubsub/connectionPool + */ + +'use strict'; + +var arrify = require('arrify'); +var common = require('@google-cloud/common'); +var each = require('async-each'); +var events = require('events'); +var extend = require('extend'); +var is = require('is'); +var util = require('util'); +var uuid = require('uuid'); + +/** + * @type {module:pubsub/message} + * @private + */ +var Message = require('./message.js'); + +/** + * + */ +function ConnectionPool(subscription, options) { + this.subscription = subscription; + this.connections = new Map(); + this.isPaused = false; + this.isOpen = false; + + this.settings = extend({ + maxConnections: 5, + ackDeadline: 10000 + }, options); + + events.EventEmitter.call(this); + + this.open(); +} + +util.inherits(ConnectionPool, events.EventEmitter); + +/** + * + */ +ConnectionPool.prototype.acquire = function(id, callback) { + var self = this; + + if (is.fn(id)) { + callback = id; + id = getFirstConnectionId(); + } + + id = id || getFirstConnectionId(); + + if (!this.isOpen) { + callback(new Error('Connection pool is closed.')); + return; + } + + if (this.connections.has(id)) { + callback(null, this.connections.get(id)); + return; + } + + this.once('connected', function() { + self.acquire(id, callback); + }); + + function getFirstConnectionId() { + return self.connections.keys().next().value; + } +}; + +/** + * + */ +ConnectionPool.prototype.createConnection = function(id) { + var self = this; + + this.subscription.request({ + client: 'subscriberClient', + method: 'streamingPull', + returnFn: true + }, function(err, requestFn) { + if (err) { + self.emit('error', err); + return; + } + + var connection = requestFn(); + + connection.on('error', function(err) { + self.emit('error', err); + }); + + connection.on('data', function(data) { + arrify(data.receivedMessages).forEach(function(message) { + self.emit('message', new Message(self.subscription, id, message)); + }); + }); + + connection.once('metadata', function() { + self.connections.set(id, connection); + self.emit('connected'); + }); + + connection.once('close', function() { + self.connections.delete(id); + + if (self.isOpen) { + self.createConnection(id); + } + }); + + if (self.isPaused) { + connection.pause(); + } + + connection.write({ + subscription: self.subscription.name, + streamAckDeadlineSeconds: self.settings.ackDeadline / 1000 + }); + }); +}; + +/** + * + */ +ConnectionPool.prototype.drain = function(callback) { + var connections = Array.from(this.connections.values()); + + this.isOpen = false; + callback = callback || common.util.noop; + + each(connections, function(connection, callback) { + connection.end(callback); + }, callback); +}; + +/** + * + */ +ConnectionPool.prototype.open = function() { + for (var i = 0; i < this.settings.maxConnections; i++) { + this.createConnection(i); + } + + this.isOpen = true; +}; + +/** + * + */ +ConnectionPool.prototype.pause = function() { + this.isPaused = true; + + this.connections.forEach(function(connection) { + connection.pause(); + }); +}; + +/** + * + */ +ConnectionPool.prototype.resume = function() { + this.isPaused = false; + + this.connections.forEach(function(connection) { + connection.resume(); + }); +}; + +module.exports = ConnectionPool; diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js index ce7a219a94b..c796e4c0fec 100644 --- a/packages/pubsub/src/message.js +++ b/packages/pubsub/src/message.js @@ -23,8 +23,9 @@ /** * */ -function Message(subscription, resp) { +function Message(subscription, connectionId, resp) { this.subscription = subscription; + this.connectionId = connectionId; this.ackId = resp.ackId; this.id = resp.message.messageId; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index fde1231137b..276523f569b 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -30,22 +30,22 @@ var prop = require('propprop'); var util = require('util'); /** - * @type {module:pubsub/histogram} + * @type {module:pubsub/connectionPool} * @private */ -var Histogram = require('./histogram.js'); +var ConnectionPool = require('./connection-pool.js'); /** - * @type {module:pubsub/iam} + * @type {module:pubsub/histogram} * @private */ -var IAM = require('./iam.js'); +var Histogram = require('./histogram.js'); /** - * @type {module:pubsub/message} + * @type {module:pubsub/iam} * @private */ -var Message = require('./message.js'); +var IAM = require('./iam.js'); /** * @type {module:pubsub/snapshot} @@ -155,8 +155,9 @@ function Subscription(pubsub, name, options) { this.projectId = pubsub.projectId; this.name = Subscription.formatName_(pubsub.projectId, name); - this.connection = null; + this.connectionPool = null; this.ackDeadline = options.ackDeadline || 10000; + this.maxConnections = options.maxConnections; this.inventory_ = { lease: [], @@ -243,13 +244,22 @@ Subscription.prototype.ack_ = function(message) { this.breakLease_(message); this.histogram.add(Date.now() - message.received_); - if (this.connection) { - this.connection.write({ ackIds: [message.ackId] }); + if (!this.connectionPool) { + this.inventory_.ack.push(message.ackId); + this.setFlushTimeout_(); return; } - this.inventory_.ack.push(message.ackId); - this.setFlushTimeout_(); + var self = this; + + this.connectionPool.acquire(message.connectionId, function(err, connection) { + if (err) { + self.emit('error', err); + return; + } + + connection.write({ ackIds: [message.ackId] }); + }); }; /** @@ -261,9 +271,9 @@ Subscription.prototype.breakLease_ = function(message) { this.inventory_.lease.splice(0, messageIndex); this.inventory_.bytes -= message.data.length; - if (this.connection) { - if (this.connection.isPaused() && !this.hasMaxMessages_()) { - this.connection.resume(); + if (this.connectionPool) { + if (this.connectionPool.isPaused && !this.hasMaxMessages_()) { + this.connectionPool.resume(); } } @@ -293,9 +303,9 @@ Subscription.prototype.close = function(callback) { * */ Subscription.prototype.closeConnection_ = function(callback) { - if (this.connection) { - this.connection.end(callback || common.util.noop); - this.connection = null; + if (this.connectionPool) { + this.connectionPool.drain(callback || common.util.noop); + this.connectionPool = null; } else if (is.fn(callback)) { setImmediate(callback); } @@ -416,6 +426,8 @@ Subscription.prototype.delete = function(gaxOpts, callback) { * */ Subscription.prototype.flushQueues_ = function() { + var self = this; + var acks = this.inventory_.ack; var nacks = this.inventory_.nack; @@ -423,27 +435,32 @@ Subscription.prototype.flushQueues_ = function() { return; } - this.inventory_.ack = []; - this.inventory_.nack = []; + if (this.connectionPool) { + this.connectionPool.acquire(function(err, connection) { + if (err) { + self.emit('error', err); + return; + } - if (this.connection) { - var reqOpts = {}; + var reqOpts = {}; - if (acks.length) { - reqOpts.ackIds = acks; - } + if (acks.length) { + reqOpts.ackIds = acks; + } - if (nacks.length) { - reqOpts.modifyDeadlineAckIds = nacks; - reqOpts.modifyDeadlineSeconds = Array(nacks.length).fill(0); - } + if (nacks.length) { + reqOpts.modifyDeadlineAckIds = nacks; + reqOpts.modifyDeadlineSeconds = Array(nacks.length).fill(0); + } + + connection.write(reqOpts); - this.connection.write(reqOpts); + self.inventory_.ack = []; + self.inventory_.nack = []; + }); return; } - var self = this; - if (acks.length) { this.request({ client: 'subscriberClient', @@ -455,6 +472,8 @@ Subscription.prototype.flushQueues_ = function() { }, function(err) { if (err) { self.emit('error', err); + } else { + self.inventory_.ack = []; } }); } @@ -471,6 +490,8 @@ Subscription.prototype.flushQueues_ = function() { }, function(err) { if (err) { self.emit('error', err); + } else { + self.inventory_.nack = []; } }); } @@ -512,9 +533,7 @@ Subscription.prototype.hasMaxMessages_ = function() { /** * */ -Subscription.prototype.leaseMessage_ = function(data) { - var message = new Message(this, data); - +Subscription.prototype.leaseMessage_ = function(message) { this.inventory_.lease.push(message.ackId); this.inventory_.bytes += message.data.length; this.setLeaseTimeout_(); @@ -585,16 +604,23 @@ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { Subscription.prototype.nack_ = function(message) { this.breakLease_(message); - if (this.connection) { - this.connection.write({ - modifyDeadlineAckIds: [message.ackId], - modifyDeadlineSeconds: [0] - }); + if (!this.connectionPool) { + this.inventory_.nack.push(message.ackId); + this.setFlushTimeout_(); return; } - this.inventory_.nack.push(message.ackId); - this.setFlushTimeout_(); + this.connectionPool.acquire(message.connectionId, function(err, connection) { + if (err) { + self.emit('error', err); + return; + } +console.log('nacking'); + connection.write({ + modifyDeadlineAckIds: [message.ackId], + modifyDeadlineSeconds: [0] + }); + }); } /** @@ -603,57 +629,27 @@ Subscription.prototype.nack_ = function(message) { Subscription.prototype.openConnection_ = function() { var self = this; - var reqOpts = { - subscription: this.name, - streamAckDeadlineSeconds: this.ackDeadline / 1000 - }; - - this.request({ - client: 'subscriberClient', - method: 'streamingPull', - returnFn: true - }, function(err, requestFn) { - if (err) { - self.emit('error', err); - return; - } - - var connection = requestFn(); - - connection.on('error', function(err) { - self.emit('error', err); - }); - - connection.on('data', function(data) { - arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', self.leaseMessage_(message)); - }); - - if (self.hasMaxMessages_()) { - connection.pause(); - } - }); + var pool = this.connectionPool = new ConnectionPool(this, { + ackDeadline: this.ackDeadline, + maxConnections: this.maxConnections + }); - // this event is the closest thing that we have to an indication that a - // connection was opened successfully. when we know it was opened we'll - // expose it to the client. this should prevent writes from happening - // before it was opened - connection.on('metadata', function() { - self.connection = connection; + pool.on('error', function(err) { + self.emit('error', err); + }); - clearTimeout(self.flushTimeoutHandle_); - self.flushTimeoutHandle_ = null; - self.flushQueues_(); - }); + pool.on('message', function(message) { + self.emit('message', self.leaseMessage_(message)); - connection.on('close', function() { - if (!self.userClosed_) { - self.closeConnection_(); - self.openConnection_(); - } - }); + if (self.hasMaxMessages_()) { + pool.pause(); + } + }); - connection.write(reqOpts); + pool.once('connected', function() { + clearTimeout(self.flushTimeoutHandle_); + self.flushTimeoutHandle_ = null; + self.flushQueues_(); }); }; @@ -673,10 +669,20 @@ Subscription.prototype.renewLeases_ = function() { this.ackDeadline = this.histogram.percentile(99); var ackDeadlineSeconds = this.ackDeadline / 1000; - if (this.connection) { - this.connection.write({ - modifyDeadlineAckIds: ackIds, - modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) + if (this.connectionPool) { + this.connectionPool.acquire(function(err, connection) { + if (err) { + self.emit('error', err); + return; + } +console.log({ + modifyDeadlineAckIds: ackIds, + modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) + }); + connection.write({ + modifyDeadlineAckIds: ackIds, + modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) + }); }); this.setLeaseTimeout_(); From 1e330d3d54f89e4d74f665d38bcd6db6bc0c8bf9 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 18:40:32 -0400 Subject: [PATCH 24/67] manually track publisher queue size --- packages/pubsub/src/publisher.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 0326a7b0481..be1e9448014 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -42,21 +42,10 @@ function Publisher(topic, options) { this.api = topic.api; this.inventory_ = { - queued: [] + queued: [], + queuedBytes: 0 }; - Object.defineProperty(this.inventory_, 'queueBytes', { - get: function() { - var size = 0; - - this.queued.forEach(function(message) { - size += message.data.size; - }); - - return size; - } - }); - this.settings = { batching: { maxBytes: Math.min(options.batching.maxBytes, Math.pow(1024, 2) * 9), @@ -100,6 +89,8 @@ Publisher.prototype.publish = function(data, attrs, callback) { attrs: attrs, callback: callback }); + + this.inventory_.queueBytes += data.size; }; /** @@ -112,6 +103,7 @@ Publisher.prototype.publish_ = function() { var callbacks = messages.map(prop('callback')); this.inventory_.queued = []; + this.inventory_.queuedBytes = 0; var reqOpts = { topic: this.topic.name, From 1bb715b3d8915aa3f71393fbbbbe76184f586aa5 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 18:44:41 -0400 Subject: [PATCH 25/67] track publisher callbacks in separate array --- packages/pubsub/src/publisher.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index be1e9448014..baaf30783cd 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -42,6 +42,7 @@ function Publisher(topic, options) { this.api = topic.api; this.inventory_ = { + callbacks: [], queued: [], queuedBytes: 0 }; @@ -86,11 +87,11 @@ Publisher.prototype.publish = function(data, attrs, callback) { this.inventory_.queued.push({ data: data, - attrs: attrs, - callback: callback + attributes: attrs }); this.inventory_.queueBytes += data.size; + this.inventory_.callbacks.push(callback); }; /** @@ -99,20 +100,16 @@ Publisher.prototype.publish = function(data, attrs, callback) { Publisher.prototype.publish_ = function() { var self = this; + var callbacks = this.inventory_.callbacks; var messages = this.inventory_.queued; - var callbacks = messages.map(prop('callback')); + this.inventory_.callbacks = []; this.inventory_.queued = []; this.inventory_.queuedBytes = 0; var reqOpts = { topic: this.topic.name, - messages: messages.map(function(message) { - return { - data: message.data, - attributes: message.attrs - }; - }) + messages: messages }; this.topic.request({ From 86aa39fff5b93fe382ffb6da54e8e9cdfea7aea0 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 18:46:23 -0400 Subject: [PATCH 26/67] renamed callback to onEndCallback --- packages/pubsub/src/connection-pool.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 75a621cef9e..6d1151016fa 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -149,8 +149,8 @@ ConnectionPool.prototype.drain = function(callback) { this.isOpen = false; callback = callback || common.util.noop; - each(connections, function(connection, callback) { - connection.end(callback); + each(connections, function(connection, onEndCallback) { + connection.end(onEndCallback); }, callback); }; From 85e24710b235d9a2de930ebe050c670750eed30b Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 18:49:03 -0400 Subject: [PATCH 27/67] store histogram values at second level percision --- packages/pubsub/src/histogram.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pubsub/src/histogram.js b/packages/pubsub/src/histogram.js index f878049d99f..d2b3d031267 100644 --- a/packages/pubsub/src/histogram.js +++ b/packages/pubsub/src/histogram.js @@ -37,6 +37,7 @@ function Histogram() { Histogram.prototype.add = function(value) { value = Math.max(value, MIN_VALUE); value = Math.min(value, MAX_VALUE); + value = Math.ceil(value / 1000) * 1000; if (!this.data.has(value)) { this.data.set(value, 0); From 1c17045e86834bb7bc0c5e789b94e75f129a1742 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 19:37:49 -0400 Subject: [PATCH 28/67] refactor publishing logic --- packages/pubsub/src/publisher.js | 59 ++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index baaf30783cd..03fff8fd896 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -72,26 +72,33 @@ Publisher.prototype.publish = function(data, attrs, callback) { } var opts = this.settings.batching; + var newPayloadSize = this.inventory_.queueBytes + data.size; + + // if this message puts us over the maxBytes option, then let's ship + // what we have and add it to the next batch + if (newPayloadSize > opts.maxBytes) { + this.publishImmediately_(); + this.queue_(data, attrs, callback); + return; + } + + // haven't hit maxBytes? add it to the queue! + this.queue_(data, attrs, callback); - var hasMaxMessages = this.inventory_.queued.length >= opts.maxMessages; - var hasMaxBytes = (this.inventory_.queueBytes + data.size) >= opts.maxBytes; + // next lets check if this message brings us to the message cap or if we + // magically hit the max byte limit + var hasMaxMessages = this.inventory_.queued.length === opts.maxMessages; + + if (newPayloadSize === opts.maxBytes || hasMaxMessages) { + this.publishImmediately_(); + return; + } - if (hasMaxMessages || hasMaxBytes) { - clearTimeout(this.timeoutHandle_); - this.timeoutHandle_ = null; - this.publish_(); - } else if (!this.timeoutHandle_) { + // otherwise let's set a timeout to send the next batch + if (!this.timeoutHandle_) { this.timeoutHandle_ = setTimeout( this.publish_.bind(this), opts.maxMilliseconds); } - - this.inventory_.queued.push({ - data: data, - attributes: attrs - }); - - this.inventory_.queueBytes += data.size; - this.inventory_.callbacks.push(callback); }; /** @@ -125,6 +132,28 @@ Publisher.prototype.publish_ = function() { }); }; +/** + * + */ +Publisher.prototype.publishImmediately_ = function() { + clearTimeout(this.timeoutHandle_); + this.timeoutHandle_ = null; + this.publish_(); +}; + +/** + * + */ +Publisher.prototype.queue_ = function(data, attrs, callback) { + this.inventory_.queued.push({ + data: data, + attributes: attrs + }); + + this.inventory_.queueBytes += data.size; + this.inventory_.callbacks.push(callback); +}; + /*! Developer Documentation * * All async methods (except for streams) will return a Promise in the event From 71df034767e7ec047c055f4983b2d93120461ce6 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 19:38:34 -0400 Subject: [PATCH 29/67] remove console logs --- packages/pubsub/src/subscription.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 276523f569b..565a4e7e637 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -615,7 +615,7 @@ Subscription.prototype.nack_ = function(message) { self.emit('error', err); return; } -console.log('nacking'); + connection.write({ modifyDeadlineAckIds: [message.ackId], modifyDeadlineSeconds: [0] @@ -675,10 +675,7 @@ Subscription.prototype.renewLeases_ = function() { self.emit('error', err); return; } -console.log({ - modifyDeadlineAckIds: ackIds, - modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) - }); + connection.write({ modifyDeadlineAckIds: ackIds, modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) From 7a2f9c030f025bbd9693fa73b770b393c5bbf913 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 20:01:27 -0400 Subject: [PATCH 30/67] refactored connection pool --- packages/pubsub/src/connection-pool.js | 58 +++++++++++++++----------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 6d1151016fa..80de2236aac 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -64,23 +64,29 @@ ConnectionPool.prototype.acquire = function(id, callback) { if (is.fn(id)) { callback = id; - id = getFirstConnectionId(); + id = null; } - id = id || getFirstConnectionId(); - if (!this.isOpen) { callback(new Error('Connection pool is closed.')); return; } - if (this.connections.has(id)) { - callback(null, this.connections.get(id)); + // it's possible that by the time a user acks the connection could have + // closed, so in that case we'll just return any connection + if (!this.connections.has(id)) { + id = getFirstConnectionId(); + } + + var connection = this.connections.get(id); + + if (connection) { + callback(null, connection); return; } - this.once('connected', function() { - self.acquire(id, callback); + this.once('connected', function(connection) { + callback(null, connection); }); function getFirstConnectionId() { @@ -91,7 +97,21 @@ ConnectionPool.prototype.acquire = function(id, callback) { /** * */ -ConnectionPool.prototype.createConnection = function(id) { +ConnectionPool.prototype.close = function(callback) { + var connections = Array.from(this.connections.values()); + + this.isOpen = false; + callback = callback || common.util.noop; + + each(connections, function(connection, onEndCallback) { + connection.end(onEndCallback); + }, callback); +}; + +/** + * + */ +ConnectionPool.prototype.createConnection = function() { var self = this; this.subscription.request({ @@ -104,6 +124,7 @@ ConnectionPool.prototype.createConnection = function(id) { return; } + var id = uuid.v4(); var connection = requestFn(); connection.on('error', function(err) { @@ -117,8 +138,7 @@ ConnectionPool.prototype.createConnection = function(id) { }); connection.once('metadata', function() { - self.connections.set(id, connection); - self.emit('connected'); + self.emit('connected', connection); }); connection.once('close', function() { @@ -137,21 +157,9 @@ ConnectionPool.prototype.createConnection = function(id) { subscription: self.subscription.name, streamAckDeadlineSeconds: self.settings.ackDeadline / 1000 }); - }); -}; - -/** - * - */ -ConnectionPool.prototype.drain = function(callback) { - var connections = Array.from(this.connections.values()); - - this.isOpen = false; - callback = callback || common.util.noop; - each(connections, function(connection, onEndCallback) { - connection.end(onEndCallback); - }, callback); + self.connections.set(id, connection); + }); }; /** @@ -159,7 +167,7 @@ ConnectionPool.prototype.drain = function(callback) { */ ConnectionPool.prototype.open = function() { for (var i = 0; i < this.settings.maxConnections; i++) { - this.createConnection(i); + this.createConnection(); } this.isOpen = true; From e1d1b8c68566dd60672fe22d5283084951695ecf Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 25 Jul 2017 20:17:55 -0400 Subject: [PATCH 31/67] small refactors based on pr feedback --- packages/pubsub/src/publisher.js | 21 ++++++++++----------- packages/pubsub/src/subscription.js | 1 + 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 03fff8fd896..8de1e2e8aba 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -41,6 +41,11 @@ function Publisher(topic, options) { this.topic = topic; this.api = topic.api; + // this object keeps track of all messages scheduled to be published + // queued is essentially the `messages` field for the publish rpc req opts + // queuedBytes is used to track the size of the combined payload + // callbacks is an array of callbacks - each callback is associated with a + // specific message. this.inventory_ = { callbacks: [], queued: [], @@ -77,7 +82,7 @@ Publisher.prototype.publish = function(data, attrs, callback) { // if this message puts us over the maxBytes option, then let's ship // what we have and add it to the next batch if (newPayloadSize > opts.maxBytes) { - this.publishImmediately_(); + this.publish_(); this.queue_(data, attrs, callback); return; } @@ -90,7 +95,7 @@ Publisher.prototype.publish = function(data, attrs, callback) { var hasMaxMessages = this.inventory_.queued.length === opts.maxMessages; if (newPayloadSize === opts.maxBytes || hasMaxMessages) { - this.publishImmediately_(); + this.publish_(); return; } @@ -107,6 +112,9 @@ Publisher.prototype.publish = function(data, attrs, callback) { Publisher.prototype.publish_ = function() { var self = this; + clearTimeout(this.timeoutHandle_); + this.timeoutHandle_ = null; + var callbacks = this.inventory_.callbacks; var messages = this.inventory_.queued; @@ -132,15 +140,6 @@ Publisher.prototype.publish_ = function() { }); }; -/** - * - */ -Publisher.prototype.publishImmediately_ = function() { - clearTimeout(this.timeoutHandle_); - this.timeoutHandle_ = null; - this.publish_(); -}; - /** * */ diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 565a4e7e637..51305f94df7 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -562,6 +562,7 @@ Subscription.prototype.listenForEvents_ = function() { self.messageListeners++; if (!self.connection) { + self.userClosed_ = false; self.openConnection_(); } } From d3895738ab0b8683ee1055ca7245b9be17574bfc Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 27 Jul 2017 13:20:27 -0400 Subject: [PATCH 32/67] system test updates --- packages/pubsub/src/subscription.js | 11 ++- .../src/v1/subscriber_client_config.json | 2 +- packages/pubsub/system-test/pubsub.js | 82 +++++++++++-------- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 51305f94df7..e4dd130606c 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -303,8 +303,10 @@ Subscription.prototype.close = function(callback) { * */ Subscription.prototype.closeConnection_ = function(callback) { + callback = callback || common.util.noop; + if (this.connectionPool) { - this.connectionPool.drain(callback || common.util.noop); + this.connectionPool.close(callback); this.connectionPool = null; } else if (is.fn(callback)) { setImmediate(callback); @@ -744,10 +746,7 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { if (is.string(snapshot)) { reqOpts.snapshot = Snapshot.formatName_(this.pubsub.projectId, snapshot); } else if (is.date(snapshot)) { - reqOpts.time = { - seconds: Math.floor(snapshot.getTime() / 1000), - nanos: snapshot.getMilliseconds() * 1e6 - }; + reqOpts.time = snapshot; } else { throw new Error('Either a snapshot name or Date is needed to seek to.'); } @@ -757,7 +756,7 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { method: 'seek', reqOpts: reqOpts, gaxOpts: gaxOpts - }, callback) + }, callback); }; /** diff --git a/packages/pubsub/src/v1/subscriber_client_config.json b/packages/pubsub/src/v1/subscriber_client_config.json index 2c8efffba81..a8e53152d79 100644 --- a/packages/pubsub/src/v1/subscriber_client_config.json +++ b/packages/pubsub/src/v1/subscriber_client_config.json @@ -77,7 +77,7 @@ "retry_params_name": "messaging" }, "StreamingPull": { - "timeout_millis": 60000, + "timeout_millis": 900000, "retry_codes_name": "pull", "retry_params_name": "messaging" }, diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 93da54e98d9..5cba5fa9ab9 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -196,6 +196,7 @@ describe('pubsub', function() { var TOPIC_NAME = generateTopicName(); var topic = pubsub.topic(TOPIC_NAME); var publisher = topic.publisher(); + var subscription; var SUB_NAMES = [ generateSubName(), @@ -203,8 +204,8 @@ describe('pubsub', function() { ]; var SUBSCRIPTIONS = [ - topic.subscription(SUB_NAMES[0], { ackDeadlineSeconds: 30 }), - topic.subscription(SUB_NAMES[1], { ackDeadlineSeconds: 60 }) + topic.subscription(SUB_NAMES[0], { ackDeadline: 30000 }), + topic.subscription(SUB_NAMES[1], { ackDeadline: 60000 }) ]; before(function(done) { @@ -333,9 +334,11 @@ describe('pubsub', function() { }); it('should error when using a non-existent subscription', function(done) { - var subscription = topic.subscription(generateSubName()); + var subscription = topic.subscription(generateSubName(), { + maxConnections: 1 + }); - subscription.on('error', function(err) { + subscription.once('error', function(err) { assert.strictEqual(err.code, 5); subscription.close(done); }); @@ -346,8 +349,8 @@ describe('pubsub', function() { }); it('should receive the published messages', function(done) { - var subscription = topic.subscription(SUB_NAMES[1]); var messageCount = 0; + var subscription = topic.subscription(SUB_NAMES[1]); subscription.on('error', done); @@ -516,47 +519,56 @@ describe('pubsub', function() { return subscription.create().then(function() { return publisher.publish(new Buffer('Hello, world!')); - }).then(function(data) { - messageId = data[0][0]; + }).then(function(messageIds) { + messageId = messageIds[0]; }); }); - function checkMessage() { - return new Promise(function(resolve, reject) { - function onError(err) { - subscription.removeListener('message', onMessage); - reject(err); - } + it('should seek to a snapshot', function(done) { + var snapshotName = generateSnapshotName(); - function onMessage(message) { - subscription.removeListener('error', onError); - resolve(message); - } + subscription.createSnapshot(snapshotName, function(err, snapshot) { + assert.ifError(err); - subscription.once('error', onError); - subscription.once('message', onMessage); - }); - } + var messageCount = 0; - it('should seek to a snapshot', function() { - var snapshotName = generateSnapshotName(); + subscription.on('error', done); + subscription.on('message', function(message) { + assert.strictEqual(message.id, messageId); - return subscription.createSnapshot(snapshotName).then(function() { - return checkMessage(); - }).then(function() { - return subscription.seek(snapshotName); - }).then(function() { - return checkMessage(); + message.ack(); + + if (++messageCount === 1) { + snapshot.seek(function(err) { + assert.ifError(err); + }); + return; + } + + assert.strictEqual(messageCount, 2); + subscription.close(done); + }); }); }); - it('should seek to a date', function() { - var date = new Date(); + it('should seek to a date', function(done) { + var messageCount = 0; + + subscription.on('error', done); + subscription.on('message', function(message) { + assert.strictEqual(message.id, messageId); - return checkMessage().then(function() { - return subscription.seek(date); - }).then(function() { - return checkMessage(); + message.ack(); + + if (++messageCount === 1) { + subscription.seek(message.publishTime, function(err) { + assert.ifError(err); + }); + return; + } + + assert.strictEqual(messageCount, 2); + subscription.close(done); }); }); }); From 0e3a6efb778965b6adc37c2ebef09866555d75ea Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 27 Jul 2017 20:10:56 -0400 Subject: [PATCH 33/67] update unit tests for index.js --- packages/pubsub/src/index.js | 60 +- packages/pubsub/test/index.js | 1190 +++++++++++++++------------------ 2 files changed, 577 insertions(+), 673 deletions(-) diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index 5e77848b29e..b33295553fe 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -73,15 +73,7 @@ function PubSub(options) { scopes: v1.ALL_SCOPES }, options); - this.defaultBaseUrl_ = 'pubsub.googleapis.com'; - - if (options.servicePath) { - this.defaultBaseUrl_ = options.servicePath; - - if (options.port) { - this.defaultBaseUrl_ += ':' + options.port; - } - } + this.determineBaseUrl_(); this.api = {}; this.auth = googleAuth(this.options); @@ -106,8 +98,6 @@ function PubSub(options) { * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) * @param {number} options.ackDeadlineSeconds - The maximum time after receiving * a message that you must ack a message before it is redelivered. - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') * @param {number|date} options.messageRetentionDuration - Set this to override * the default duration of 7 days. This value is expected in seconds. * Acceptable values are in the range of 10 minutes and 7 days. @@ -173,6 +163,8 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { options = {}; } + options = options || {}; + var subscription = this.subscription(name, options); var reqOpts = extend({ @@ -182,18 +174,20 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { delete reqOpts.gaxOpts; - if (reqOpts.messageRetentionDuration) { + if (options.messageRetentionDuration) { reqOpts.retainAckedMessages = true; reqOpts.messageRetentionDuration = { - seconds: reqOpts.messageRetentionDuration, + seconds: options.messageRetentionDuration, nanos: 0 }; } - if (reqOpts.pushEndpoint) { + if (options.pushEndpoint) { + delete reqOpts.pushEndpoint; + reqOpts.pushConfig = { - pushEndpoint: reqOpts.pushEndpoint + pushEndpoint: options.pushEndpoint }; } @@ -268,6 +262,34 @@ PubSub.prototype.createTopic = function(name, gaxOpts, callback) { }); }; +/** + * Determine the appropriate endpoint to use for API requests, first trying the + * local `apiEndpoint` parameter. If the `apiEndpoint` parameter is null we try + * Pub/Sub emulator environment variable (PUBSUB_EMULATOR_HOST), otherwise the + * default JSON API. + * + * @private + */ +PubSub.prototype.determineBaseUrl_ = function() { + var apiEndpoint = this.options.apiEndpoint; + + if (!apiEndpoint && !process.env.PUBSUB_EMULATOR_HOST) { + return; + } + + var baseUrl = apiEndpoint || process.env.PUBSUB_EMULATOR_HOST; + var leadingProtocol = new RegExp('^https*://'); + var trailingSlashes = new RegExp('/*$'); + + var baseUrlParts = baseUrl + .replace(leadingProtocol, '') + .replace(trailingSlashes, '') + .split(':'); + + this.options.servicePath = baseUrlParts[0]; + this.options.port = baseUrlParts[1]; +}; + /** * Get a list of snapshots. * @@ -651,6 +673,10 @@ PubSub.prototype.request = function(config, callback) { } function prepareGaxRequest(callback) { + if (global.GCLOUD_SANDBOX_ENV) { + return; + } + self.auth.getProjectId(function(err, projectId) { if (err) { callback(err); @@ -679,10 +705,6 @@ PubSub.prototype.request = function(config, callback) { } function makeRequestCallback() { - if (global.GCLOUD_SANDBOX_ENV) { - return; - } - prepareGaxRequest(function(err, requestFn) { if (err) { callback(err); diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index eaa656babda..276bc83c306 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -23,14 +23,16 @@ var path = require('path'); var proxyquire = require('proxyquire'); var util = require('@google-cloud/common').util; +var v1 = require('../src/v1/index.js'); + var SubscriptionCached = require('../src/subscription.js'); var SubscriptionOverride; var Topic = require('../src/topic.js'); -function Subscription(a, b) { +function Subscription(a, b, c) { var OverrideFn = SubscriptionOverride || SubscriptionCached; - return new OverrideFn(a, b); + return new OverrideFn(a, b, c); } var promisified = false; @@ -50,15 +52,6 @@ var fakeUtil = extend({}, util, { } }); -function FakeGrpcService() { - this.calledWith_ = arguments; -} - -var grpcServiceRequestOverride; -FakeGrpcService.prototype.request = function() { - return (grpcServiceRequestOverride || util.noop).apply(this, arguments); -}; - function FakeSnapshot() { this.calledWith_ = arguments; } @@ -85,6 +78,16 @@ var fakePaginator = { } }; +var googleAutoAuthOverride; +function fakeGoogleAutoAuth() { + return (googleAutoAuthOverride || util.noop).apply(null, arguments); +} + +var v1Override; +function fakeV1() { + return (v1Override || util.noop).apply(null, arguments); +} + var GAX_CONFIG_PUBLISHER_OVERRIDE = {}; var GAX_CONFIG_SUBSCRIBER_OVERRIDE = {}; @@ -101,7 +104,7 @@ var GAX_CONFIG = { } }; -describe('PubSub', function() { +describe.only('PubSub', function() { var PubSub; var PROJECT_ID = 'test-project'; var pubsub; @@ -116,13 +119,11 @@ describe('PubSub', function() { paginator: fakePaginator, util: fakeUtil }, - '@google-cloud/common-grpc': { - Service: FakeGrpcService - }, + 'google-auto-auth': fakeGoogleAutoAuth, './snapshot.js': FakeSnapshot, './subscription.js': Subscription, './topic.js': Topic, - + './v1': fakeV1, './v1/publisher_client_config.json': GAX_CONFIG.Publisher, './v1/subscriber_client_config.json': GAX_CONFIG.Subscriber }); @@ -135,7 +136,8 @@ describe('PubSub', function() { }); beforeEach(function() { - grpcServiceRequestOverride = null; + v1Override = null; + googleAutoAuthOverride = null; SubscriptionOverride = null; pubsub = new PubSub(OPTIONS); pubsub.projectId = PROJECT_ID; @@ -175,52 +177,288 @@ describe('PubSub', function() { fakeUtil.normalizeArguments = normalizeArguments; }); - it('should inherit from GrpcService', function() { - assert(pubsub instanceof FakeGrpcService); + it('should attempt to determine the service path and port', function() { + var determineBaseUrl_ = PubSub.prototype.determineBaseUrl_; + var called = false; + + PubSub.prototype.determineBaseUrl_ = function() { + PubSub.prototype.determineBaseUrl_ = determineBaseUrl_; + called = true; + }; + + var pubsub = new PubSub({}); + assert(called); + }); + + it('should initialize the API object', function() { + assert.deepEqual(pubsub.api, {}); + }); + + it('should cache a local google-auto-auth instance', function() { + var fakeGoogleAutoAuthInstance = {}; + var options = { + a: 'b', + c: 'd' + }; + + googleAutoAuthOverride = function(options_) { + assert.deepEqual(options_, extend({ + scopes: v1.ALL_SCOPES + }, options)); + return fakeGoogleAutoAuthInstance; + }; + + var pubsub = new PubSub(options); + assert.strictEqual(pubsub.auth, fakeGoogleAutoAuthInstance); + }); + + it('should localize the options provided', function() { + assert.deepEqual(pubsub.options, extend({ + scopes: v1.ALL_SCOPES + }, OPTIONS)); + }); + + it('should set the projectId', function() { + assert.strictEqual(pubsub.projectId, PROJECT_ID); + }); + + it('should default the projectId to the token', function() { + var pubsub = new PubSub({}); + assert.strictEqual(pubsub.projectId, '{{projectId}}'); + }); + }); + + + describe('createSubscription', function() { + var TOPIC_NAME = 'topic'; + var TOPIC = { + name: 'projects/' + PROJECT_ID + '/topics/' + TOPIC_NAME + }; + + var SUB_NAME = 'subscription'; + var SUBSCRIPTION = { + name: 'projects/' + PROJECT_ID + '/subscriptions/' + SUB_NAME + }; + + var apiResponse = { + name: 'subscription-name' + }; + + it('should throw if no Topic is provided', function() { + assert.throws(function() { + pubsub.createSubscription(); + }, /A Topic is required for a new subscription\./); + }); + + it('should throw if no subscription name is provided', function() { + assert.throws(function() { + pubsub.createSubscription(TOPIC_NAME); + }, /A subscription name is required./); + }); + + it('should not require configuration options', function(done) { + pubsub.request = function(config, callback) { + callback(null, apiResponse); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, done); + }); + + it('should allow undefined/optional configuration options', function(done) { + pubsub.request = function(config, callback) { + callback(null, apiResponse); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, undefined, done); + }); + + it('should create a Subscription', function(done) { + var opts = { a: 'b', c: 'd' }; + + pubsub.request = util.noop; + + pubsub.subscription = function(subName, options) { + assert.strictEqual(subName, SUB_NAME); + assert.deepEqual(options, opts); + setImmediate(done); + return SUBSCRIPTION; + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, opts, assert.ifError); + }); + + it('should create a Topic object from a string', function(done) { + pubsub.request = util.noop; + + pubsub.topic = function(topicName) { + assert.strictEqual(topicName, TOPIC_NAME); + setImmediate(done); + return TOPIC; + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, assert.ifError); + }); + + it('should send correct request', function(done) { + var options = { + gaxOpts: {} + }; + + pubsub.topic = function(topicName) { + return { + name: topicName + }; + }; + + pubsub.subscription = function(subName) { + return { + name: subName + }; + }; - var calledWith = pubsub.calledWith_[0]; + pubsub.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'createSubscription'); + assert.strictEqual(config.reqOpts.topic, TOPIC_NAME); + assert.strictEqual(config.reqOpts.name, SUB_NAME); + assert.strictEqual(config.gaxOpts, options.gaxOpts); + done(); + }; - var baseUrl = 'pubsub.googleapis.com'; - assert.strictEqual(calledWith.baseUrl, baseUrl); + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, options, assert.ifError); + }); - var protosDir = path.resolve(__dirname, '../protos'); - assert.strictEqual(calledWith.protosDir, protosDir); + it('should pass options to the api request', function(done) { + var options = { + ackDeadlineSeconds: 90, + retainAckedMessages: true, + pushEndpoint: 'https://domain/push', + }; - assert.deepStrictEqual(calledWith.protoServices, { - Publisher: { - path: 'google/pubsub/v1/pubsub.proto', - service: 'pubsub.v1' - }, - Subscriber: { - path: 'google/pubsub/v1/pubsub.proto', - service: 'pubsub.v1' + var expectedBody = extend({ + topic: TOPIC_NAME, + name: SUB_NAME + }, options, { + pushConfig: { + pushEndpoint: options.pushEndpoint } }); - assert.deepEqual(calledWith.scopes, [ - 'https://www.googleapis.com/auth/pubsub', - 'https://www.googleapis.com/auth/cloud-platform' - ]); - assert.deepEqual(calledWith.packageJson, require('../package.json')); + delete expectedBody.pushEndpoint; + + pubsub.topic = function() { + return { + name: TOPIC_NAME + }; + }; + + pubsub.subscription = function() { + return { + name: SUB_NAME + }; + }; + + pubsub.request = function(config, callback) { + assert.notStrictEqual(config.reqOpts, options); + assert.deepEqual(config.reqOpts, expectedBody); + done(); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, options, assert.ifError); }); - it('should set the defaultBaseUrl_', function() { - assert.strictEqual(pubsub.defaultBaseUrl_, 'pubsub.googleapis.com'); + describe('message retention', function() { + it('should accept a number', function(done) { + var threeDaysInSeconds = 3 * 24 * 60 * 60; + + pubsub.request = function(config) { + assert.strictEqual(config.reqOpts.retainAckedMessages, true); + + assert.strictEqual( + config.reqOpts.messageRetentionDuration.seconds, + threeDaysInSeconds + ); + + assert.strictEqual(config.reqOpts.messageRetentionDuration.nanos, 0); + + done(); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, { + messageRetentionDuration: threeDaysInSeconds + }, assert.ifError); + }); }); - it('should use the PUBSUB_EMULATOR_HOST env var', function() { - var pubSubHost = 'pubsub-host'; - process.env.PUBSUB_EMULATOR_HOST = pubSubHost; + describe('error', function() { + var error = new Error('Error.'); + var apiResponse = { name: SUB_NAME }; + + beforeEach(function() { + pubsub.request = function(config, callback) { + callback(error, apiResponse); + }; + }); + + it('should re-use existing subscription', function(done) { + var apiResponse = { code: 6 }; - var pubsub = new PubSub({ projectId: 'project-id' }); - delete process.env.PUBSUB_EMULATOR_HOST; + pubsub.subscription = function() { + return SUBSCRIPTION; + }; + + pubsub.request = function(config, callback) { + callback({ code: 6 }, apiResponse); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, subscription) { + assert.ifError(err); + assert.strictEqual(subscription, SUBSCRIPTION); + done(); + }); + }); + + it('should return error & API response to the callback', function(done) { + pubsub.request = function(config, callback) { + callback(error, apiResponse); + }; - var calledWith = pubsub.calledWith_[0]; - assert.strictEqual(calledWith.baseUrl, pubSubHost); + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { + assert.strictEqual(err, error); + assert.strictEqual(sub, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); }); - it('should localize the options provided', function() { - assert.strictEqual(pubsub.options, OPTIONS); + describe('success', function() { + var apiResponse = { name: SUB_NAME }; + + beforeEach(function() { + pubsub.request = function(config, callback) { + callback(null, apiResponse); + }; + }); + + it('should return Subscription & resp to the callback', function(done) { + var subscription = {}; + + pubsub.subscription = function() { + return subscription; + }; + + pubsub.request = function(config, callback) { + callback(null, apiResponse); + }; + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { + assert.ifError(err); + assert.strictEqual(sub, subscription); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); }); }); @@ -228,23 +466,25 @@ describe('PubSub', function() { it('should make the correct API request', function(done) { var topicName = 'new-topic-name'; var formattedName = 'formatted-name'; + var gaxOpts = {}; - var formatName_ = Topic.formatName_; - Topic.formatName_ = function(projectId, name) { - Topic.formatName_ = formatName_; - assert.strictEqual(projectId, pubsub.projectId); + pubsub.topic = function(name) { assert.strictEqual(name, topicName); - return formattedName; + + return { + name: formattedName + }; }; - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Publisher'); - assert.strictEqual(protoOpts.method, 'createTopic'); - assert.strictEqual(reqOpts.name, formattedName); + pubsub.request = function(config) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'createTopic'); + assert.deepEqual(config.reqOpts, { name: formattedName }); + assert.deepEqual(config.gaxOpts, gaxOpts); done(); }; - pubsub.createTopic(topicName, function() {}); + pubsub.createTopic(topicName, gaxOpts, function() {}); }); describe('error', function() { @@ -252,7 +492,7 @@ describe('PubSub', function() { var apiResponse = {}; beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { + pubsub.request = function(config, callback) { callback(error, apiResponse); }; }); @@ -271,7 +511,7 @@ describe('PubSub', function() { var apiResponse = {}; beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { + pubsub.request = function(config, callback) { callback(null, apiResponse); }; }); @@ -302,106 +542,154 @@ describe('PubSub', function() { }); }); - describe('getSnapshots', function() { - var SNAPSHOT_NAME = 'fake-snapshot'; - var apiResponse = { snapshots: [{ name: SNAPSHOT_NAME }]}; + describe('determineBaseUrl_', function() { + function setHost(host) { + process.env.PUBSUB_EMULATOR_HOST = host; + } beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); - }; + delete process.env.PUBSUB_EMULATOR_HOST; }); - it('should accept a query and a callback', function(done) { - pubsub.getSnapshots({}, done); - }); + it('should do nothing if correct options are not set', function() { + pubsub.determineBaseUrl_(); - it('should accept just a callback', function(done) { - pubsub.getSnapshots(done); + assert.strictEqual(pubsub.options.servicePath, undefined); + assert.strictEqual(pubsub.options.port, undefined); }); - it('should build the right request', function(done) { - var options = { a: 'b', c: 'd' }; - var originalOptions = extend({}, options); - var expectedOptions = extend({}, options, { - project: 'projects/' + pubsub.projectId - }); + it('should use the apiEndpoint option', function() { + var defaultBaseUrl_ = 'defaulturl'; + var testingUrl = 'localhost:8085'; - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'listSnapshots'); - assert.deepEqual(reqOpts, expectedOptions); - assert.deepEqual(options, originalOptions); - done(); - }; + setHost(defaultBaseUrl_); + pubsub.options.apiEndpoint = testingUrl; + pubsub.determineBaseUrl_(); - pubsub.getSnapshots(options, assert.ifError); + assert.strictEqual(pubsub.options.servicePath, 'localhost'); + assert.strictEqual(pubsub.options.port, '8085'); }); - it('should return Snapshot instances with metadata', function(done) { - var snapshot = {}; + it('should remove slashes from the baseUrl', function() { + var expectedBaseUrl = 'localhost:8080'; - pubsub.snapshot = function(name) { - assert.strictEqual(name, SNAPSHOT_NAME); - return snapshot; - }; + setHost('localhost:8080/'); + pubsub.determineBaseUrl_(); + assert.strictEqual(pubsub.options.servicePath, 'localhost'); + assert.strictEqual(pubsub.options.port, '8080'); - pubsub.getSnapshots(function(err, snapshots) { - assert.ifError(err); - assert.strictEqual(snapshots[0], snapshot); - assert.strictEqual(snapshots[0].metadata, apiResponse.snapshots[0]); - done(); + setHost('localhost:8081//'); + pubsub.determineBaseUrl_(); + assert.strictEqual(pubsub.options.servicePath, 'localhost'); + assert.strictEqual(pubsub.options.port, '8081'); + }); + + it('should set the port to undefined if not set', function() { + setHost('localhost'); + pubsub.determineBaseUrl_(); + assert.strictEqual(pubsub.options.servicePath, 'localhost'); + assert.strictEqual(pubsub.options.port, undefined); + }); + + describe('with PUBSUB_EMULATOR_HOST environment variable', function() { + var PUBSUB_EMULATOR_HOST = 'localhost:9090'; + + beforeEach(function() { + setHost(PUBSUB_EMULATOR_HOST); + }); + + after(function() { + delete process.env.PUBSUB_EMULATOR_HOST; + }); + + it('should use the PUBSUB_EMULATOR_HOST env var', function() { + pubsub.determineBaseUrl_(); + assert.strictEqual(pubsub.options.servicePath, 'localhost'); + assert.strictEqual(pubsub.options.port, '9090'); }); }); + }); - it('should return a query if more results exist', function() { - var token = 'next-page-token'; + describe('getSnapshots', function() { + var SNAPSHOT_NAME = 'fake-snapshot'; + var apiResponse = { snapshots: [{ name: SNAPSHOT_NAME }]}; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, { nextPageToken: token }); + beforeEach(function() { + pubsub.request = function(config, callback) { + callback(null, apiResponse.snapshots, {}, apiResponse); }; + }); - var query = { pageSize: 1 }; + it('should accept a query and a callback', function(done) { + pubsub.getSnapshots({}, done); + }); - pubsub.getSnapshots(query, function(err, snapshots, nextQuery) { - assert.ifError(err); - assert.strictEqual(query.pageSize, nextQuery.pageSize); - assert.equal(query.pageToken, token); + it('should accept just a callback', function(done) { + pubsub.getSnapshots(done); + }); + + it('should build the right request', function(done) { + var options = { a: 'b', c: 'd', gaxOpts: {} }; + var originalOptions = extend({}, options); + var expectedOptions = extend({}, options, { + project: 'projects/' + pubsub.projectId }); + + delete expectedOptions.gaxOpts; + + pubsub.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'listSnapshots'); + assert.deepEqual(config.reqOpts, expectedOptions); + assert.deepEqual(config.gaxOpts, options.gaxOpts); + done(); + }; + + pubsub.getSnapshots(options, assert.ifError); }); - it('should pass error if api returns an error', function(done) { - var error = new Error('Error'); + it('should return Snapshot instances with metadata', function(done) { + var snapshot = {}; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(error); + pubsub.snapshot = function(name) { + assert.strictEqual(name, SNAPSHOT_NAME); + return snapshot; }; - pubsub.getSnapshots(function(err) { - assert.equal(err, error); + pubsub.getSnapshots(function(err, snapshots) { + assert.ifError(err); + assert.strictEqual(snapshots[0], snapshot); + assert.strictEqual(snapshots[0].metadata, apiResponse.snapshots[0]); done(); }); }); - it('should pass apiResponse to callback', function(done) { - var resp = { success: true }; + it('should pass back all parameters', function(done) { + var err_ = new Error('abc'); + var snapshots_ = []; + var nextQuery_ = {}; + var apiResponse_ = {}; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); + pubsub.request = function(config, callback) { + callback(err_, snapshots_, nextQuery_, apiResponse_); }; pubsub.getSnapshots(function(err, snapshots, nextQuery, apiResponse) { - assert.ifError(err); - assert.equal(resp, apiResponse); + assert.strictEqual(err, err_); + assert.deepEqual(snapshots, snapshots_); + assert.strictEqual(nextQuery, nextQuery_); + assert.strictEqual(apiResponse, apiResponse_); done(); }); }); }); describe('getSubscriptions', function() { + var apiResponse = { subscriptions: [{ name: 'fake-subscription' }] }; + beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, { subscriptions: [{ name: 'fake-subscription' }] }); + pubsub.request = function(config, callback) { + callback(null, apiResponse.subscriptions, {}, apiResponse); }; }); @@ -414,49 +702,25 @@ describe('PubSub', function() { }); it('should pass the correct arguments to the API', function(done) { - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'listSubscriptions'); - assert.strictEqual(reqOpts.project, 'projects/' + pubsub.projectId); + var options = { gaxOpts: {} }; + var project = 'projects/' + pubsub.projectId; + + pubsub.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'listSubscriptions'); + assert.deepEqual(config.reqOpts, { project: project }); + assert.strictEqual(config.gaxOpts, options.gaxOpts); done(); }; - pubsub.getSubscriptions(assert.ifError); - }); - - describe('topics', function() { - var TOPIC; - var TOPIC_NAME = 'topic'; - - beforeEach(function() { - TOPIC = new Topic(pubsub, TOPIC_NAME); - }); - - it('should subscribe to a topic by string', function(done) { - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Publisher'); - assert.strictEqual(protoOpts.method, 'listTopicSubscriptions'); - assert.strictEqual(reqOpts.topic, TOPIC_NAME); - done(); - }; - - pubsub.getSubscriptions({ topic: TOPIC_NAME }, assert.ifError); - }); - - it('should subscribe to a topic by Topic instance', function(done) { - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(reqOpts.topic, TOPIC.name); - done(); - }; - - pubsub.getSubscriptions({ topic: TOPIC }, assert.ifError); - }); + pubsub.getSubscriptions(options, assert.ifError); }); it('should pass options to API request', function(done) { var opts = { pageSize: 10, pageToken: 'abc' }; - pubsub.request = function(protoOpts, reqOpts) { + pubsub.request = function(config) { + var reqOpts = config.reqOpts; assert.strictEqual(reqOpts.pageSize, opts.pageSize); assert.strictEqual(reqOpts.pageToken, opts.pageToken); done(); @@ -465,73 +729,29 @@ describe('PubSub', function() { pubsub.getSubscriptions(opts, assert.ifError); }); - it('should pass error & response if api returns an error', function(done) { - var error = new Error('Error'); - var resp = { error: true }; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(error, resp); - }; - - pubsub.getSubscriptions(function(err, subs, nextQuery, apiResponse) { - assert.equal(err, error); - assert.deepEqual(apiResponse, resp); - done(); - }); - }); - - describe('returning Subscription instances', function() { - it('should handle subscriptions.list response', function(done) { - pubsub.getSubscriptions(function(err, subscriptions) { - assert.ifError(err); - assert(subscriptions[0] instanceof SubscriptionCached); - done(); - }); - }); - - it('should handle topics.subscriptions.list response', function(done) { - var subName = 'sub-name'; - var subFullName = - 'projects/' + PROJECT_ID + '/subscriptions/' + subName; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, { subscriptions: [subName] }); - }; - - pubsub.getSubscriptions(function(err, subscriptions) { - assert.ifError(err); - assert(subscriptions[0] instanceof SubscriptionCached); - assert.equal(subscriptions[0].name, subFullName); - done(); - }); - }); - }); - - it('should return a query if more results exist', function() { - var token = 'next-page-token'; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, { nextPageToken: token }); - }; - - var query = { maxResults: 1 }; - - pubsub.getSubscriptions(query, function(err, subscriptions, nextQuery) { + it('should return Subscription instances', function(done) { + pubsub.getSubscriptions(function(err, subscriptions) { assert.ifError(err); - assert.strictEqual(query.maxResults, nextQuery.maxResults); - assert.equal(query.pageToken, token); + assert(subscriptions[0] instanceof SubscriptionCached); + done(); }); }); - it('should pass apiResponse to callback', function(done) { - var resp = { success: true }; + it('should pass back all params', function(done) { + var err_ = new Error('err'); + var subs_ = []; + var nextQuery_ = {}; + var apiResponse_ = {}; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); + pubsub.request = function(config, callback) { + callback(err_, subs_, nextQuery_, apiResponse_); }; pubsub.getSubscriptions(function(err, subs, nextQuery, apiResponse) { - assert.equal(resp, apiResponse); + assert.strictEqual(err, err_); + assert.deepEqual(subs, subs_); + assert.strictEqual(nextQuery, nextQuery_); + assert.strictEqual(apiResponse, apiResponse_); done(); }); }); @@ -542,8 +762,8 @@ describe('PubSub', function() { var apiResponse = { topics: [{ name: topicName }]}; beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); + pubsub.request = function(config, callback) { + callback(null, apiResponse.topics, {}, apiResponse); }; }); @@ -556,20 +776,23 @@ describe('PubSub', function() { }); it('should build the right request', function(done) { - var options = { a: 'b', c: 'd' }; + var options = { a: 'b', c: 'd', gaxOpts: {} }; var originalOptions = extend({}, options); var expectedOptions = extend({}, options, { project: 'projects/' + pubsub.projectId }); - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Publisher'); - assert.strictEqual(protoOpts.method, 'listTopics'); - assert.deepEqual(reqOpts, expectedOptions); - assert.deepEqual(options, originalOptions); + delete expectedOptions.gaxOpts; + + pubsub.request = function(config) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'listTopics'); + assert.deepEqual(config.reqOpts, expectedOptions); + assert.strictEqual(config.gaxOpts, options.gaxOpts); done(); }; - pubsub.getTopics(options, function() {}); + + pubsub.getTopics(options, assert.ifError); }); it('should return Topic instances with metadata', function(done) { @@ -588,289 +811,135 @@ describe('PubSub', function() { }); }); - it('should return a query if more results exist', function() { - var token = 'next-page-token'; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, { nextPageToken: token }); - }; - var query = { pageSize: 1 }; - pubsub.getTopics(query, function(err, topics, nextQuery) { - assert.ifError(err); - assert.strictEqual(query.pageSize, nextQuery.pageSize); - assert.equal(query.pageToken, token); - }); - }); + it('should pass back all params', function(done) { + var err_ = new Error('err'); + var topics_ = []; + var nextQuery_ = {}; + var apiResponse_ = {}; - it('should pass error if api returns an error', function() { - var error = new Error('Error'); - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(error); + pubsub.request = function(config, callback) { + callback(err_, topics_, nextQuery_, apiResponse_); }; - pubsub.getTopics(function(err) { - assert.equal(err, error); - }); - }); - it('should pass apiResponse to callback', function(done) { - var resp = { success: true }; - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); - }; pubsub.getTopics(function(err, topics, nextQuery, apiResponse) { - assert.equal(resp, apiResponse); + assert.strictEqual(err, err_); + assert.deepEqual(topics, topics_); + assert.strictEqual(nextQuery, nextQuery_); + assert.strictEqual(apiResponse, apiResponse_); done(); }); }); }); - describe('subscribe', function() { - var TOPIC_NAME = 'topic'; - var TOPIC = { - name: 'projects/' + PROJECT_ID + '/topics/' + TOPIC_NAME - }; - - var SUB_NAME = 'subscription'; - var SUBSCRIPTION = { - name: 'projects/' + PROJECT_ID + '/subscriptions/' + SUB_NAME - }; - - var apiResponse = { - name: 'subscription-name' + describe('request', function() { + var CONFIG = { + client: 'fakeClient', + method: 'fakeMethod', + reqOpts: { a: 'a' }, + gaxOpts: {} }; - it('should throw if no Topic is provided', function() { - assert.throws(function() { - pubsub.subscribe(); - }, /A Topic is required for a new subscription\./); - }); - - it('should not require a subscription name', function(done) { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); + beforeEach(function() { + pubsub.auth = { + getProjectId: function(callback) { + callback(null, PROJECT_ID); + } }; - pubsub.subscribe(TOPIC_NAME, done); - }); - - it('should not require a sub name and accept options', function(done) { - var opts = {}; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); + pubsub.api = { + fakeClient: { + fakeMethod: function(reqOpts, gaxOpts, callback) { + callback(); // in most cases, the done fn + } + } }; - pubsub.subscribe(TOPIC_NAME, opts, done); - }); - - it('should not require configuration options', function(done) { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); + fakeUtil.replaceProjectIdToken = function(reqOpts) { + return reqOpts; }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, done); }); - it('should allow undefined/optional configuration options', function(done) { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); + it('should get the project id', function(done) { + pubsub.auth.getProjectId = function(callback) { + assert.strictEqual(typeof callback, 'function'); + done(); }; - pubsub.subscribe(TOPIC_NAME, SUB_NAME, undefined, done); + pubsub.request(CONFIG, assert.ifError); }); - it('should create a Subscription', function(done) { - var opts = { a: 'b', c: 'd' }; + it('should return auth errors to the callback', function(done) { + var error = new Error('err'); - pubsub.request = util.noop; - - pubsub.subscription = function(subName, options) { - assert.strictEqual(subName, SUB_NAME); - assert.deepEqual(options, opts); - setImmediate(done); - return SUBSCRIPTION; + pubsub.auth.getProjectId = function(callback) { + callback(error); }; - pubsub.subscribe(TOPIC_NAME, SUB_NAME, opts, assert.ifError); + pubsub.request(CONFIG, function(err) { + assert.strictEqual(err, error); + done(); + }); }); - it('should create a Topic object from a string', function(done) { - pubsub.request = util.noop; - - pubsub.topic = function(topicName) { - assert.strictEqual(topicName, TOPIC_NAME); - setImmediate(done); - return TOPIC; + it('should replace the project id token on reqOpts', function(done) { + fakeUtil.replaceProjectIdToken = function(reqOpts, projectId) { + assert.deepEqual(reqOpts, CONFIG.reqOpts); + assert.strictEqual(projectId, PROJECT_ID); + done(); }; - pubsub.subscribe(TOPIC_NAME, SUB_NAME, assert.ifError); + pubsub.request(CONFIG, assert.ifError); }); - it('should send correct request', function(done) { - pubsub.topic = function(topicName) { - return { - name: topicName - }; + it('should call the specified method', function(done) { + pubsub.api.fakeClient.fakeMethod = function(reqOpts, gaxOpts, callback) { + assert.deepEqual(reqOpts, CONFIG.reqOpts); + assert.strictEqual(gaxOpts, CONFIG.gaxOpts); + callback(); // the done function }; - pubsub.subscription = function(subName) { - return { - name: subName - }; - }; - - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'createSubscription'); - assert.strictEqual(protoOpts.timeout, pubsub.timeout); - assert.strictEqual(reqOpts.topic, TOPIC_NAME); - assert.strictEqual(reqOpts.name, SUB_NAME); - done(); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, assert.ifError); + pubsub.request(CONFIG, done); }); - it('should pass options to the api request', function(done) { - var options = { - ackDeadlineSeconds: 90, - autoAck: true, - encoding: 'utf-8', - interval: 3, - maxInProgress: 5, - retainAckedMessages: true, - pushEndpoint: 'https://domain/push', - timeout: 30000 - }; - - var expectedBody = extend({ - topic: TOPIC_NAME, - name: SUB_NAME - }, options, { - pushConfig: { - pushEndpoint: options.pushEndpoint + it('should instantiate the client lazily', function(done) { + var fakeClientInstance = { + fakeMethod: function(reqOpts, gaxOpts, callback) { + assert.strictEqual(pubsub.api.fakeClient, fakeClientInstance); + callback(); // the done function } - }); - - delete expectedBody.autoAck; - delete expectedBody.encoding; - delete expectedBody.interval; - delete expectedBody.maxInProgress; - delete expectedBody.pushEndpoint; - delete expectedBody.timeout; - - pubsub.topic = function() { - return { - name: TOPIC_NAME - }; }; - pubsub.subscription = function() { + v1Override = function(options) { + assert.strictEqual(options, pubsub.options); + return { - name: SUB_NAME + fakeClient: function(options) { + assert.strictEqual(options, pubsub.options); + return fakeClientInstance; + } }; }; - pubsub.request = function(protoOpts, reqOpts) { - assert.notStrictEqual(reqOpts, options); - assert.deepEqual(reqOpts, expectedBody); - done(); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, options, assert.ifError); - }); - - describe('message retention', function() { - it('should accept a number', function(done) { - var threeDaysInSeconds = 3 * 24 * 60 * 60; - - pubsub.request = function(protoOpts, reqOpts) { - assert.strictEqual(reqOpts.retainAckedMessages, true); - - assert.strictEqual( - reqOpts.messageRetentionDuration.seconds, - threeDaysInSeconds - ); - - assert.strictEqual(reqOpts.messageRetentionDuration.nanos, 0); - - done(); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, { - messageRetentionDuration: threeDaysInSeconds - }, assert.ifError); - }); + delete pubsub.api.fakeClient; + pubsub.request(CONFIG, done); }); - describe('error', function() { - var error = new Error('Error.'); - var apiResponse = { name: SUB_NAME }; + it('should return the rpc function when returnFn is set', function(done) { + var config = extend({ + returnFn: true + }, CONFIG); - beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(error, apiResponse); - }; - }); - - it('should re-use existing subscription', function(done) { - var apiResponse = { code: 409 }; - - pubsub.subscription = function() { - return SUBSCRIPTION; - }; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback({ code: 409 }, apiResponse); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, function(err, subscription) { - assert.ifError(err); - assert.strictEqual(subscription, SUBSCRIPTION); - done(); - }); - }); - - it('should return error & API response to the callback', function(done) { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(error, apiResponse); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { - assert.strictEqual(err, error); - assert.strictEqual(sub, null); - assert.strictEqual(resp, apiResponse); - done(); - }); + pubsub.request(config, function(err, requestFn) { + assert.ifError(err); + requestFn(done); }); }); - describe('success', function() { - var apiResponse = { name: SUB_NAME }; - - beforeEach(function() { - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); - }; - }); - - it('should return Subscription & resp to the callback', function(done) { - var subscription = {}; - - pubsub.subscription = function() { - return subscription; - }; - - pubsub.request = function(protoOpts, reqOpts, callback) { - callback(null, apiResponse); - }; - - pubsub.subscribe(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { - assert.ifError(err); - assert.strictEqual(sub, subscription); - assert.strictEqual(resp, apiResponse); - done(); - }); - }); + it('should do nothing if sandbox env var is set', function(done) { + process.env.GCLOUD_SANDBOX_ENV = true; + pubsub.request(CONFIG, done); // should not fire done + process.evn.GCLOUD_SANDBOX_ENV = false; + done(); }); }); @@ -903,53 +972,25 @@ describe('PubSub', function() { }); it('should pass specified name to the Subscription', function(done) { - SubscriptionOverride = function(pubsub, options) { - assert.equal(options.name, SUB_NAME); + SubscriptionOverride = function(pubsub, name) { + assert.equal(name, SUB_NAME); done(); }; - pubsub.subscription(SUB_NAME, {}); + pubsub.subscription(SUB_NAME); }); it('should honor settings', function(done) { - SubscriptionOverride = function(pubsub, options) { - assert.deepEqual(options, CONFIG); + SubscriptionOverride = function(pubsub, name, options) { + assert.strictEqual(options, CONFIG); done(); }; pubsub.subscription(SUB_NAME, CONFIG); }); - it('should not require a name', function(done) { - SubscriptionOverride = function(pubsub, options) { - assert.deepEqual(options, { - name: undefined - }); - done(); - }; - - pubsub.subscription(); - }); - - it('should not require a name and accept options', function(done) { - SubscriptionOverride = function(pubsub, options) { - var expectedOptions = extend({}, CONFIG); - expectedOptions.name = undefined; - - assert.deepEqual(options, expectedOptions); - done(); - }; - - pubsub.subscription(CONFIG); - }); - - it('should not require options', function(done) { - SubscriptionOverride = function(pubsub, options) { - assert.deepEqual(options, { - name: SUB_NAME - }); - done(); - }; - - pubsub.subscription(SUB_NAME); + it('should throw if a name is not provided', function() { + assert.throws(function() { + return pubsub.subscription(); + }, /A name must be specified for a subscription\./); }); }); @@ -957,170 +998,11 @@ describe('PubSub', function() { it('should throw if a name is not provided', function() { assert.throws(function() { pubsub.topic(); - }, /A name must be specified for a new topic\./); + }, /A name must be specified for a topic\./); }); it('should return a Topic object', function() { assert(pubsub.topic('new-topic') instanceof Topic); }); }); - - describe('request', function() { - var TIMEOUT = Math.random(); - - beforeEach(function() { - GAX_CONFIG_PUBLISHER_OVERRIDE.methods = { - MethodName: { - timeout_millis: TIMEOUT - } - }; - }); - - after(function() { - GAX_CONFIG_PUBLISHER_OVERRIDE.methods = {}; - }); - - it('should pass through the request', function(done) { - var args = [ - { - service: 'Publisher', - method: 'MethodName' - }, - { - value: true - }, - { - anotherValue: true - } - ]; - - grpcServiceRequestOverride = function() { - assert.strictEqual(this, pubsub); - assert.strictEqual(args[0], arguments[0]); - assert.strictEqual(args[1], arguments[1]); - assert.strictEqual(args[2], arguments[2]); - done(); - }; - - pubsub.request.apply(pubsub, args); - }); - - it('should assign a timeout', function(done) { - grpcServiceRequestOverride = function(protoOpts) { - assert.strictEqual(protoOpts.timeout, TIMEOUT); - done(); - }; - - pubsub.request({ - service: 'Publisher', - method: 'MethodName' - }); - }); - - it('should not override a timeout if set', function(done) { - var timeout = 0; - - grpcServiceRequestOverride = function(protoOpts) { - assert.strictEqual(protoOpts.timeout, timeout); - done(); - }; - - pubsub.request({ - service: 'Publisher', - method: 'MethodName', - timeout: timeout - }); - }); - - it('should camel case the method name', function(done) { - grpcServiceRequestOverride = function(protoOpts) { - assert.strictEqual(protoOpts.timeout, TIMEOUT); - done(); - }; - - pubsub.request({ - service: 'Publisher', - method: 'methodName' - }); - }); - }); - - describe('determineBaseUrl_', function() { - function setHost(host) { - process.env.PUBSUB_EMULATOR_HOST = host; - } - - beforeEach(function() { - delete process.env.PUBSUB_EMULATOR_HOST; - }); - - it('should set base url to parameter sent', function() { - var defaultBaseUrl_ = 'defaulturl'; - var testingUrl = 'localhost:8085'; - - setHost(defaultBaseUrl_); - pubsub.defaultBaseUrl_ = defaultBaseUrl_; - - pubsub.determineBaseUrl_(testingUrl); - assert.strictEqual(pubsub.baseUrl_, testingUrl); - }); - - it('should default to defaultBaseUrl_', function() { - var defaultBaseUrl_ = 'defaulturl'; - pubsub.defaultBaseUrl_ = defaultBaseUrl_; - - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, defaultBaseUrl_); - }); - - it('should remove slashes from the baseUrl', function() { - var expectedBaseUrl = 'localhost:8080'; - - setHost('localhost:8080/'); - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, expectedBaseUrl); - - setHost('localhost:8080//'); - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, expectedBaseUrl); - }); - - it('should remove the protocol if specified', function() { - setHost('http://localhost:8080'); - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, 'localhost:8080'); - - setHost('https://localhost:8080'); - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, 'localhost:8080'); - }); - - it('should not set customEndpoint_ when using default baseurl', function() { - var pubsub = new PubSub({ projectId: PROJECT_ID }); - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.customEndpoint_, undefined); - }); - - describe('with PUBSUB_EMULATOR_HOST environment variable', function() { - var PUBSUB_EMULATOR_HOST = 'localhost:9090'; - - beforeEach(function() { - setHost(PUBSUB_EMULATOR_HOST); - }); - - after(function() { - delete process.env.PUBSUB_EMULATOR_HOST; - }); - - it('should use the PUBSUB_EMULATOR_HOST env var', function() { - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.baseUrl_, PUBSUB_EMULATOR_HOST); - }); - - it('should set customEndpoint_', function() { - pubsub.determineBaseUrl_(); - assert.strictEqual(pubsub.customEndpoint_, true); - }); - }); - }); }); From 5e9e939e7bff0dc941f91ba05f55b294b39374ee Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 27 Jul 2017 20:14:11 -0400 Subject: [PATCH 34/67] update unit tests for iam.js --- packages/pubsub/src/iam.js | 2 +- packages/pubsub/test/iam.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pubsub/src/iam.js b/packages/pubsub/src/iam.js index 21c6ab012ed..914c43fbb41 100644 --- a/packages/pubsub/src/iam.js +++ b/packages/pubsub/src/iam.js @@ -69,7 +69,7 @@ var util = require('util'); */ function IAM(pubsub, id) { var config = { - baseUrl: pubsub.defaultBaseUrl_, + baseUrl: 'pubsub.googleapis.com', protosDir: path.resolve(__dirname, '../protos'), protoServices: { IAMPolicy: { diff --git a/packages/pubsub/test/iam.js b/packages/pubsub/test/iam.js index 46d77e74158..8c346a02761 100644 --- a/packages/pubsub/test/iam.js +++ b/packages/pubsub/test/iam.js @@ -72,7 +72,7 @@ describe('IAM', function() { var config = iam.calledWith_[0]; var options = iam.calledWith_[1]; - assert.strictEqual(config.baseUrl, PUBSUB.defaultBaseUrl_); + assert.strictEqual(config.baseUrl, 'pubsub.googleapis.com'); var protosDir = path.resolve(__dirname, '../protos'); assert.strictEqual(config.protosDir, protosDir); From 57f02f1c36d0d3d960faa967f63aa4181ac9180d Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 27 Jul 2017 20:26:24 -0400 Subject: [PATCH 35/67] update unit tests for snapshot.js --- packages/pubsub/src/snapshot.js | 1 - packages/pubsub/test/index.js | 2 +- packages/pubsub/test/snapshot.js | 76 +++++++++++++------------------- 3 files changed, 31 insertions(+), 48 deletions(-) diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index 5858b256265..c4c36085abd 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -89,7 +89,6 @@ var is = require('is'); */ function Snapshot(parent, name) { this.parent = parent; - this.api = parent.api; this.name = Snapshot.formatName_(parent.projectId, name); if (is.fn(parent.createSnapshot)) { diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 276bc83c306..13f26659841 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -104,7 +104,7 @@ var GAX_CONFIG = { } }; -describe.only('PubSub', function() { +describe('PubSub', function() { var PubSub; var PROJECT_ID = 'test-project'; var pubsub; diff --git a/packages/pubsub/test/snapshot.js b/packages/pubsub/test/snapshot.js index d97f76a6559..b6701f1164c 100644 --- a/packages/pubsub/test/snapshot.js +++ b/packages/pubsub/test/snapshot.js @@ -17,36 +17,21 @@ 'use strict'; var assert = require('assert'); -var proxyquire = require('proxyquire'); - -function FakeGrpcServiceObject() { - this.calledWith_ = arguments; -} describe('Snapshot', function() { - var Snapshot; + var Snapshot = require('../src/snapshot.js'); + var snapshot; var SNAPSHOT_NAME = 'a'; var PROJECT_ID = 'grape-spaceship-123'; - var PUBSUB = { - projectId: PROJECT_ID - }; - var SUBSCRIPTION = { - parent: PUBSUB, + projectId: PROJECT_ID, + api: {}, createSnapshot: function() {}, seek: function() {} }; - before(function() { - Snapshot = proxyquire('../src/snapshot.js', { - '@google-cloud/common-grpc': { - ServiceObject: FakeGrpcServiceObject - } - }); - }); - describe('initialization', function() { var FULL_SNAPSHOT_NAME = 'a/b/c/d'; var formatName_; @@ -58,10 +43,18 @@ describe('Snapshot', function() { }; }); + beforeEach(function() { + snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); + }); + after(function() { Snapshot.formatName_ = formatName_; }); + it('should localize the parent', function() { + assert.strictEqual(snapshot.parent, SUBSCRIPTION); + }); + describe('name', function() { it('should create and cache the full name', function() { Snapshot.formatName_ = function(projectId, name) { @@ -70,11 +63,11 @@ describe('Snapshot', function() { return FULL_SNAPSHOT_NAME; }; - var snapshot = new Snapshot(PUBSUB, SNAPSHOT_NAME); + var snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); assert.strictEqual(snapshot.name, FULL_SNAPSHOT_NAME); }); - it('should pull the projectId from subscription parent', function() { + it('should pull the projectId from parent object', function() { Snapshot.formatName_ = function(projectId, name) { assert.strictEqual(projectId, PROJECT_ID); assert.strictEqual(name, SNAPSHOT_NAME); @@ -86,37 +79,15 @@ describe('Snapshot', function() { }); }); - it('should inherit from GrpcServiceObject', function() { - var snapshot = new Snapshot(PUBSUB, SNAPSHOT_NAME); - var calledWith = snapshot.calledWith_[0]; - - assert(snapshot instanceof FakeGrpcServiceObject); - assert.strictEqual(calledWith.parent, PUBSUB); - assert.strictEqual(calledWith.id, FULL_SNAPSHOT_NAME); - assert.deepEqual(calledWith.methods, { - delete: { - protoOpts: { - service: 'Subscriber', - method: 'deleteSnapshot' - }, - reqOpts: { - snapshot: FULL_SNAPSHOT_NAME - } - } - }); - }); - describe('with Subscription parent', function() { it('should include the create method', function(done) { - SUBSCRIPTION.createSnapshot = function(callback) { + SUBSCRIPTION.createSnapshot = function(name, callback) { + assert.strictEqual(name, SNAPSHOT_NAME); callback(); // The done function }; var snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); - var calledWith = snapshot.calledWith_[0]; - - assert(calledWith.methods.create); - calledWith.createMethod(done); + snapshot.create(done); }); it('should create a seek method', function(done) { @@ -146,4 +117,17 @@ describe('Snapshot', function() { assert.strictEqual(name, EXPECTED); }); }); + + describe('delete', function() { + it('should make the correct request', function(done) { + snapshot.parent.request = function(config, callback) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'deleteSnapshot'); + assert.deepEqual(config.reqOpts, { snapshot: snapshot.name }); + callback(); // the done fn + }; + + snapshot.delete(done); + }); + }); }); From 25307612d69586f8ec85ad359ba47d4f1f0fa559 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 27 Jul 2017 21:04:36 -0400 Subject: [PATCH 36/67] update unit tests for topic.js --- packages/pubsub/src/topic.js | 21 +- packages/pubsub/test/topic.js | 358 +++++++++++++++------------------- 2 files changed, 172 insertions(+), 207 deletions(-) diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index bb16fb01e72..27169e46561 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -53,7 +53,6 @@ var Publisher = require('./publisher.js'); function Topic(pubsub, name, options) { this.name = Topic.formatName_(pubsub.projectId, name); this.pubsub = pubsub; - this.projectId = pubsub.projectId; this.request = pubsub.request.bind(pubsub); /** @@ -347,16 +346,8 @@ Topic.prototype.getSubscriptions = function(options, callback) { if (subscriptions) { arguments[1] = subscriptions.map(function(sub) { - // Depending on if we're using a subscriptions.list or - // topics.subscriptions.list API endpoint, we will get back a - // Subscription resource or just the name of the subscription. - var subscriptionInstance = self.subscription(sub.name || sub); - - if (sub.name) { - subscriptionInstance.metadata = sub; - } - - return subscriptionInstance; + // ListTopicSubscriptions only returns sub names + return self.subscription(sub); }); } @@ -430,6 +421,14 @@ Topic.prototype.subscription = function(name, options) { return this.pubsub.subscription(name, options); }; +/*! Developer Documentation + * + * These methods can be agto-paginated. + */ +common.paginator.extend(Topic, [ + 'getSubscriptions' +]); + /*! Developer Documentation * * All async methods (except for streams) will return a Promise in the event diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index 48985c6353a..444c19bfe79 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -18,7 +18,6 @@ var assert = require('assert'); var extend = require('extend'); -var GrpcServiceObject = require('@google-cloud/common-grpc').ServiceObject; var nodeutil = require('util'); var proxyquire = require('proxyquire'); var util = require('@google-cloud/common').util; @@ -31,42 +30,58 @@ var fakeUtil = extend({}, util, { } promisified = true; - assert.deepEqual(options.exclude, ['subscription']); + assert.deepEqual(options.exclude, [ + 'publisher', + 'subscription' + ]); } }); -function FakeGrpcServiceObject() { - this.calledWith_ = arguments; - GrpcServiceObject.apply(this, arguments); +function FakeIAM() { + this.calledWith_ = [].slice.call(arguments); } -nodeutil.inherits(FakeGrpcServiceObject, GrpcServiceObject); - -function FakeIAM() { +function FakePublisher() { this.calledWith_ = [].slice.call(arguments); } -describe('Topic', function() { +var extended = false; +var fakePaginator = { + extend: function(Class, methods) { + if (Class.name !== 'Topic') { + return; + } + + assert.deepEqual(methods, ['getSubscriptions']); + extended = true; + }, + streamify: function(methodName) { + return methodName; + } +}; + +describe.only('Topic', function() { var Topic; var topic; var PROJECT_ID = 'test-project'; var TOPIC_NAME = 'projects/' + PROJECT_ID + '/topics/test-topic'; var TOPIC_UNFORMATTED_NAME = TOPIC_NAME.split('/').pop(); + var PUBSUB = { projectId: PROJECT_ID, - createTopic: util.noop + createTopic: util.noop, + request: util.noop }; before(function() { Topic = proxyquire('../src/topic.js', { '@google-cloud/common': { + paginator: fakePaginator, util: fakeUtil }, - '@google-cloud/common-grpc': { - ServiceObject: FakeGrpcServiceObject - }, - './iam.js': FakeIAM + './iam.js': FakeIAM, + './publisher.js': FakePublisher }); }); @@ -76,80 +91,50 @@ describe('Topic', function() { }); describe('initialization', function() { - it('should inherit from GrpcServiceObject', function() { - var pubsubInstance = extend({}, PUBSUB, { - createTopic: { - bind: function(context) { - assert.strictEqual(context, pubsubInstance); - } - } - }); + it('should extend the correct methods', function() { + assert(extended); // See `fakePaginator.extend` + }); - var topic = new Topic(pubsubInstance, TOPIC_NAME); - assert(topic instanceof GrpcServiceObject); - - var calledWith = topic.calledWith_[0]; - - assert.strictEqual(calledWith.parent, pubsubInstance); - assert.strictEqual(calledWith.id, TOPIC_NAME); - assert.deepEqual(calledWith.methods, { - create: true, - delete: { - protoOpts: { - service: 'Publisher', - method: 'deleteTopic' - }, - reqOpts: { - topic: TOPIC_NAME - } - }, - exists: true, - get: true, - getMetadata: { - protoOpts: { - service: 'Publisher', - method: 'getTopic' - }, - reqOpts: { - topic: TOPIC_NAME - } - } - }); + it('should streamify the correct methods', function() { + assert.strictEqual(topic.getSubscriptionsStream, 'getSubscriptions'); }); - it('should create an iam object', function() { - assert.deepEqual(topic.iam.calledWith_, [PUBSUB, TOPIC_NAME]); + it('should promisify all the things', function() { + assert(promisified); }); - it('should format name', function(done) { + it('should format the name', function() { + var formattedName = 'a/b/c/d'; + var formatName_ = Topic.formatName_; - Topic.formatName_ = function() { + Topic.formatName_ = function(projectId, name) { + assert.strictEqual(projectId, PROJECT_ID); + assert.strictEqual(name, TOPIC_NAME); + Topic.formatName_ = formatName_; - done(); + + return formattedName; }; - new Topic(PUBSUB, TOPIC_NAME); + + var topic = new Topic(PUBSUB, TOPIC_NAME); + assert.strictEqual(topic.name, formattedName); }); - }); - describe('formatMessage_', function() { - var messageString = 'string'; - var messageBuffer = new Buffer(messageString); + it('should localize the parent object', function() { + assert.strictEqual(topic.pubsub, PUBSUB); + }); - var messageObjectWithString = { data: messageString }; - var messageObjectWithBuffer = { data: messageBuffer }; + it('should localize the request function', function(done) { + PUBSUB.request = function(callback) { + callback(); // the done fn + }; - it('should handle string data', function() { - assert.deepEqual( - Topic.formatMessage_(messageObjectWithString), - { data: new Buffer(JSON.stringify(messageString)).toString('base64') } - ); + var topic = new Topic(PUBSUB, TOPIC_NAME); + topic.request(done); }); - it('should handle buffer data', function() { - assert.deepEqual( - Topic.formatMessage_(messageObjectWithBuffer), - { data: messageBuffer.toString('base64') } - ); + it('should create an iam object', function() { + assert.deepEqual(topic.iam.calledWith_, [PUBSUB, TOPIC_NAME]); }); }); @@ -165,189 +150,170 @@ describe('Topic', function() { }); }); - describe('getSubscriptions', function() { - it('should accept just a callback', function(done) { - topic.parent.getSubscriptions = function(options, callback) { - assert.deepEqual(options, { topic: topic }); - callback(); + describe('create', function() { + it('should call the parent createTopic method', function(done) { + PUBSUB.createTopic = function(name, callback) { + assert.strictEqual(name, topic.name); + callback(); // the done fn }; - topic.getSubscriptions(done); + topic.create(done); }); + }); - it('should pass correct args to pubsub#getSubscriptions', function(done) { - var opts = { a: 'b', c: 'd' }; + describe('createSubscription', function() { + it('should call the parent createSubscription method', function(done) { + var NAME = 'sub-name'; + var OPTIONS = { a: 'a' }; - topic.parent = { - getSubscriptions: function(options, callback) { - assert.deepEqual(options, opts); - assert.deepEqual(options.topic, topic); - callback(); - } + PUBSUB.createSubscription = function(topic_, name, options, callback) { + assert.strictEqual(topic_, topic); + assert.strictEqual(name, NAME); + assert.strictEqual(options, OPTIONS); + callback(); // the done fn }; - topic.getSubscriptions(opts, done); + topic.createSubscription(NAME, OPTIONS, done); }); }); - describe('getSubscriptionsStream', function() { - it('should return a stream', function(done) { - var fakeStream = {}; - - topic.parent.getSubscriptionsStream = function(options) { - assert.deepEqual(options, { topic: topic }); - setImmediate(done); - return fakeStream; + describe('delete', function() { + it('should make the proper request', function(done) { + topic.request = function(config, callback) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'deleteTopic'); + assert.deepEqual(config.reqOpts, { topic: topic.name }); + callback(); // the done fn }; - var stream = topic.getSubscriptionsStream(); - assert.strictEqual(stream, fakeStream); + topic.delete(done); }); - it('should pass correct args to getSubscriptionsStream', function(done) { - var opts = { a: 'b', c: 'd' }; + it('should optionally accept gax options', function(done) { + var options = {}; - topic.parent = { - getSubscriptionsStream: function(options) { - assert.deepEqual(options, opts); - assert.deepEqual(options.topic, topic); - done(); - } + topic.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, options); + callback(); }; - topic.getSubscriptionsStream(opts); + topic.delete(options, done); }); }); - describe('publish', function() { - var message = 'howdy'; - var attributes = { - key: 'value' - }; - - it('should throw if no message is provided', function() { - assert.throws(function() { - topic.publish(); - }, /Cannot publish without a message\./); + describe('getMetadata', function() { + it('should make the proper request', function(done) { + topic.request = function(config, callback) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'getTopic'); + assert.deepEqual(config.reqOpts, { topic: topic.name }); + callback(); // the done fn + }; - assert.throws(function() { - topic.publish([]); - }, /Cannot publish without a message\./); + topic.getMetadata(done); }); - it('should send correct api request', function(done) { - topic.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Publisher'); - assert.strictEqual(protoOpts.method, 'publish'); - - assert.strictEqual(reqOpts.topic, topic.name); - assert.deepEqual(reqOpts.messages, [ - { data: new Buffer(JSON.stringify(message)).toString('base64') } - ]); + it('should optionally accept gax options', function(done) { + var options = {}; - done(); + topic.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, options); + callback(); }; - topic.publish(message, assert.ifError); + topic.getMetadata(options, done); }); + }); - it('should honor the timeout setting', function(done) { + describe('getSubscriptions', function() { + it('should make the correct request', function(done) { var options = { - timeout: 10 + gaxOpts: {}, + a: 'a', + b: 'b' }; - topic.parent.request = function(protoOpts) { - assert.strictEqual(protoOpts.timeout, options.timeout); - done(); - }; + var expectedOptions = extend({ + topic: topic.name + }, options); - topic.publish(message, options, assert.ifError); - }); - - it('should send correct api request for raw message', function(done) { - topic.parent.request = function(protoOpts, reqOpts) { - assert.deepEqual(reqOpts.messages, [ - { - data: new Buffer(JSON.stringify(message)).toString('base64'), - attributes: attributes - } - ]); + delete expectedOptions.gaxOpts; + topic.request = function(config) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'listTopicSubscriptions'); + assert.deepEqual(config.reqOpts, expectedOptions); + assert.strictEqual(config.gaxOpts, options.gaxOpts); done(); }; - topic.publish({ - data: message, - attributes: attributes - }, { raw: true }, assert.ifError); + topic.getSubscriptions(options, assert.ifError); }); - it('should clone the provided message', function(done) { - var message = { - data: 'data' - }; - var originalMessage = extend({}, message); - - topic.parent.request = function() { - assert.deepEqual(message, originalMessage); + it('should accept only a callback', function(done) { + topic.request = function(config) { + assert.deepEqual(config.reqOpts, { topic: topic.name }); + assert.strictEqual(config.gaxOpts, undefined); done(); }; - topic.publish(message, { raw: true }, assert.ifError); - }); - - it('should execute callback', function(done) { - topic.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, {}); - }; - - topic.publish(message, done); + topic.getSubscriptions(assert.ifError); }); - it('should execute callback with error', function(done) { - var error = new Error('Error.'); - var apiResponse = {}; + it('should create subscription objects', function(done) { + var fakeSubs = ['a', 'b', 'c']; - topic.parent.request = function(protoOpts, reqOpts, callback) { - callback(error, apiResponse); + topic.subscription = function(name) { + return { + name: name + }; }; - topic.publish(message, function(err, ackIds, apiResponse_) { - assert.strictEqual(err, error); - assert.strictEqual(ackIds, null); - assert.strictEqual(apiResponse_, apiResponse); + topic.request = function(config, callback) { + callback(null, fakeSubs); + }; + topic.getSubscriptions(function(err, subscriptions) { + assert.ifError(err); + assert.deepEqual(subscriptions, [ + { name: 'a' }, + { name: 'b' }, + { name: 'c' } + ]); done(); }); }); - it('should execute callback with apiResponse', function(done) { - var resp = { success: true }; + it('should pass all params to the callback', function(done) { + var err_ = new Error('err'); + var subs_ = []; + var nextQuery_ = {}; + var apiResponse_ = {}; - topic.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); + topic.request = function(config, callback) { + callback(err_, subs_, nextQuery_, apiResponse_); }; - topic.publish(message, function(err, ackIds, apiResponse) { - assert.deepEqual(resp, apiResponse); + topic.getSubscriptions(function(err, subs, nextQuery, apiResponse) { + assert.strictEqual(err, err_); + assert.deepEqual(subs, subs_); + assert.strictEqual(nextQuery, nextQuery_); + assert.strictEqual(apiResponse, apiResponse_); done(); }); }); }); - describe('subscribe', function() { - it('should pass correct arguments to pubsub#subscribe', function(done) { - var subscriptionName = 'subName'; - var opts = {}; + describe('publisher', function() { + it('should return a Publisher instance', function() { + var options = {}; - topic.parent.subscribe = function(t, subName, options, callback) { - assert.deepEqual(t, topic); - assert.equal(subName, subscriptionName); - assert.deepEqual(options, opts); - callback(); - }; + var publisher = topic.publisher(options); + var args = publisher.calledWith_; - topic.subscribe(subscriptionName, opts, done); + assert(publisher instanceof FakePublisher); + assert.strictEqual(args[0], topic); + assert.strictEqual(args[1], options); }); }); From 0d0422afa1f784105c3f38e38ea52025edf9a27b Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 10:49:05 -0400 Subject: [PATCH 37/67] update unit tests for subscription.js --- packages/pubsub/src/subscription.js | 45 +- packages/pubsub/test/subscription.js | 1945 ++++++++++++++------------ packages/pubsub/test/topic.js | 2 +- 3 files changed, 1086 insertions(+), 906 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index e4dd130606c..f7ec44df314 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -152,12 +152,11 @@ function Subscription(pubsub, name, options) { this.request = pubsub.request.bind(pubsub); this.histogram = new Histogram(); - this.projectId = pubsub.projectId; this.name = Subscription.formatName_(pubsub.projectId, name); this.connectionPool = null; this.ackDeadline = options.ackDeadline || 10000; - this.maxConnections = options.maxConnections; + this.maxConnections = options.maxConnections || 5; this.inventory_ = { lease: [], @@ -176,6 +175,7 @@ function Subscription(pubsub, name, options) { this.userClosed_ = false; events.EventEmitter.call(this); + this.messageListeners = 0; if (options.topic) { this.create = pubsub.createSubscription.bind(pubsub, options.topic, name); @@ -268,7 +268,7 @@ Subscription.prototype.ack_ = function(message) { Subscription.prototype.breakLease_ = function(message) { var messageIndex = this.inventory_.lease.indexOf(message.ackId); - this.inventory_.lease.splice(0, messageIndex); + this.inventory_.lease.splice(messageIndex, 1); this.inventory_.bytes -= message.data.length; if (this.connectionPool) { @@ -303,10 +303,8 @@ Subscription.prototype.close = function(callback) { * */ Subscription.prototype.closeConnection_ = function(callback) { - callback = callback || common.util.noop; - if (this.connectionPool) { - this.connectionPool.close(callback); + this.connectionPool.close(callback || common.util.noop); this.connectionPool = null; } else if (is.fn(callback)) { setImmediate(callback); @@ -563,7 +561,7 @@ Subscription.prototype.listenForEvents_ = function() { if (event === 'message') { self.messageListeners++; - if (!self.connection) { + if (!self.connectionPool) { self.userClosed_ = false; self.openConnection_(); } @@ -613,6 +611,8 @@ Subscription.prototype.nack_ = function(message) { return; } + var self = this; + this.connectionPool.acquire(message.connectionId, function(err, connection) { if (err) { self.emit('error', err); @@ -684,25 +684,22 @@ Subscription.prototype.renewLeases_ = function() { modifyDeadlineSeconds: Array(ackIds.length).fill(ackDeadlineSeconds) }); }); - - this.setLeaseTimeout_(); - return; + } else { + this.request({ + client: 'subscriberClient', + method: 'modifyAckDeadline', + reqOpts: { + subscription: self.name, + ackIds: ackIds, + ackDeadlineSeconds: ackDeadlineSeconds + } + }, function(err) { + if (err) { + self.emit('error', err); + } + }); } - this.request({ - client: 'subscriberClient', - method: 'modifyAckDeadline', - reqOpts: { - subscription: self.name, - ackIds: ackIds, - ackDeadlineSeconds: ackDeadlineSeconds - } - }, function(err) { - if (err) { - self.emit('error', err); - } - }); - this.setLeaseTimeout_(); }; diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index a84bc9b9c2d..1651c266541 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -17,21 +17,35 @@ 'use strict'; var assert = require('assert'); +var common = require('@google-cloud/common'); +var events = require('events'); var extend = require('extend'); +var is = require('is'); +var os = require('os'); var proxyquire = require('proxyquire'); -var util = require('@google-cloud/common').util; +var util = require('util'); var promisified = false; -var fakeUtil = extend({}, util, { - promisifyAll: function(Class) { - if (Class.name === 'Subscription') { - promisified = true; +var fakeUtil = extend({}, common.util, { + promisifyAll: function(Class, options) { + if (Class.name !== 'Subscription') { + return; } + + promisified = true; + assert.deepEqual(options.exclude, ['snapshot']); } }); -function FakeGrpcServiceObject() { - this.calledWith_ = arguments; +function FakeConnectionPool() { + this.calledWith_ = [].slice.call(arguments); + events.EventEmitter.call(this); +} + +util.inherits(FakeConnectionPool, events.EventEmitter); + +function FakeHistogram() { + this.calledWith_ = [].slice.call(arguments); } function FakeIAM() { @@ -39,48 +53,20 @@ function FakeIAM() { } function FakeSnapshot() { - this.calledWith_ = arguments; + this.calledWith_ = [].slice.call(arguments); } -var formatMessageOverride; - -describe('Subscription', function() { +describe.only('Subscription', function() { var Subscription; var subscription; var PROJECT_ID = 'test-project'; var SUB_NAME = 'test-subscription'; var SUB_FULL_NAME = 'projects/' + PROJECT_ID + '/subscriptions/' + SUB_NAME; + var PUBSUB = { projectId: PROJECT_ID, - request: util.noop - }; - var message = 'howdy'; - var messageBuffer = new Buffer(message).toString('base64'); - var messageBinary = new Buffer(message).toString('binary'); - var messageObj = { - receivedMessages: [{ - ackId: 'abc', - message: { - data: messageBuffer, - messageId: 7 - } - }] - }; - var expectedMessage = { - ackId: 'abc', - data: message, - id: 7 - }; - var expectedMessageAsBinary = { - ackId: 'abc', - data: messageBinary, - id: 7 - }; - var expectedMessageAsBase64 = { - ackId: 'abc', - data: messageBuffer, - id: 7 + request: fakeUtil.noop }; before(function() { @@ -88,27 +74,16 @@ describe('Subscription', function() { '@google-cloud/common': { util: fakeUtil }, - '@google-cloud/common-grpc': { - ServiceObject: FakeGrpcServiceObject - }, + './connection-pool.js': FakeConnectionPool, + './histogram.js': FakeHistogram, './iam.js': FakeIAM, './snapshot.js': FakeSnapshot }); - - var formatMessage = Subscription.formatMessage_; - Subscription.formatMessage_ = function() { - return (formatMessageOverride || formatMessage).apply(null, arguments); - }; }); beforeEach(function() { - subscription = new Subscription(PUBSUB, { name: SUB_NAME }); - PUBSUB.request = util.noop; - subscription.parent = PUBSUB; - }); - - afterEach(function() { - formatMessageOverride = null; + PUBSUB.request = fakeUtil.noop; + subscription = new Subscription(PUBSUB, SUB_NAME); }); describe('initialization', function() { @@ -116,247 +91,121 @@ describe('Subscription', function() { assert(promisified); }); - describe('name', function() { - var FORMATTED_NAME = 'formatted-name'; - var GENERATED_NAME = 'generated-name'; - - var formatName_; - var generateName_; - - before(function() { - formatName_ = Subscription.formatName_; - generateName_ = Subscription.generateName_; - - Subscription.formatName_ = function() { - return FORMATTED_NAME; - }; - - Subscription.generateName_ = function() { - return GENERATED_NAME; - }; - }); - - afterEach(function() { - Subscription.formatName_ = function() { - return FORMATTED_NAME; - }; - - Subscription.generateName_ = function() { - return GENERATED_NAME; - }; - }); - - after(function() { - Subscription.formatName_ = formatName_; - Subscription.generateName_ = generateName_; - }); - - it('should generate name', function(done) { - Subscription.formatName_ = function(projectId, name) { - assert.strictEqual(name, GENERATED_NAME); - done(); - }; - - new Subscription(PUBSUB, {}); - }); - - it('should format name', function() { - Subscription.formatName_ = function(projectId, name) { - assert.strictEqual(projectId, PROJECT_ID); - assert.strictEqual(name, SUB_NAME); - return FORMATTED_NAME; - }; - - var subscription = new Subscription(PUBSUB, { name: SUB_NAME }); - assert.strictEqual(subscription.name, FORMATTED_NAME); - }); + it('should localize the pubsub object', function() { + assert.strictEqual(subscription.pubsub, PUBSUB); }); - it('should honor configuration settings', function() { - var CONFIG = { - name: SUB_NAME, - autoAck: true, - interval: 100, - maxInProgress: 3, - encoding: 'binary', - timeout: 30000 + it('should localize pubsub request method', function(done) { + PUBSUB.request = function(callback) { + callback(); // the done fn }; - var sub = new Subscription(PUBSUB, CONFIG); - assert.strictEqual(sub.autoAck, CONFIG.autoAck); - assert.strictEqual(sub.interval, CONFIG.interval); - assert.strictEqual(sub.encoding, CONFIG.encoding); - assert.strictEqual(sub.maxInProgress, CONFIG.maxInProgress); - assert.strictEqual(sub.timeout, CONFIG.timeout); - }); - - it('should be closed', function() { - assert.strictEqual(subscription.closed, true); - }); - - it('should default autoAck to false if not specified', function() { - assert.strictEqual(subscription.autoAck, false); - }); - it('should set default interval if one is not specified', function() { - assert.equal(subscription.interval, 10); + var subscription = new Subscription(PUBSUB, SUB_NAME); + subscription.request(done); }); - it('should start inProgressAckIds as an empty object', function() { - assert.deepEqual(subscription.inProgressAckIds, {}); + it('should create a histogram instance', function() { + assert(subscription.histogram instanceof FakeHistogram); }); - it('should default maxInProgress to Infinity if not specified', function() { - assert.strictEqual(subscription.maxInProgress, Infinity); - }); + it('should format the sub name', function() { + var formattedName = 'a/b/c/d'; + var formatName = Subscription.formatName_; - it('should set messageListeners to 0', function() { - assert.strictEqual(subscription.messageListeners, 0); - }); + Subscription.formatName_ = function(projectId, name) { + assert.strictEqual(projectId, PROJECT_ID); + assert.strictEqual(name, SUB_NAME); - it('should not be paused', function() { - assert.strictEqual(subscription.paused, false); - }); + Subscription.formatName_ = formatName; - it('should default encoding to utf-8 if not specified', function() { - assert.strictEqual(subscription.encoding, 'utf-8'); - }); + return formattedName; + }; - it('should default timeout to 92 seconds', function() { - assert.strictEqual(subscription.timeout, 92000); + var subscription = new Subscription(PUBSUB, SUB_NAME); + assert.strictEqual(subscription.name, formattedName); }); - it('should create an iam object', function() { - assert.deepEqual(subscription.iam.calledWith_, [ - PUBSUB, - SUB_FULL_NAME - ]); - }); + it('should honor configuration settings', function() { + var options = { + ackDeadline: 5000, + maxConnections: 2, + flowControl: { + maxBytes: 5, + maxMessages: 10 + } + }; - it('should inherit from GrpcServiceObject', function() { - assert(subscription instanceof FakeGrpcServiceObject); + var subscription = new Subscription(PUBSUB, SUB_NAME, options); - var calledWith = subscription.calledWith_[0]; + assert.strictEqual(subscription.ackDeadline, options.ackDeadline); + assert.strictEqual(subscription.maxConnections, options.maxConnections); - assert.strictEqual(calledWith.parent, PUBSUB); - assert.strictEqual(calledWith.id, SUB_FULL_NAME); - assert.deepEqual(calledWith.methods, { - exists: true, - get: true, - getMetadata: { - protoOpts: { - service: 'Subscriber', - method: 'getSubscription' - }, - reqOpts: { - subscription: subscription.name - } - } + assert.deepEqual(subscription.flowControl, { + maxBytes: options.flowControl.maxBytes, + maxMessages: options.flowControl.maxMessages }); }); - it('should allow creating if it is a Topic', function(done) { - var topicInstance = {}; + it('should set sensible defaults', function() { + assert.strictEqual(subscription.ackDeadline, 10000); + assert.strictEqual(subscription.maxConnections, 5); + assert.strictEqual(subscription.userClosed_, false); + assert.strictEqual(subscription.messageListeners, 0); - var pubSubInstance = extend({}, PUBSUB, { - subscribe: { - bind: function(context, topic) { - assert.strictEqual(context, pubSubInstance); - assert.strictEqual(topic, topicInstance); - done(); - } - } + assert.deepEqual(subscription.flowControl, { + maxBytes: os.freemem() * 0.2, + maxMessages: Infinity }); + }); - var subscription = new Subscription(pubSubInstance, { - name: SUB_NAME, - topic: topicInstance - }); + it('should create an inventory object', function() { + assert(is.object(subscription.inventory_)); + assert(is.array(subscription.inventory_.lease)); + assert(is.array(subscription.inventory_.ack)); + assert(is.array(subscription.inventory_.nack)); + assert.strictEqual(subscription.inventory_.bytes, 0); + }); - var calledWith = subscription.calledWith_[0]; - assert.deepEqual(calledWith.methods.create, true); + it('should inherit from EventEmitter', function() { + assert(subscription instanceof events.EventEmitter); }); - }); - describe('formatMessage_', function() { - it('should decode stringified JSON to object', function() { - var obj = { hi: 'there' }; - var stringified = new Buffer(JSON.stringify(obj)).toString('base64'); - var attributes = {}; - var publishTime = { - seconds: '1480413405', - nanos: 617000000 + it('should make a create method if a topic is found', function(done) { + var TOPIC_NAME = 'test-topic'; + + PUBSUB.createSubscription = function(topic, subName, callback) { + assert.strictEqual(topic, TOPIC_NAME); + assert.strictEqual(subName, SUB_NAME); + callback(); // the done function }; - var seconds = parseInt(publishTime.seconds, 10); - var milliseconds = parseInt(publishTime.nanos, 10) / 1e6; - var expectedDate = new Date(seconds * 1000 + milliseconds); - - var msg = Subscription.formatMessage_({ - ackId: 'abc', - message: { - data: stringified, - messageId: 7, - attributes: attributes, - publishTime: publishTime - } + var subscription = new Subscription(PUBSUB, SUB_NAME, { + topic: TOPIC_NAME }); - assert.deepEqual(msg, { - ackId: 'abc', - id: 7, - data: obj, - attributes: attributes, - timestamp: expectedDate - }); + subscription.create(done); }); - it('should work if nanos is 0', function() { - var obj = { hi: 'there' }; - var stringified = new Buffer(JSON.stringify(obj)).toString('base64'); - var attributes = {}; - var publishTime = { - seconds: '1480413405', - nanos: 0 - }; + it('should create an IAM object', function() { + assert(subscription.iam instanceof FakeIAM); - var seconds = parseInt(publishTime.seconds, 10); - var milliseconds = parseInt(publishTime.nanos, 10) / 1e6; - var expectedDate = new Date(seconds * 1000 + milliseconds); - - var msg = Subscription.formatMessage_({ - ackId: 'abc', - message: { - data: stringified, - messageId: 7, - attributes: attributes, - publishTime: publishTime - } - }); + var args = subscription.iam.calledWith_; - assert.deepEqual(msg, { - ackId: 'abc', - id: 7, - data: obj, - attributes: attributes, - timestamp: expectedDate - }); + assert.strictEqual(args[0], PUBSUB); + assert.strictEqual(args[1], subscription.name); }); - it('should decode buffer to string', function() { - var msg = Subscription.formatMessage_(messageObj.receivedMessages[0]); - assert.deepEqual(msg, expectedMessage); - }); + it('should listen for events', function() { + var called = false; + var listenForEvents = Subscription.prototype.listenForEvents_; - it('should decode buffer to base64', function() { - var msg = Subscription - .formatMessage_(messageObj.receivedMessages[0], 'base64'); - assert.deepEqual(msg, expectedMessageAsBase64); - }); + Subscription.prototype.listenForEvents_ = function() { + Subscription.prototype.listenForEvents_ = listenForEvents; + called = true; + }; - it('should decode buffer to specified encoding', function() { - var msg = Subscription - .formatMessage_(messageObj.receivedMessages[0], 'binary'); - assert.deepEqual(msg, expectedMessageAsBinary); + var subscription = new Subscription(PUBSUB, SUB_NAME); + assert(called); }); }); @@ -372,922 +221,1256 @@ describe('Subscription', function() { }); }); - describe('ack', function() { - it('should throw if no IDs are provided', function() { - assert.throws(function() { - subscription.ack(); - }, /At least one ID must be specified before it can be acknowledged\./); - assert.throws(function() { - subscription.ack([]); - }, /At least one ID must be specified before it can be acknowledged\./); - }); - - it('should accept a single id', function() { - assert.doesNotThrow(function() { - subscription.ack(1, util.noop); - }); - }); + describe('ack_', function() { + var MESSAGE = { + ackId: 'abc', + received_: 12345, + connectionId: 'def' + }; - it('should accept an array of ids', function() { - assert.doesNotThrow(function() { - subscription.ack([1], util.noop); - }); + beforeEach(function() { + subscription.breakLease_ = fakeUtil.noop; + subscription.histogram.add = fakeUtil.noop; }); - it('should make an array out of ids', function(done) { - var ID = 'abc'; - - subscription.parent.request = function(protoOpts, reqOpts) { - assert.deepEqual(reqOpts.ackIds, [ID]); + it('should break the lease on the message', function(done) { + subscription.breakLease_ = function(message) { + assert.strictEqual(message, MESSAGE); done(); }; - subscription.ack(ID, assert.ifError); + subscription.ack_(MESSAGE); }); - it('should make correct api request', function(done) { - var IDS = [1, 2, 3]; - - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'acknowledge'); - - assert.strictEqual(reqOpts.subscription, subscription.name); - assert.strictEqual(reqOpts.ackIds, IDS); - - done(); - }; - - subscription.ack(IDS, assert.ifError); - }); + it('should add the time it took to ack to the histogram', function(done) { + var fakeNow = 12381832; + var now = global.Date.now; - it('should honor the timeout setting', function(done) { - var options = { - timeout: 10 + global.Date.now = function() { + global.Date.now = now; + return fakeNow; }; - subscription.parent.request = function(protoOpts) { - assert.strictEqual(protoOpts.timeout, options.timeout); + subscription.histogram.add = function(time) { + assert.strictEqual(time, fakeNow - MESSAGE.received_); done(); }; - subscription.ack('abc', options, assert.ifError); + subscription.ack_(MESSAGE); }); - it('should not require a callback', function() { - assert.doesNotThrow(function() { - subscription.ack('abc'); - subscription.ack('abc', { - timeout: 10 - }); + describe('without connection pool', function() { + it('should store the ack id in the inventory object', function(done) { + subscription.setFlushTimeout_ = function() { + assert.deepEqual(subscription.inventory_.ack, [MESSAGE.ackId]); + done(); + }; + + subscription.ack_(MESSAGE); }); }); - it('should unmark the ack ids as being in progress', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); - }; - - subscription.inProgressAckIds = { id1: true, id2: true, id3: true }; + describe('with connection pool', function() { + beforeEach(function() { + subscription.setFlushTimeout_ = function() { + throw new Error('Should not be called.'); + }; + }); - subscription.ack(['id1', 'id2'], function(err) { - assert.ifError(err); + it('should write to the connection it came in on', function(done) { + var fakeConnection = { + write: function(data) { + assert.deepEqual(data, { ackIds: [MESSAGE.ackId] }); + done(); + } + }; - var inProgressAckIds = subscription.inProgressAckIds; - assert.strictEqual(inProgressAckIds.id1, undefined); - assert.strictEqual(inProgressAckIds.id2, undefined); - assert.strictEqual(inProgressAckIds.id3, true); + subscription.connectionPool = { + acquire: function(connectionId, callback) { + assert.strictEqual(connectionId, MESSAGE.connectionId); + callback(null, fakeConnection); + } + }; - done(); + subscription.ack_(MESSAGE); }); - }); - it('should not unmark if there was an error', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(new Error('Error.')); - }; + it('should emit an error when unable to get a connection', function(done) { + var error = new Error('err'); - subscription.inProgressAckIds = { id1: true, id2: true, id3: true }; + subscription.connectionPool = { + acquire: function(connectionId, callback) { + callback(error); + } + }; - subscription.ack(['id1', 'id2'], function() { - var inProgressAckIds = subscription.inProgressAckIds; - assert.strictEqual(inProgressAckIds.id1, true); - assert.strictEqual(inProgressAckIds.id2, true); - assert.strictEqual(inProgressAckIds.id3, true); + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); - done(); + subscription.ack_(MESSAGE); }); }); + }); - it('should refresh paused status', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); - }; - - subscription.refreshPausedStatus_ = done; + describe('breakLease_', function() { + var MESSAGE = { + ackId: 'abc', + data: new Buffer('hello') + }; - subscription.ack(1, assert.ifError); + beforeEach(function() { + subscription.inventory_.lease.push(MESSAGE.ackId); + subscription.inventory_.bytes += MESSAGE.data.length; }); - it('should pass error to callback', function(done) { - var error = new Error('Error.'); + it('should remove the message from the lease array', function() { + assert.strictEqual(subscription.inventory_.lease.length, 1); + assert.strictEqual(subscription.inventory_.bytes, MESSAGE.data.length); - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(error); - }; + subscription.breakLease_(MESSAGE); - subscription.ack(1, function(err) { - assert.strictEqual(err, error); - done(); - }); + assert.strictEqual(subscription.inventory_.lease.length, 0); + assert.strictEqual(subscription.inventory_.bytes, 0); }); - it('should pass apiResponse to callback', function(done) { - var resp = { success: true }; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); - }; - subscription.ack(1, function(err, apiResponse) { - assert.deepEqual(resp, apiResponse); - done(); - }); - }); - }); + describe('with connection pool', function() { + it('should resume receiving messages if paused', function(done) { + subscription.connectionPool = { + isPaused: true, + resume: done + }; - describe('createSnapshot', function() { - var SNAPSHOT_NAME = 'a'; + subscription.hasMaxMessages_ = function() { + return false; + }; - it('should throw if a name is not provided', function() { - assert.throws(function() { - subscription.createSnapshot(); - }, /A name is required to create a snapshot\./); - }); + subscription.breakLease_(MESSAGE); + }); - it('should make the correct api request', function(done) { - var FULL_SNAPSHOT_NAME = 'a/b/c/d'; + it('should not resume if it is not paused', function() { + subscription.connectionPool = { + isPaused: false, + resume: function() { + throw new Error('Should not be called.'); + } + }; - FakeSnapshot.formatName_ = function(projectId, name) { - assert.strictEqual(projectId, PROJECT_ID); - assert.strictEqual(name, SNAPSHOT_NAME); - return FULL_SNAPSHOT_NAME; - }; + subscription.hasMaxMessages_ = function() { + return false; + }; - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'createSnapshot'); + subscription.breakLease_(MESSAGE); + }); - assert.strictEqual(reqOpts.name, FULL_SNAPSHOT_NAME); - assert.strictEqual(reqOpts.subscription, subscription.name); + it('should not resume if the max message limit is hit', function() { + subscription.connectionPool = { + isPaused: true, + resume: function() { + throw new Error('Should not be called.'); + } + }; - done(); - }; + subscription.hasMaxMessages_ = function() { + return true; + }; - subscription.createSnapshot(SNAPSHOT_NAME, assert.ifError); + subscription.breakLease_(MESSAGE); + }); }); - it('should return an error to the callback', function(done) { - var error = new Error('err'); - var resp = {}; - - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(error, resp); - }; - - function callback(err, snapshot, apiResponse) { - assert.strictEqual(err, error); - assert.strictEqual(snapshot, null); - assert.strictEqual(apiResponse, resp); - done(); - } + it('should quit auto-leasing if all leases are gone', function(done) { + subscription.leaseTimeoutHandle_ = setTimeout(done, 1); + subscription.breakLease_(MESSAGE); - subscription.createSnapshot(SNAPSHOT_NAME, callback); + assert.strictEqual(subscription.leaseTimeoutHandle_, null); + setImmediate(done); }); - it('should return a snapshot object to the callback', function(done) { - var fakeSnapshot = {}; - var resp = {}; - - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); - }; - - subscription.snapshot = function(name) { - assert.strictEqual(name, SNAPSHOT_NAME); - return fakeSnapshot; - }; - - function callback(err, snapshot, apiResponse) { - assert.strictEqual(err, null); - assert.strictEqual(snapshot, fakeSnapshot); - assert.strictEqual(snapshot.metadata, resp); - assert.strictEqual(apiResponse, resp); - done(); - } + it('should continue to auto-lease if leases exist', function(done) { + subscription.inventory_.lease.push(MESSAGE.ackId); + subscription.inventory_.lease.push('abcd'); - subscription.createSnapshot(SNAPSHOT_NAME, callback); + subscription.leaseTimeoutHandle_ = setTimeout(done, 1); + subscription.breakLease_(MESSAGE); }); }); - describe('delete', function() { - it('should delete a subscription', function(done) { - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'deleteSubscription'); + describe('close', function() { + it('should set the userClosed_ flag', function() { + subscription.close(); - assert.strictEqual(reqOpts.subscription, subscription.name); + assert.strictEqual(subscription.userClosed_, true); + }); - done(); - }; + it('should stop auto-leasing', function(done) { + subscription.leaseTimeoutHandle_ = setTimeout(done, 1); + subscription.close(); - subscription.delete(); + assert.strictEqual(subscription.leaseTimeoutHandle_, null); + setImmediate(done); }); - it('should close a subscription once deleted', function() { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); - }; - subscription.closed = false; - subscription.delete(); - assert.strictEqual(subscription.closed, true); - }); + it('should stop any queued flushes', function(done) { + subscription.flushTimeoutHandle_ = setTimeout(done, 1); + subscription.close(); - it('should remove all listeners', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); - }; - subscription.removeAllListeners = function() { - done(); - }; - subscription.delete(); + assert.strictEqual(subscription.flushTimeoutHandle_, null); + setImmediate(done); }); - it('should execute callback when deleted', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); - }; - subscription.delete(done); + it('should flush immediately', function(done) { + subscription.flushQueues_ = done; + subscription.close(); }); - it('should execute callback with an api error', function(done) { - var error = new Error('Error.'); - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(error); + it('should call closeConnection_', function(done) { + subscription.closeConnection_ = function(callback) { + callback(); // the done fn }; - subscription.delete(function(err) { - assert.equal(err, error); - done(); - }); - }); - it('should execute callback with apiResponse', function(done) { - var resp = { success: true }; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); - }; - subscription.delete(function(err, apiResponse) { - assert.deepEqual(resp, apiResponse); - done(); - }); + subscription.close(done); }); }); - describe('pull', function() { - beforeEach(function() { - subscription.ack = util.noop; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, messageObj); - }; + describe('closeConnection_', function() { + afterEach(function() { + fakeUtil.noop = function() {}; }); - it('should not require configuration options', function(done) { - subscription.pull(done); - }); + describe('with connection pool', function() { + beforeEach(function() { + subscription.connectionPool = { + close: function(callback) { + setImmediate(callback); // the done fn + } + }; + }); - it('should default returnImmediately to false', function(done) { - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(reqOpts.returnImmediately, false); - done(); - }; - subscription.pull({}, assert.ifError); - }); + it('should call close on the connection pool', function(done) { + subscription.closeConnection_(done); + assert.strictEqual(subscription.connectionPool, null); + }); - it('should honor options', function(done) { - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(reqOpts.returnImmediately, true); - done(); - }; - subscription.pull({ returnImmediately: true }, assert.ifError); + it('should use a noop when callback is absent', function(done) { + fakeUtil.noop = done; + subscription.closeConnection_(done); + assert.strictEqual(subscription.connectionPool, null); + }); }); - it('should make correct api request', function(done) { - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'pull'); - assert.strictEqual(protoOpts.timeout, 92000); - - assert.strictEqual(reqOpts.subscription, subscription.name); - assert.strictEqual(reqOpts.returnImmediately, false); - assert.strictEqual(reqOpts.maxMessages, 1); + describe('without connection pool', function() { + beforeEach(function() { + subscription.connectionPool = null; + }); - done(); - }; + it('should exec the callback if one is passed in', function(done) { + subscription.closeConnection_(done); + }); - subscription.pull({ maxResults: 1 }, assert.ifError); + it('should optionally accept a callback', function() { + subscription.closeConnection_(); + }); }); + }); - it('should pass a timeout if specified', function(done) { - var timeout = 30000; - - var subscription = new Subscription(PUBSUB, { - name: SUB_NAME, - timeout: timeout - }); + describe('createSnapshot', function() { + var SNAPSHOT_NAME = 'test-snapshot'; - subscription.parent = { - request: function(protoOpts) { - assert.strictEqual(protoOpts.timeout, 30000); - done(); - } + beforeEach(function() { + subscription.snapshot = function(name) { + return { + name: name + }; }; - - subscription.pull(assert.ifError); }); - it('should store the active request', function() { - var requestInstance = {}; + it('should throw an error if a snapshot name is not found', function() { + assert.throws(function() { + subscription.createSnapshot(); + }, /A name is required to create a snapshot\./); + }); - subscription.parent.request = function() { - return requestInstance; + it('should make the correct request', function(done) { + subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'createSnapshot'); + assert.deepEqual(config.reqOpts, { + name: SNAPSHOT_NAME, + subscription: subscription.name + }); + done(); }; - subscription.pull(assert.ifError); - assert.strictEqual(subscription.activeRequest_, requestInstance); + subscription.createSnapshot(SNAPSHOT_NAME, assert.ifError); }); - it('should clear the active request', function(done) { - var requestInstance = {}; + it('should optionally accept gax options', function(done) { + var gaxOpts = {}; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - setImmediate(function() { - callback(null, {}); - assert.strictEqual(subscription.activeRequest_, null); - done(); - }); - - return requestInstance; + subscription.request = function(config) { + assert.strictEqual(config.gaxOpts, gaxOpts); + done(); }; - subscription.pull(assert.ifError); + subscription.createSnapshot(SNAPSHOT_NAME, gaxOpts, assert.ifError); }); - it('should pass error to callback', function(done) { - var error = new Error('Error.'); - subscription.parent.request = function(protoOpts, reqOpts, callback) { + it('should pass back any errors to the callback', function(done) { + var error = new Error('err'); + + subscription.request = function(config, callback) { callback(error); }; - subscription.pull(function(err) { - assert.equal(err, error); + + subscription.createSnapshot(SNAPSHOT_NAME, function(err) { + assert.strictEqual(err, error); done(); }); }); - it('should not return messages if request timed out', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback({ code: 504 }); + it('should return a snapshot object with metadata', function(done) { + var apiResponse = {}; + var fakeSnapshot = {}; + + subscription.snapshot = function() { + return fakeSnapshot; + }; + + subscription.request = function(config, callback) { + callback(null, apiResponse); }; - subscription.pull({}, function(err, messages) { + subscription.createSnapshot(SNAPSHOT_NAME, function(err, snapshot, resp) { assert.ifError(err); - assert.deepEqual(messages, []); + assert.strictEqual(snapshot, fakeSnapshot); + assert.strictEqual(snapshot.metadata, apiResponse); + assert.strictEqual(resp, apiResponse); done(); }); }); + }); - it('should call formatMessage_ with encoding', function(done) { - subscription.encoding = 'encoding-value'; - - formatMessageOverride = function(msg, encoding) { - assert.strictEqual(msg, messageObj.receivedMessages[0]); - assert.strictEqual(encoding, subscription.encoding); - setImmediate(done); - return msg; + describe('delete', function() { + it('should make the correct request', function(done) { + subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'deleteSubscription'); + assert.deepEqual(config.reqOpts, { subscription: subscription.name }); + done(); }; - subscription.pull({}, assert.ifError); + subscription.delete(assert.ifError); }); - it('should decorate the message', function(done) { - subscription.decorateMessage_ = function() { + it('should optionally accept gax options', function(done) { + var gaxOpts = {}; + + subscription.request = function(config) { + assert.strictEqual(config.gaxOpts, gaxOpts); done(); }; - subscription.pull({}, assert.ifError); + subscription.delete(gaxOpts, assert.ifError); }); - it('should refresh paused status', function(done) { - subscription.refreshPausedStatus_ = function() { - done(); - }; - - subscription.pull({}, assert.ifError); - }); + describe('success', function() { + var apiResponse = {}; - describe('autoAck false', function() { beforeEach(function() { - subscription.autoAck = false; + subscription.request = function(config, callback) { + callback(null, apiResponse); + }; }); - it('should not ack', function() { - subscription.ack = function() { - throw new Error('Should not have acked.'); - }; - subscription.pull({}, assert.ifError); + it('should return the api response', function(done) { + subscription.delete(function(err, resp) { + assert.ifError(err); + assert.strictEqual(resp, apiResponse); + done(); + }); }); - it('should execute callback with message', function(done) { - subscription.decorateMessage_ = function(msg) { return msg; }; - subscription.pull({}, function(err, msgs) { + it('should remove all message listeners', function(done) { + var called = false; + + subscription.removeAllListeners = function(name) { + assert.strictEqual(name, 'message'); + called = true; + }; + + subscription.delete(function(err) { assert.ifError(err); - assert.deepEqual(msgs, [expectedMessage]); + assert(called); done(); }); }); - it('should pass apiResponse to callback', function(done) { - subscription.pull(function(err, msgs, apiResponse) { + it('should close the subscription', function(done) { + var called = false; + + subscription.close = function() { + called = true; + }; + + subscription.delete(function(err) { assert.ifError(err); - assert.strictEqual(apiResponse, messageObj); + assert(called); done(); }); }); }); - describe('autoAck true', function() { + describe('error', function() { + var error = new Error('err'); + beforeEach(function() { - subscription.autoAck = true; - subscription.ack = function(id, callback) { - callback(); + subscription.request = function(config, callback) { + callback(error); }; }); - it('should ack', function(done) { - subscription.ack = function() { + it('should return the error to the callback', function(done) { + subscription.delete(function(err) { + assert.strictEqual(err, error); done(); - }; - subscription.pull({}, assert.ifError); + }); }); - it('should not autoAck if no messages returned', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, { receivedMessages: [] }); + it('should not remove all the listeners', function(done) { + subscription.removeAllListeners = function() { + done(new Error('Should not be called.')); }; - subscription.ack = function() { - throw new Error('I should not run.'); - }; - subscription.pull(function() { + + subscription.delete(function() { done(); }); }); - it('should pass id to ack', function(done) { - subscription.ack = function(id) { - assert.equal(id, expectedMessage.ackId); + it('should not close the subscription', function(done) { + subscription.close = function() { + done(new Error('Should not be called.')); + }; + + subscription.delete(function() { done(); + }); + }); + }); + }); + + describe('flushQueues_', function() { + beforeEach(function() { + subscription.inventory_.ack = ['abc', 'def']; + subscription.inventory_.nack = ['ghi', 'jkl']; + }); + + describe('with connection pool', function() { + var fakeConnection; + + beforeEach(function() { + fakeConnection = { + write: fakeUtil.noop + }; + + subscription.connectionPool = { + acquire: function(callback) { + callback(null, fakeConnection); + } }; - subscription.pull({}, assert.ifError); }); - it('should pass callback to ack', function(done) { - subscription.pull({}, done); + it('should do nothing if theres nothing to ack/nack', function() { + subscription.inventory_.ack = []; + subscription.inventory_.nack = []; + + subscription.connectionPool.acquire = function() { + throw new Error('Should not be called.'); + }; + + subscription.flushQueues_(); }); - it('should invoke callback with error from ack', function(done) { - var error = new Error('Error.'); - subscription.ack = function(id, callback) { + it('should emit any connection acquiring errors', function(done) { + var error = new Error('err'); + + subscription.connectionPool.acquire = function(callback) { callback(error); }; - subscription.pull({}, function(err) { - assert.equal(err, error); + + subscription.on('error', function(err) { + assert.strictEqual(err, error); done(); }); - }); - it('should execute callback', function(done) { - subscription.pull({}, done); + subscription.flushQueues_(); }); - it('should return pull response as apiResponse', function(done) { - var resp = { - receivedMessages: [{ - ackId: 1, - message: { - messageId: 'abc', - data: new Buffer('message').toString('base64') - } - }] + it('should write the acks to the connection', function(done) { + fakeConnection.write = function(reqOpts) { + assert.deepEqual(reqOpts.ackIds, ['abc', 'def']); + done(); }; - subscription.ack = function(id, callback) { - callback(null, { success: true }); - }; + subscription.flushQueues_(); + }); - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); + it('should write the nacks to the connection', function(done) { + fakeConnection.write = function(reqOpts) { + assert.deepEqual(reqOpts.modifyDeadlineAckIds, ['ghi', 'jkl']); + assert.deepEqual(reqOpts.modifyDeadlineSeconds, [0, 0]); + done(); }; - subscription.pull({}, function(err, msgs, apiResponse) { - assert.deepEqual(resp, apiResponse); - done(); - }); + subscription.flushQueues_(); }); - }); - }); - describe('seek', function() { - it('should throw if a name or date is not provided', function() { - assert.throws(function() { - subscription.seek(); - }, /Either a snapshot name or Date is needed to seek to\./); + it('should clear the inventory after writing', function() { + subscription.flushQueues_(); + + assert.strictEqual(subscription.inventory_.ack.length, 0); + assert.strictEqual(subscription.inventory_.nack.length, 0); + }); }); - it('should make the correct api request', function(done) { - var FAKE_SNAPSHOT_NAME = 'a'; - var FAKE_FULL_SNAPSHOT_NAME = 'a/b/c/d'; + describe('without connection pool', function() { + it('should do nothing if theres nothing to ack/nack', function() { + subscription.inventory_.ack = []; + subscription.inventory_.nack = []; - FakeSnapshot.formatName_ = function(projectId, name) { - assert.strictEqual(projectId, PROJECT_ID); - assert.strictEqual(name, FAKE_SNAPSHOT_NAME); - return FAKE_FULL_SNAPSHOT_NAME; - }; + subscription.request = function() { + throw new Error('Should not be called.'); + }; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'seek'); + subscription.flushQueues_(); + }); - assert.strictEqual(reqOpts.subscription, subscription.name); - assert.strictEqual(reqOpts.snapshot, FAKE_FULL_SNAPSHOT_NAME); + describe('acking', function() { + beforeEach(function() { + subscription.inventory_.nack = []; + }); - // done function - callback(); - }; + it('should make the correct request', function(done) { + subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'acknowledge'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + ackIds: ['abc', 'def'] + }); + done(); + }; - subscription.seek(FAKE_SNAPSHOT_NAME, done); - }); + subscription.flushQueues_(); + }); - it('should optionally accept a Date object', function(done) { - var date = new Date(); + it('should emit any request errors', function(done) { + var error = new Error('err'); - subscription.parent.request = function(protoOpts, reqOpts, callback) { - var seconds = Math.floor(date.getTime() / 1000); - assert.strictEqual(reqOpts.time.seconds, seconds); + subscription.request = function(config, callback) { + callback(error); + }; - var nanos = date.getMilliseconds() * 1e6; - assert.strictEqual(reqOpts.time.nanos, nanos); + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); - // done function - callback(); - }; + subscription.flushQueues_(); + }); - subscription.seek(date, done); - }); - }); + it('should clear the inventory on success', function(done) { + subscription.request = function(config, callback) { + callback(null); + assert.strictEqual(subscription.inventory_.ack.length, 0); + done(); + }; - describe('setAckDeadline', function() { - it('should set the ack deadline', function(done) { - subscription.parent.request = function(protoOpts, reqOpts) { - assert.strictEqual(protoOpts.service, 'Subscriber'); - assert.strictEqual(protoOpts.method, 'modifyAckDeadline'); + subscription.flushQueues_(); + }); + }); - assert.deepEqual(reqOpts, { - subscription: subscription.name, - ackIds: ['abc'], - ackDeadlineSeconds: 10 + describe('nacking', function() { + beforeEach(function() { + subscription.inventory_.ack = []; }); - done(); - }; + it('should make the correct request', function(done) { + subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'modifyAckDeadline'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + ackIds: ['ghi', 'jkl'], + ackDeadlineSeconds: 0 + }); + done(); + }; + + subscription.flushQueues_(); + }); + + it('should emit any request errors', function(done) { + var error = new Error('err'); + + subscription.request = function(config, callback) { + callback(error); + }; + + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + subscription.flushQueues_(); + }); - subscription.setAckDeadline({ ackIds: ['abc'], seconds: 10 }, done); + it('should clear the inventory on success', function(done) { + subscription.request = function(config, callback) { + callback(null); + assert.strictEqual(subscription.inventory_.nack.length, 0); + done(); + }; + + subscription.flushQueues_(); + }); + }); }); + }); - it('should execute the callback', function(done) { - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(); + describe('getMetadata', function() { + it('should make the correct request', function(done) { + subscription.request = function(config, callback) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'getSubscription'); + assert.deepEqual(config.reqOpts, { subscription: subscription.name }); + callback(); // the done fn }; - subscription.setAckDeadline({}, done); + + subscription.getMetadata(done); }); - it('should execute the callback with apiResponse', function(done) { - var resp = { success: true }; - subscription.parent.request = function(protoOpts, reqOpts, callback) { - callback(null, resp); + it('should optionally accept gax options', function(done) { + var gaxOpts = {}; + + subscription.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, gaxOpts); + callback(); // the done fn }; - subscription.setAckDeadline({}, function(err, apiResponse) { - assert.deepEqual(resp, apiResponse); - done(); - }); + + subscription.getMetadata(gaxOpts, done); }); }); - describe('snapshot', function() { - it('should call through to pubsub#snapshot', function() { - var FAKE_SNAPSHOT_NAME = 'a'; - var FAKE_SNAPSHOT = {}; + describe('hasMaxMessages_', function() { + it('should return true if the number of leases == maxMessages', function() { + subscription.inventory_.lease = ['a', 'b', 'c']; + subscription.flowControl.maxMessages = 3; - PUBSUB.snapshot = function(name) { - assert.strictEqual(this, subscription); - assert.strictEqual(name, FAKE_SNAPSHOT_NAME); - return FAKE_SNAPSHOT; - }; + assert(subscription.hasMaxMessages_()); + }); + + it('should return true if bytes == maxBytes', function() { + subscription.inventory_.bytes = 1000; + subscription.flowControl.maxBytes = 1000; + + assert(subscription.hasMaxMessages_()); + }); + + it('should return false if neither condition is met', function() { + subscription.inventory_.lease = ['a', 'b']; + subscription.flowControl.maxMessages = 3; + + subscription.inventory_.bytes = 900; + subscription.flowControl.maxBytes = 1000; - var snapshot = subscription.snapshot(FAKE_SNAPSHOT_NAME); - assert.strictEqual(snapshot, FAKE_SNAPSHOT); + assert.strictEqual(subscription.hasMaxMessages_(), false); }); }); - describe('decorateMessage_', function() { - var message = { - ackId: 'b' + describe('leaseMessage_', function() { + var MESSAGE = { + ackId: 'abc', + data: new Buffer('hello') }; + it('should add the ackId to the inventory', function() { + subscription.leaseMessage_(MESSAGE); + assert.deepEqual(subscription.inventory_.lease, [MESSAGE.ackId]); + }); + + it('should update the byte count', function() { + assert.strictEqual(subscription.inventory_.bytes, 0); + subscription.leaseMessage_(MESSAGE); + assert.strictEqual(subscription.inventory_.bytes, 5); + }); + + it('should begin auto-leasing', function(done) { + subscription.setLeaseTimeout_ = done; + subscription.leaseMessage_(MESSAGE); + }); + it('should return the message', function() { - var decoratedMessage = subscription.decorateMessage_(message); - assert.strictEqual(decoratedMessage.ackId, message.ackId); + var message = subscription.leaseMessage_(MESSAGE); + assert.strictEqual(message, MESSAGE); }); + }); - it('should mark the message as being in progress', function() { - subscription.decorateMessage_(message); - assert.strictEqual(subscription.inProgressAckIds[message.ackId], true); + describe('listenForEvents_', function() { + beforeEach(function() { + subscription.openConnection_ = fakeUtil.noop; + subscription.closeConnection_ = fakeUtil.noop; }); - describe('ack', function() { - it('should add an ack function to ack', function() { - var decoratedMessage = subscription.decorateMessage_(message); - assert.equal(typeof decoratedMessage.ack, 'function'); + describe('on new listener', function() { + it('should increment messageListeners', function() { + assert.strictEqual(subscription.messageListeners, 0); + subscription.on('message', fakeUtil.noop); + assert.strictEqual(subscription.messageListeners, 1); }); - it('should pass the ackId to subscription.ack', function(done) { - subscription.ack = function(ackId, callback) { - assert.strictEqual(ackId, message.ackId); - callback(); + it('should ignore non-message events', function() { + subscription.on('data', fakeUtil.noop); + assert.strictEqual(subscription.messageListeners, 0); + }); + + it('should open a connection', function(done) { + subscription.openConnection_ = done; + subscription.on('message', fakeUtil.noop); + }); + + it('should set the userClosed_ flag to false', function() { + subscription.userClosed_ = true; + subscription.on('message', fakeUtil.noop); + assert.strictEqual(subscription.userClosed_, false); + }); + + it('should not open a connection when one exists', function() { + subscription.connectionPool = {}; + + subscription.openConnection_ = function() { + throw new Error('Should not be called.'); }; - subscription.decorateMessage_(message).ack(done); + subscription.on('message', fakeUtil.noop); }); }); - describe('skip', function() { - it('should add a skip function', function() { - var decoratedMessage = subscription.decorateMessage_(message); - assert.equal(typeof decoratedMessage.skip, 'function'); + describe('on remove listener', function() { + var noop = function() {}; + + it('should decrement messageListeners', function() { + subscription.on('message', fakeUtil.noop); + subscription.on('message', noop); + assert.strictEqual(subscription.messageListeners, 2); + + subscription.removeListener('message', noop); + assert.strictEqual(subscription.messageListeners, 1); }); - it('should unmark the message as being in progress', function() { - subscription.decorateMessage_(message).skip(); + it('should ignore non-message events', function() { + subscription.on('message', fakeUtil.noop); + subscription.on('message', noop); + assert.strictEqual(subscription.messageListeners, 2); - var inProgressAckIds = subscription.inProgressAckIds; - assert.strictEqual(inProgressAckIds[message.ackId], undefined); + subscription.removeListener('data', noop); + assert.strictEqual(subscription.messageListeners, 2); }); - it('should refresh the paused status', function(done) { - subscription.refreshPausedStatus_ = done; - subscription.decorateMessage_(message).skip(); + it('should close the connection when no listeners', function(done) { + subscription.closeConnection_ = done; + + subscription.on('message', noop); + subscription.removeListener('message', noop); }); }); }); - describe('refreshPausedStatus_', function() { - it('should pause if the ackIds in progress is too high', function() { - subscription.inProgressAckIds = { id1: true, id2: true, id3: true }; - - subscription.maxInProgress = 2; - subscription.refreshPausedStatus_(); - assert.strictEqual(subscription.paused, true); + describe('modifyPushConfig', function() { + var fakeConfig = {}; - subscription.maxInProgress = 3; - subscription.refreshPausedStatus_(); - assert.strictEqual(subscription.paused, true); + it('should make the correct request', function(done) { + subscription.request = function(config, callback) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'modifyPushConfig'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + pushConfig: fakeConfig + }); + callback(); // the done fn + }; - subscription.maxInProgress = Infinity; - subscription.refreshPausedStatus_(); - assert.strictEqual(subscription.paused, false); + subscription.modifyPushConfig(fakeConfig, done); }); - it('should start pulling if paused and listeners exist', function(done) { - subscription.startPulling_ = done; + it('should optionally accept gaxOpts', function(done) { + var gaxOpts = {}; + + subscription.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, gaxOpts); + callback(); // the done fn + }; - subscription.inProgressAckIds = { id1: true, id2: true, id3: true }; - subscription.paused = true; - subscription.maxInProgress = Infinity; - subscription.messageListeners = 1; - subscription.refreshPausedStatus_(); + subscription.modifyPushConfig(fakeConfig, gaxOpts, done); }); }); - describe('listenForEvents_', function() { - afterEach(function() { - subscription.removeAllListeners(); + describe('nack_', function() { + var MESSAGE = { + ackId: 'abc', + received_: 12345, + connectionId: 'def' + }; + + beforeEach(function() { + subscription.breakLease_ = fakeUtil.noop; }); - it('should start pulling once a message listener is bound', function(done) { - subscription.startPulling_ = function() { + it('should break the lease on the message', function(done) { + subscription.breakLease_ = function(message) { + assert.strictEqual(message, MESSAGE); done(); }; - subscription.on('message', util.noop); + + subscription.nack_(MESSAGE); }); - it('should track the number of listeners', function() { - subscription.startPulling_ = util.noop; + describe('without connection pool', function() { + it('should store the ack id in the inventory object', function(done) { + subscription.setFlushTimeout_ = function() { + assert.deepEqual(subscription.inventory_.nack, [MESSAGE.ackId]); + done(); + }; - assert.strictEqual(subscription.messageListeners, 0); + subscription.nack_(MESSAGE); + }); + }); + + describe('with connection pool', function() { + beforeEach(function() { + subscription.setFlushTimeout_ = function() { + throw new Error('Should not be called.'); + }; + }); - subscription.on('message', util.noop); - assert.strictEqual(subscription.messageListeners, 1); + it('should write to the connection it came in on', function(done) { + var fakeConnection = { + write: function(data) { + assert.deepEqual(data, { + modifyDeadlineAckIds: [MESSAGE.ackId], + modifyDeadlineSeconds: [0] + }); + done(); + } + }; - subscription.removeListener('message', util.noop); - assert.strictEqual(subscription.messageListeners, 0); + subscription.connectionPool = { + acquire: function(connectionId, callback) { + assert.strictEqual(connectionId, MESSAGE.connectionId); + callback(null, fakeConnection); + } + }; + + subscription.nack_(MESSAGE); + }); + + it('should emit an error when unable to get a connection', function(done) { + var error = new Error('err'); + + subscription.connectionPool = { + acquire: function(connectionId, callback) { + callback(error); + } + }; + + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + subscription.nack_(MESSAGE); + }); }); + }); - it('should only run a single pulling loop', function() { - var startPullingCallCount = 0; + describe('openConnection_', function() { + it('should create a ConnectionPool instance', function() { + subscription.openConnection_(); + assert(subscription.connectionPool instanceof FakeConnectionPool); - subscription.startPulling_ = function() { - startPullingCallCount++; - }; + var args = subscription.connectionPool.calledWith_; + assert.strictEqual(args[0], subscription); + assert.deepEqual(args[1], { + ackDeadline: subscription.ackDeadline, + maxConnections: subscription.maxConnections + }); + }); + + it('should emit pool errors', function(done) { + var error = new Error('err'); - subscription.on('message', util.noop); - subscription.on('message', util.noop); + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); - assert.strictEqual(startPullingCallCount, 1); + subscription.openConnection_(); + subscription.connectionPool.emit('error', error); }); - it('should close when no more message listeners are bound', function() { - subscription.startPulling_ = util.noop; - subscription.on('message', util.noop); - subscription.on('message', util.noop); - // 2 listeners: sub should be open. - assert.strictEqual(subscription.closed, false); - subscription.removeListener('message', util.noop); - // 1 listener: sub should be open. - assert.strictEqual(subscription.closed, false); - subscription.removeListener('message', util.noop); - // 0 listeners: sub should be closed. - assert.strictEqual(subscription.closed, true); + it('should lease & emit messages from pool', function(done) { + var message = {}; + var leasedMessage = {}; + + subscription.leaseMessage_ = function(message_) { + assert.strictEqual(message_, message); + return leasedMessage; + }; + + subscription.on('message', function(message) { + assert.strictEqual(message, leasedMessage); + done(); + }); + + subscription.openConnection_(); + subscription.connectionPool.emit('message', message); }); - it('should abort the HTTP request when listeners removed', function(done) { - subscription.startPulling_ = util.noop; + it('should pause the pool if sub is at max messages', function(done) { + var message = {}; + var leasedMessage = {}; + + subscription.leaseMessage_ = function() { + return leasedMessage; + }; - subscription.activeRequest_ = { - abort: done + subscription.hasMaxMessages_ = function() { + return true; }; - subscription.on('message', util.noop); - subscription.removeAllListeners(); + subscription.openConnection_(); + subscription.connectionPool.pause = done; + subscription.connectionPool.emit('message', message); + }); + + it('should flush the queue when connected', function(done) { + subscription.flushQueues_ = function() { + assert.strictEqual(subscription.flushTimeoutHandle_, null); + done(); + }; + + subscription.flushTimeoutHandle_ = setTimeout(done, 1); + subscription.openConnection_(); + subscription.connectionPool.emit('connected'); }); }); - describe('startPulling_', function() { + describe('renewLeases_', function() { + var fakeDeadline = 9999; + beforeEach(function() { - subscription.pull = util.noop; + subscription.inventory_.lease = ['abc', 'def']; + + subscription.histogram.percentile = function() { + return fakeDeadline; + }; }); - it('should not pull if subscription is closed', function() { - subscription.pull = function() { - throw new Error('Should not be called.'); + it('should update the ackDeadline', function() { + subscription.request = subscription.setLeaseTimeout_ = fakeUtil.noop; + + subscription.histogram.percentile = function(percent) { + assert.strictEqual(percent, 99); + return fakeDeadline; }; - subscription.closed = true; - subscription.startPulling_(); + subscription.renewLeases_(); + assert.strictEqual(subscription.ackDeadline, fakeDeadline); }); - it('should not pull if subscription is paused', function() { - subscription.pull = function() { - throw new Error('Should not be called.'); + it('should set the auto-lease timeout', function(done) { + subscription.request = fakeUtil.noop; + subscription.setLeaseTimeout_ = done; + subscription.renewLeases_(); + }); + + describe('with connection pool', function() { + var fakeConnection; + + beforeEach(function() { + fakeConnection = { + acquire: fakeUtil.noop + }; + + subscription.connectionPool = { + acquire: function(callback) { + callback(null, fakeConnection); + } + }; + }); + + it('should not renew leases if inventory is empty', function() { + subscription.connectionPool.acquire = function() { + throw new Error('Should not have been called.'); + }; + + subscription.inventory_.lease = []; + subscription.renewLeases_(); + }); + + it('should emit any pool acquiring errors', function(done) { + var error = new Error('err'); + + subscription.connectionPool.acquire = function(callback) { + callback(error); + }; + + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + subscription.renewLeases_(); + }); + + it('should write to the connection', function(done) { + fakeConnection.write = function(reqOpts) { + assert.deepEqual(reqOpts, { + modifyDeadlineAckIds: ['abc', 'def'], + modifyDeadlineSeconds: Array(2).fill(fakeDeadline / 1000) + }); + done(); + }; + + subscription.renewLeases_(); + }); + }); + + describe('without connection pool', function() { + it('should make the correct request', function(done) { + subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'modifyAckDeadline'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + ackIds: ['abc', 'def'], + ackDeadlineSeconds: fakeDeadline / 1000 + }); + done(); + }; + + subscription.renewLeases_(); + }); + + it('should emit any request errors', function(done) { + var error = new Error('err'); + + subscription.request = function(config, callback) { + callback(error); + }; + + subscription.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + subscription.renewLeases_(); + }); + }); + }); + + describe('seek', function() { + var FAKE_SNAPSHOT_NAME = 'a'; + var FAKE_FULL_SNAPSHOT_NAME = 'a/b/c/d'; + + beforeEach(function() { + FakeSnapshot.formatName_ = function() { + return FAKE_FULL_SNAPSHOT_NAME; }; + }); - subscription.paused = true; - subscription.startPulling_(); + it('should throw if a name or date is not provided', function() { + assert.throws(function() { + subscription.seek(); + }, /Either a snapshot name or Date is needed to seek to\./); }); - it('should set returnImmediately to false when pulling', function(done) { - subscription.pull = function(options) { - assert.strictEqual(options.returnImmediately, false); - done(); + it('should make the correct api request', function(done) { + FakeSnapshot.formatName_ = function(projectId, name) { + assert.strictEqual(projectId, PROJECT_ID); + assert.strictEqual(name, FAKE_SNAPSHOT_NAME); + return FAKE_FULL_SNAPSHOT_NAME; + }; + + subscription.request = function(config, callback) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'seek'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + snapshot: FAKE_FULL_SNAPSHOT_NAME + }); + callback(); // the done fn }; - subscription.closed = false; - subscription.startPulling_(); + subscription.seek(FAKE_SNAPSHOT_NAME, done); }); - it('should not set maxResults if no maxInProgress is set', function(done) { - subscription.pull = function(options) { - assert.strictEqual(options.maxResults, undefined); - done(); + it('should optionally accept a Date object', function(done) { + var date = new Date(); + + subscription.request = function(config, callback) { + assert.strictEqual(config.reqOpts.time, date); + callback(); // the done fn }; - subscription.closed = false; - subscription.startPulling_(); + subscription.seek(date, done); }); - it('should set maxResults properly with maxInProgress', function(done) { - subscription.pull = function(options) { - assert.strictEqual(options.maxResults, 1); - done(); + it('should optionally accept gax options', function(done) { + var gaxOpts = {}; + + subscription.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, gaxOpts); + callback(); // the done fn }; - subscription.closed = false; - subscription.maxInProgress = 4; - subscription.inProgressAckIds = { id1: true, id2: true, id3: true }; - subscription.startPulling_(); + subscription.seek(FAKE_SNAPSHOT_NAME, gaxOpts, done); }); + }); - it('should emit an error event if one is encountered', function(done) { - var error = new Error('Error.'); - subscription.pull = function(options, callback) { - subscription.pull = function() {}; - setImmediate(function() { - callback(error); - }); + describe('setFlushTimeout_', function() { + var fakeTimeoutHandle = 1234; + var globalSetTimeout; + + before(function() { + globalSetTimeout = global.setTimeout; + }); + + after(function() { + global.setTimeout = globalSetTimeout; + }); + + it('should set a timeout to call flushQueues', function(done) { + subscription.flushQueues_ = done; + + global.setTimeout = function(callback, duration) { + assert.strictEqual(duration, 1000); + setImmediate(callback); // the done fn + return fakeTimeoutHandle; }; - subscription.closed = false; - subscription - .once('error', function(err) { - assert.equal(err, error); - done(); - }) - .startPulling_(); + subscription.setFlushTimeout_(); + assert.strictEqual(subscription.flushTimeoutHandle_, fakeTimeoutHandle); }); - it('should emit an error event with apiResponse', function(done) { - var error = new Error('Error.'); - var resp = { success: false }; - subscription.pull = function(options, callback) { - subscription.pull = function() {}; - setImmediate(function() { - callback(error, null, resp); - }); + it('should not set a timeout if one already exists', function() { + subscription.flushQueues_ = function() { + throw new Error('Should not be called.'); }; - subscription.closed = false; - subscription - .once('error', function(err, apiResponse) { - assert.equal(err, error); - assert.deepEqual(resp, apiResponse); - done(); - }) - .startPulling_(); + global.setTimeout = function() { + throw new Error('Should not be called.'); + }; + + subscription.flushTimeoutHandle_ = fakeTimeoutHandle; + subscription.setFlushTimeout_(); }); + }); - it('should emit a message event', function(done) { - subscription.pull = function(options, callback) { - callback(null, [{ hi: 'there' }]); + describe('setLeaseTimeout_', function() { + var fakeTimeoutHandle = 1234; + var fakeRandom = 2; + + var globalSetTimeout; + var globalMathRandom; + + before(function() { + globalSetTimeout = global.setTimeout; + globalMathRandom = global.Math.random; + }); + + after(function() { + global.setTimeout = globalSetTimeout; + global.Math.random = globalMathRandom; + }); + + it('should set a timeout to call renewLeases_', function(done) { + var ackDeadline = subscription.ackDeadline = 1000; + + global.Math.random = function() { + return fakeRandom; }; - subscription - .once('message', function(msg) { - assert.deepEqual(msg, { hi: 'there' }); - done(); - }); + global.setTimeout = function(callback, duration) { + assert.strictEqual(duration, fakeRandom * ackDeadline * 0.9); + setImmediate(callback); // the done fn + return fakeTimeoutHandle; + }; + + subscription.renewLeases_ = done; + subscription.setLeaseTimeout_(); + assert.strictEqual(subscription.leaseTimeoutHandle_, fakeTimeoutHandle); }); - it('should emit a message event with apiResponse', function(done) { - var resp = { success: true, msgs: [{ hi: 'there' }] }; - subscription.pull = function(options, callback) { - callback(null, [{ hi: 'there' }], resp); + it('should not set a timeout if one already exists', function() { + subscription.renewLeases_ = function() { + throw new Error('Should not be called.'); }; - subscription - .once('message', function(msg, apiResponse) { - assert.deepEqual(resp, apiResponse); - done(); + + global.Math.random = function() { + throw new Error('Should not be called.'); + }; + + global.setTimeout = function() { + throw new Error('Should not be called.'); + }; + + subscription.leaseTimeoutHandle_ = fakeTimeoutHandle; + subscription.setLeaseTimeout_(); + }); + }); + + describe('setMetadata', function() { + var METADATA = {}; + + it('should make the correct request', function(done) { + subscription.request = function(config, callback) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'updateSubscription'); + assert.deepEqual(config.reqOpts, { + subscription: subscription.name, + updateMask: METADATA }); + callback(); // the done fn + }; + + subscription.setMetadata(METADATA, done); }); - it('should pull at specified interval', function(done) { - var INTERVAL = 5; - subscription.pull = function(options, callback) { - assert.strictEqual(options.returnImmediately, false); - // After pull is called once, overwrite with `done`. - // This is to override the function passed to `setTimeout`, so we are - // sure it's the same pull function when we execute it. - subscription.pull = function() { - done(); - }; - callback(); + it('should optionally accept gax options', function(done) { + var gaxOpts = {}; + + subscription.request = function(config, callback) { + assert.strictEqual(config.gaxOpts, gaxOpts); + callback(); // the done fn }; - var setTimeout = global.setTimeout; - global.setTimeout = function(fn, interval) { - global.setTimeout = setTimeout; - assert.equal(interval, INTERVAL); - // This should execute the `done` function from when we overrided it - // above. - fn(); + + subscription.setMetadata(METADATA, gaxOpts, done); + }); + }); + + describe('snapshot', function() { + var SNAPSHOT_NAME = 'a'; + + it('should call through to pubsub.snapshot', function(done) { + PUBSUB.snapshot = function(name) { + assert.strictEqual(this, subscription); + assert.strictEqual(name, SNAPSHOT_NAME); + done(); }; - subscription.closed = false; - subscription.interval = INTERVAL; - subscription.startPulling_(); + subscription.snapshot(SNAPSHOT_NAME); }); }); }); diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index 444c19bfe79..abd8ba97a38 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -60,7 +60,7 @@ var fakePaginator = { } }; -describe.only('Topic', function() { +describe('Topic', function() { var Topic; var topic; From 2844bf7a63c4ac152599df6eab35638e3e15c49d Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 11:56:59 -0400 Subject: [PATCH 38/67] add unit tests for publisher.js --- packages/pubsub/src/publisher.js | 17 +- packages/pubsub/test/publisher.js | 318 +++++++++++++++++++++++++++ packages/pubsub/test/subscription.js | 2 +- 3 files changed, 326 insertions(+), 11 deletions(-) create mode 100644 packages/pubsub/test/publisher.js diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 8de1e2e8aba..651db59bcf6 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -39,24 +39,23 @@ function Publisher(topic, options) { }, options); this.topic = topic; - this.api = topic.api; // this object keeps track of all messages scheduled to be published // queued is essentially the `messages` field for the publish rpc req opts - // queuedBytes is used to track the size of the combined payload + // bytes is used to track the size of the combined payload // callbacks is an array of callbacks - each callback is associated with a // specific message. this.inventory_ = { callbacks: [], queued: [], - queuedBytes: 0 + bytes: 0 }; this.settings = { batching: { maxBytes: Math.min(options.batching.maxBytes, Math.pow(1024, 2) * 9), maxMessages: Math.min(options.batching.maxMessages, 1000), - maxMilliseconds: options.maxMilliseconds + maxMilliseconds: options.batching.maxMilliseconds } }; @@ -77,17 +76,15 @@ Publisher.prototype.publish = function(data, attrs, callback) { } var opts = this.settings.batching; - var newPayloadSize = this.inventory_.queueBytes + data.size; + var newPayloadSize = this.inventory_.bytes + data.length; // if this message puts us over the maxBytes option, then let's ship // what we have and add it to the next batch if (newPayloadSize > opts.maxBytes) { this.publish_(); - this.queue_(data, attrs, callback); - return; } - // haven't hit maxBytes? add it to the queue! + // add it to the queue! this.queue_(data, attrs, callback); // next lets check if this message brings us to the message cap or if we @@ -120,7 +117,7 @@ Publisher.prototype.publish_ = function() { this.inventory_.callbacks = []; this.inventory_.queued = []; - this.inventory_.queuedBytes = 0; + this.inventory_.bytes = 0; var reqOpts = { topic: this.topic.name, @@ -149,7 +146,7 @@ Publisher.prototype.queue_ = function(data, attrs, callback) { attributes: attrs }); - this.inventory_.queueBytes += data.size; + this.inventory_.bytes += data.length; this.inventory_.callbacks.push(callback); }; diff --git a/packages/pubsub/test/publisher.js b/packages/pubsub/test/publisher.js new file mode 100644 index 00000000000..d1ee6918767 --- /dev/null +++ b/packages/pubsub/test/publisher.js @@ -0,0 +1,318 @@ +/** + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var common = require('@google-cloud/common'); +var extend = require('extend'); +var proxyquire = require('proxyquire'); + +var promisified = false; +var fakeUtil = extend({}, common.util, { + promisifyAll: function(Class) { + if (Class.name === 'Publisher') { + promisified = true; + } + } +}); + +describe('Publisher', function() { + var Publisher; + var publisher; + var batchOpts; + + var TOPIC_NAME = 'test-topic'; + var TOPIC = { + name: TOPIC_NAME, + request: fakeUtil.noop + }; + + before(function() { + Publisher = proxyquire('../src/publisher.js', { + '@google-cloud/common': { + util: fakeUtil + } + }); + }); + + beforeEach(function() { + TOPIC.request = fakeUtil.noop; + publisher = new Publisher(TOPIC); + batchOpts = publisher.settings.batching; + }); + + describe('initialization', function() { + it('should promisify all the things', function() { + assert(promisified); + }); + + it('should localize the topic object', function() { + assert.strictEqual(publisher.topic, TOPIC); + }); + + it('should create an inventory object', function() { + assert.deepEqual(publisher.inventory_, { + callbacks: [], + queued: [], + bytes: 0 + }); + }); + + describe('options', function() { + it('should provide default values for batching', function() { + assert.deepEqual(publisher.settings.batching, { + maxBytes: Math.pow(1024, 2) * 5, + maxMessages: 1000, + maxMilliseconds: 1000 + }); + }); + + it('should capture user specified options', function() { + var options = { + maxBytes: 10, + maxMessages: 11, + maxMilliseconds: 12 + }; + + var publisher = new Publisher(TOPIC, { + batching: options + }); + + assert.deepEqual(publisher.settings.batching, options); + }); + + it('should cap maxBytes', function() { + var expected = Math.pow(1024, 2) * 9; + + var publisher = new Publisher(TOPIC, { + batching: { maxBytes: expected + 1024 } + }); + + assert.strictEqual(publisher.settings.batching.maxBytes, expected); + }); + + it('should cap maxMessages', function() { + var publisher = new Publisher(TOPIC, { + batching: { maxMessages: 2000 } + }); + + assert.strictEqual(publisher.settings.batching.maxMessages, 1000); + }); + }); + }); + + describe('publish', function() { + var DATA = new Buffer('hello'); + var ATTRS = { a: 'a' }; + + it('should throw an error when data is not a buffer', function() { + assert.throws(function() { + publisher.publish('hello', {}, fakeUtil.noop); + }, /Data must be in the form of a Buffer\./); + }); + + it('should queue the data', function(done) { + publisher.queue_ = function(data, attrs, callback) { + assert.strictEqual(data, DATA); + assert.strictEqual(attrs, ATTRS); + callback(); // the done fn + }; + + publisher.publish(DATA, ATTRS, done); + }); + + it('should optionally accept attributes', function(done) { + publisher.queue_ = function(data, attrs, callback) { + assert.strictEqual(data, DATA); + assert.deepEqual(attrs, {}); + callback(); // the done fn + }; + + publisher.publish(DATA, done); + }); + + it('should publish if data puts payload size over cap', function(done) { + var queueCalled = false; + + publisher.publish_ = function() { + assert.strictEqual(queueCalled, false); + publisher.inventory_.bytes = 0; + }; + + publisher.queue_ = function(data, attrs, callback) { + assert.strictEqual(publisher.inventory_.bytes, 0); + queueCalled = true; + callback(); // the done fn + }; + + publisher.inventory_.bytes = batchOpts.maxBytes - 1; + publisher.publish(DATA, done); + }); + + it('should publish if data puts payload at size cap', function(done) { + var queueCalled = false; + + publisher.queue_ = function() { + queueCalled = true; + }; + + publisher.publish_ = function() { + assert(queueCalled); + done(); + }; + + publisher.inventory_.bytes = batchOpts.maxBytes - DATA.length; + publisher.publish(DATA, fakeUtil.noop); + }); + + it('should publish if data puts payload at message cap', function(done) { + var queueCalled = false; + + publisher.queue_ = function() { + queueCalled = true; + }; + + publisher.publish_ = function() { + assert(queueCalled); + done(); + }; + + publisher.inventory_.queued = Array(batchOpts.maxMessages).fill({}); + publisher.publish(DATA, fakeUtil.noop); + }); + + it('should set a timeout if a publish did not occur', function(done) { + var globalSetTimeout = global.setTimeout; + var fakeTimeoutHandle = 12345; + + global.setTimeout = function(callback, duration) { + assert.strictEqual(duration, batchOpts.maxMilliseconds); + global.setTimeout = globalSetTimeout; + setImmediate(callback); + return fakeTimeoutHandle; + }; + + publisher.publish_ = done; + publisher.publish(DATA, fakeUtil.noop); + + assert.strictEqual(publisher.timeoutHandle_, fakeTimeoutHandle); + }); + + it('should not set a timeout if one exists', function() { + var fakeTimeoutHandle = 'not-a-real-handle'; + + publisher.timeoutHandle_ = 'not-a-real-handle'; + publisher.publish(DATA, fakeUtil.noop); + assert.strictEqual(publisher.timeoutHandle_, fakeTimeoutHandle); + }); + }); + + describe('publish_', function() { + it('should cancel any publish timeouts', function(done) { + publisher.timeoutHandle_ = setTimeout(done, 1); + publisher.publish_(); + assert.strictEqual(publisher.timeoutHandle_, null); + done(); + }); + + it('should reset the inventory object', function() { + publisher.inventory_.callbacks.push(fakeUtil.noop); + publisher.inventory_.queued.push({}); + publisher.inventory_.bytes = 5; + + publisher.publish_(); + + assert.deepEqual(publisher.inventory_.callbacks, []); + assert.deepEqual(publisher.inventory_.queued, []); + assert.strictEqual(publisher.inventory_.bytes, 0); + }); + + it('should make the correct request', function(done) { + var FAKE_MESSAGE = {}; + + TOPIC.request = function(config) { + assert.strictEqual(config.client, 'publisherClient'); + assert.strictEqual(config.method, 'publish'); + assert.deepEqual(config.reqOpts, { + topic: TOPIC_NAME, + messages: [FAKE_MESSAGE] + }); + done(); + }; + + publisher.inventory_.queued.push(FAKE_MESSAGE); + publisher.publish_(); + }); + + it('should pass back the err/msg id to correct callback', function(done) { + var error = new Error('err'); + var FAKE_IDS = ['abc', 'def']; + var callbackCalls = 0; + + publisher.inventory_.callbacks = [ + function(err, messageId) { + assert.strictEqual(err, error); + assert.strictEqual(messageId, FAKE_IDS[0]); + callbackCalls += 1; + }, + function(err, messageId) { + assert.strictEqual(err, error); + assert.strictEqual(messageId, FAKE_IDS[1]); + callbackCalls += 1; + }, + function(err, messageId) { + assert.strictEqual(err, error); + assert.strictEqual(messageId, undefined); + assert.strictEqual(callbackCalls, 2); + done(); + } + ]; + + TOPIC.request = function(config, callback) { + callback(error, { messageIds: FAKE_IDS }); + }; + + publisher.publish_(); + }); + }); + + describe('queue_', function() { + var DATA = new Buffer('hello'); + var ATTRS = { a: 'a' }; + + it('should add the data and attrs to the inventory', function() { + publisher.queue_(DATA, ATTRS, fakeUtil.noop); + + assert.deepEqual(publisher.inventory_.queued, [{ + data: DATA, + attributes: ATTRS + }]); + }); + + it('should update the inventory size', function() { + publisher.queue_(DATA, ATTRS, fakeUtil.noop); + + assert.strictEqual(publisher.inventory_.bytes, DATA.length); + }); + + it('should capture the callback', function() { + publisher.queue_(DATA, ATTRS, fakeUtil.noop); + + assert.deepEqual(publisher.inventory_.callbacks, [fakeUtil.noop]); + }); + }); +}); diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index 1651c266541..e1e7c1ba5fb 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -56,7 +56,7 @@ function FakeSnapshot() { this.calledWith_ = [].slice.call(arguments); } -describe.only('Subscription', function() { +describe('Subscription', function() { var Subscription; var subscription; From b116fbb6595b3d0b3a7491289bf88835f4154e05 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 12:25:17 -0400 Subject: [PATCH 39/67] moved message.js code into connection-pool --- packages/pubsub/src/connection-pool.js | 38 +++++++++++++---- packages/pubsub/src/message.js | 56 -------------------------- packages/pubsub/src/subscription.js | 3 +- packages/pubsub/system-test/pubsub.js | 2 +- packages/pubsub/test/subscription.js | 9 +++-- 5 files changed, 40 insertions(+), 68 deletions(-) delete mode 100644 packages/pubsub/src/message.js diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 80de2236aac..110e324d098 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -29,12 +29,6 @@ var is = require('is'); var util = require('util'); var uuid = require('uuid'); -/** - * @type {module:pubsub/message} - * @private - */ -var Message = require('./message.js'); - /** * */ @@ -133,7 +127,7 @@ ConnectionPool.prototype.createConnection = function() { connection.on('data', function(data) { arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', new Message(self.subscription, id, message)); + self.emit('message', self.createMessage_(id, message)); }); }); @@ -162,6 +156,36 @@ ConnectionPool.prototype.createConnection = function() { }); }; +/** + * + */ +ConnectionPool.prototype.createMessage_ = function(connectionId, resp) { + var self = this; + + var pt = resp.message.publishTime; + var milliseconds = parseInt(pt.nanos, 10) / 1e6; + + function ack() { + self.subscription.ack_(this); + } + + function nack() { + self.subscription.nack_(this); + } + + return { + connectionId: connectionId, + ackId: resp.ackId, + id: resp.message.messageId, + data: resp.message.data, + attributes: resp.message.attributes, + publishTime: new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds), + received: Date.now(), + ack: ack, + nack: nack + }; +}; + /** * */ diff --git a/packages/pubsub/src/message.js b/packages/pubsub/src/message.js deleted file mode 100644 index c796e4c0fec..00000000000 --- a/packages/pubsub/src/message.js +++ /dev/null @@ -1,56 +0,0 @@ -/*! - * Copyright 2017 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*! - * @module pubsub/message - */ - -'use strict'; - -/** - * - */ -function Message(subscription, connectionId, resp) { - this.subscription = subscription; - this.connectionId = connectionId; - - this.ackId = resp.ackId; - this.id = resp.message.messageId; - this.data = resp.message.data; - this.attrs = resp.message.attributes; - - var pt = resp.message.publishTime; - var milliseconds = parseInt(pt.nanos, 10) / 1e6; - - this.publishTime = new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds); - this.received_ = Date.now(); -} - -/** - * - */ -Message.prototype.ack = function() { - this.subscription.ack_(this); -}; - -/** - * - */ -Message.prototype.nack = function() { - this.subscription.nack_(this); -}; - -module.exports = Message; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index f7ec44df314..306f915e7f5 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -149,6 +149,7 @@ function Subscription(pubsub, name, options) { options = options || {}; this.pubsub = pubsub; + this.projectId = pubsub.projectId; this.request = pubsub.request.bind(pubsub); this.histogram = new Histogram(); @@ -242,7 +243,7 @@ Subscription.formatName_ = function(projectId, name) { */ Subscription.prototype.ack_ = function(message) { this.breakLease_(message); - this.histogram.add(Date.now() - message.received_); + this.histogram.add(Date.now() - message.received); if (!this.connectionPool) { this.inventory_.ack.push(message.ackId); diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 5cba5fa9ab9..e2dc942c728 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -176,7 +176,7 @@ describe('pubsub', function() { assert.ifError(err); assert.deepEqual(message.data, data); - assert.deepEqual(message.attrs, attrs); + assert.deepEqual(message.attributes, attrs); done(); }); diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index e1e7c1ba5fb..d746eb40d0a 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -95,6 +95,10 @@ describe('Subscription', function() { assert.strictEqual(subscription.pubsub, PUBSUB); }); + it('should localize the project id', function() { + assert.strictEqual(subscription.projectId, PROJECT_ID); + }); + it('should localize pubsub request method', function(done) { PUBSUB.request = function(callback) { callback(); // the done fn @@ -224,7 +228,7 @@ describe('Subscription', function() { describe('ack_', function() { var MESSAGE = { ackId: 'abc', - received_: 12345, + received: 12345, connectionId: 'def' }; @@ -252,7 +256,7 @@ describe('Subscription', function() { }; subscription.histogram.add = function(time) { - assert.strictEqual(time, fakeNow - MESSAGE.received_); + assert.strictEqual(time, fakeNow - MESSAGE.received); done(); }; @@ -1019,7 +1023,6 @@ describe('Subscription', function() { describe('nack_', function() { var MESSAGE = { ackId: 'abc', - received_: 12345, connectionId: 'def' }; From 017e0064f414c700670a17ef0b8d551b85291ca5 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 14:11:30 -0400 Subject: [PATCH 40/67] add tests for connection-pool.js --- packages/pubsub/src/connection-pool.js | 6 +- packages/pubsub/test/connection-pool.js | 494 ++++++++++++++++++++++++ 2 files changed, 497 insertions(+), 3 deletions(-) create mode 100644 packages/pubsub/test/connection-pool.js diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 110e324d098..dca32e83f04 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -127,7 +127,7 @@ ConnectionPool.prototype.createConnection = function() { connection.on('data', function(data) { arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', self.createMessage_(id, message)); + self.emit('message', self.createMessage(id, message)); }); }); @@ -139,7 +139,7 @@ ConnectionPool.prototype.createConnection = function() { self.connections.delete(id); if (self.isOpen) { - self.createConnection(id); + self.createConnection(); } }); @@ -159,7 +159,7 @@ ConnectionPool.prototype.createConnection = function() { /** * */ -ConnectionPool.prototype.createMessage_ = function(connectionId, resp) { +ConnectionPool.prototype.createMessage = function(connectionId, resp) { var self = this; var pt = resp.message.publishTime; diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js new file mode 100644 index 00000000000..4fa7a1db75e --- /dev/null +++ b/packages/pubsub/test/connection-pool.js @@ -0,0 +1,494 @@ +/** + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var common = require('@google-cloud/common'); +var events = require('events'); +var extend = require('extend'); +var proxyquire = require('proxyquire'); +var uuid = require('uuid'); + +var fakeUtil = extend({}, common.util); +var fakeUuid = extend({}, uuid); + +describe('ConnectionPool', function() { + var ConnectionPool; + var pool; + + var SUB_NAME = 'test-subscription'; + var SUBSCRIPTION = { + name: SUB_NAME, + request: fakeUtil.noop + }; + + before(function() { + ConnectionPool = proxyquire('../src/connection-pool.js', { + '@google-cloud/common': { + util: fakeUtil + }, + 'uuid': fakeUuid + }); + }); + + beforeEach(function() { + SUBSCRIPTION.request = fakeUtil.noop; + pool = new ConnectionPool(SUBSCRIPTION); + }); + + describe('initialization', function() { + it('should localize the subscription', function() { + assert.strictEqual(pool.subscription, SUBSCRIPTION); + }); + + it('should create a map for the connections', function() { + assert(pool.connections instanceof Map); + }); + + it('should set isPaused to false', function() { + assert.strictEqual(pool.isPaused, false); + }); + + it('should set isOpen to false', function() { + var open = ConnectionPool.prototype.open; + + ConnectionPool.prototype.open = function() { + ConnectionPool.prototype.open = open; + }; + + var pool = new ConnectionPool(SUBSCRIPTION); + assert.strictEqual(pool.isOpen, false); + }); + + it('should provide default settings', function() { + assert.strictEqual(pool.settings.maxConnections, 5); + assert.strictEqual(pool.settings.ackDeadline, 10000); + }); + + it('should respect user specified settings', function() { + var options = { + maxConnections: 10, + ackDeadline: 100 + }; + + var pool = new ConnectionPool(SUBSCRIPTION, options); + + assert.deepEqual(pool.settings, options); + }); + + it('should inherit from EventEmitter', function() { + assert(pool instanceof events.EventEmitter); + }); + + it('should call open', function(done) { + var open = ConnectionPool.prototype.open; + + ConnectionPool.prototype.open = function() { + ConnectionPool.prototype.open = open; + done(); + }; + + var pool = new ConnectionPool(SUBSCRIPTION); + }); + }); + + describe('acquire', function() { + it('should return an error if the pool is closed', function(done) { + pool.isOpen = false; + + pool.acquire(function(err) { + assert(err instanceof Error); + assert.strictEqual(err.message, 'Connection pool is closed.'); + done(); + }); + }); + + it('should return a specified connection', function(done) { + var id = 'a'; + var fakeConnection = {}; + + pool.connections.set(id, fakeConnection); + pool.connections.set('b', {}); + + pool.acquire(id, function(err, connection) { + assert.ifError(err); + assert.strictEqual(connection, fakeConnection); + done(); + }); + }); + + it('should return any conn when the specified is missing', function(done) { + var fakeConnection = {}; + + pool.connections.set('a', fakeConnection); + + pool.acquire('b', function(err, connection) { + assert.ifError(err); + assert.strictEqual(connection, fakeConnection); + done(); + }); + }); + + it('should return any connection when id is missing', function(done) { + var fakeConnection = {}; + + pool.connections.set('a', fakeConnection); + + pool.acquire(function(err, connection) { + assert.ifError(err); + assert.strictEqual(connection, fakeConnection); + done(); + }); + }); + + it('should listen for connected event if no conn is ready', function(done) { + var fakeConnection = {}; + + pool.acquire(function(err, connection) { + assert.ifError(err); + assert.strictEqual(connection, fakeConnection); + done(); + }); + + pool.emit('connected', fakeConnection); + }); + }); + + describe('close',function() { + it('should set isOpen to false', function() { + pool.close(); + assert.strictEqual(pool.isOpen, false); + }); + + it('should call end on all active connections', function() { + function FakeConnection() { + this.endCalled = false; + } + + FakeConnection.prototype.end = function(cb) { + this.endCalled = true; + cb(); + }; + + var a = new FakeConnection(); + var b = new FakeConnection(); + + pool.connections.set('a', a); + pool.connections.set('b', b); + + pool.close(); + + assert.strictEqual(a.endCalled, true); + assert.strictEqual(b.endCalled, true); + }); + + it('should exec a callback when finished closing', function(done) { + pool.close(done); + }); + + it('should use noop when callback is omitted', function(done) { + fakeUtil.noop = function() { + fakeUtil.noop = function() {}; + done(); + }; + + pool.close(); + }); + }); + + describe('createConnection', function() { + it('should make the correct request', function(done) { + pool.subscription.request = function(config) { + assert.strictEqual(config.client, 'subscriberClient'); + assert.strictEqual(config.method, 'streamingPull'); + assert(config.returnFn); + done(); + }; + + pool.createConnection(); + }); + + it('should emit any error that occurs', function(done) { + var error = new Error('err'); + + pool.subscription.request = function(config, callback) { + callback(error); + }; + + pool.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + pool.createConnection(); + }); + + describe('connection', function() { + var fakeConnection; + var fakeId; + + beforeEach(function() { + fakeConnection = new events.EventEmitter(); + fakeConnection.write = fakeUtil.noop; + + fakeId = uuid.v4(); + + fakeUuid.v4 = function() { + return fakeId; + }; + + pool.subscription.request = function(config, callback) { + callback(null, function() { + return fakeConnection; + }); + }; + }); + + it('should create a connection', function(done) { + fakeConnection.write = function(reqOpts) { + assert.deepEqual(reqOpts, { + subscription: SUB_NAME, + streamAckDeadlineSeconds: pool.settings.ackDeadline / 1000 + }); + }; + + pool.connections.set = function(id, connection) { + assert.strictEqual(id, fakeId); + assert.strictEqual(connection, fakeConnection); + done(); + }; + + pool.createConnection(); + }); + + it('should emit errors to the pool', function(done) { + var error = new Error('err'); + + pool.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('error', error); + }); + + it('should emit messages', function(done) { + var fakeResp = {}; + var fakeMessage = {}; + + pool.createMessage = function(id, resp) { + assert.strictEqual(id, fakeId); + assert.strictEqual(resp, fakeResp); + return fakeMessage; + }; + + pool.on('message', function(message) { + assert.strictEqual(message, fakeMessage); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('data', { receivedMessages: [fakeResp] }); + }); + + it('should emit connected when ready', function(done) { + pool.on('connected', function(connection) { + assert.strictEqual(connection, fakeConnection); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('metadata'); + }); + + it('should create a new connection if the pool is open', function(done) { + var deleted = false; + + pool.connections.delete = function(id) { + assert.strictEqual(id, fakeId); + deleted = true; + }; + + pool.createConnection(); + + pool.createConnection = function() { + assert(deleted); + done(); + }; + + pool.isOpen = true; + fakeConnection.emit('close'); + }); + + it('should not create a conn if the pool is closed', function(done) { + pool.createConnection(); + + pool.connections.delete = function() { + done(); + }; + + pool.createConnection = done; // should not be called + + pool.isOpen = false; + fakeConnection.emit('close'); + }); + + it('should pause the connection if the pool is paused', function(done) { + fakeConnection.pause = done; + pool.isPaused = true; + pool.createConnection(); + }); + }); + }); + + describe('createMessage', function() { + var message; + + var CONNECTION_ID = 'abc'; + + var PT = { + seconds: 6838383, + nanos: 20323838 + }; + + var RESP = { + ackId: 'def', + message: { + messageId: 'ghi', + data: new Buffer('hello'), + attributes: { + a: 'a' + }, + publishTime: PT + } + }; + + beforeEach(function() { + message = pool.createMessage(CONNECTION_ID, RESP); + }); + + it('should capture the connection id', function() { + assert.strictEqual(message.connectionId, CONNECTION_ID); + }); + + it('should capture the message data', function() { + var expectedPublishTime = new Date( + parseInt(PT.seconds, 10) * 1000 + parseInt(PT.nanos, 10) / 1e6); + var dateNowValue = Date.now(); + + assert.strictEqual(message.ackId, RESP.ackId); + assert.strictEqual(message.id, RESP.message.messageId); + assert.strictEqual(message.data, RESP.message.data); + assert.strictEqual(message.attributes, RESP.message.attributes); + assert.deepEqual(message.publishTime, expectedPublishTime); + assert.strictEqual(message.received, dateNowValue); + }); + + it('should create an ack method', function(done) { + SUBSCRIPTION.ack_ = function(message_) { + assert.strictEqual(message_, message); + done(); + }; + + message.ack(); + }); + + it('should create a nack method', function(done) { + SUBSCRIPTION.nack_ = function(message_) { + assert.strictEqual(message_, message); + done(); + }; + + message.nack(); + }); + }); + + describe('open', function() { + it('should make the specified number of connections', function() { + var connectionCount = 0; + + pool.createConnection = function() { + connectionCount += 1; + }; + + pool.open(); + assert.strictEqual(pool.settings.maxConnections, connectionCount); + }); + + it('should set the isOpen flag to true', function() { + pool.open(); + assert(pool.isOpen); + }); + }); + + describe('pause', function() { + it('should set the isPaused flag to true', function() { + pool.pause(); + assert(pool.isPaused); + }); + + it('should pause all the connections', function() { + function FakeConnection() { + this.isPaused = false; + } + + FakeConnection.prototype.pause = function() { + this.isPaused = true; + }; + + var a = new FakeConnection(); + var b = new FakeConnection(); + + pool.connections.set('a', a); + pool.connections.set('b', b); + + pool.pause(); + + assert(a.isPaused); + assert(b.isPaused); + }); + }); + + describe('resume', function() { + it('should set the isPaused flag to false', function() { + pool.resume(); + assert.strictEqual(pool.isPaused, false); + }); + + it('should resume all the connections', function() { + function FakeConnection() { + this.isPaused = true; + } + + FakeConnection.prototype.resume = function() { + this.isPaused = false; + }; + + var a = new FakeConnection(); + var b = new FakeConnection(); + + pool.connections.set('a', a); + pool.connections.set('b', b); + + pool.resume(); + + assert.strictEqual(a.isPaused, false); + assert.strictEqual(b.isPaused, false); + }); + }); +}); From f2dfd09cc9a77695c8806be737025c6497ca14a0 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 14:50:01 -0400 Subject: [PATCH 41/67] add unit tests for histogram.js --- packages/pubsub/test/histogram.js | 120 ++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 packages/pubsub/test/histogram.js diff --git a/packages/pubsub/test/histogram.js b/packages/pubsub/test/histogram.js new file mode 100644 index 00000000000..30afcc854e6 --- /dev/null +++ b/packages/pubsub/test/histogram.js @@ -0,0 +1,120 @@ +/** + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); + +var Histogram = require('../src/histogram.js'); + +describe('Histogram', function() { + var histogram; + + var MIN_VALUE = 10000; + var MAX_VALUE = 600000; + + beforeEach(function() { + histogram = new Histogram(); + }); + + describe('initialization', function() { + it('should create a data map', function() { + assert(histogram.data instanceof Map); + }); + + it('should set the initial length to 0', function() { + assert.strictEqual(histogram.length, 0); + }); + }); + + describe('add', function() { + it('increment a value', function() { + histogram.data.set(MIN_VALUE, 1); + histogram.add(MIN_VALUE); + + assert.strictEqual(histogram.data.get(MIN_VALUE), 2); + }); + + it('should initialize a value if absent', function() { + histogram.add(MIN_VALUE); + + assert.strictEqual(histogram.data.get(MIN_VALUE), 1); + }); + + it('should adjust the length for each item added', function() { + histogram.add(MIN_VALUE); + histogram.add(MIN_VALUE); + histogram.add(MIN_VALUE * 2); + + assert.strictEqual(histogram.length, 3); + }); + + it('should cap the value', function() { + var outOfBounds = MAX_VALUE + MIN_VALUE; + + histogram.add(outOfBounds); + + assert.strictEqual(histogram.data.get(outOfBounds), undefined); + assert.strictEqual(histogram.data.get(MAX_VALUE), 1); + }); + + it('should apply a minimum', function() { + var outOfBounds = MIN_VALUE - 1000; + + histogram.add(outOfBounds); + + assert.strictEqual(histogram.data.get(outOfBounds), undefined); + assert.strictEqual(histogram.data.get(MIN_VALUE), 1); + }); + + it('should use seconds level precision', function() { + var ms = 303823; + var expected = 304000; + + histogram.add(ms); + + assert.strictEqual(histogram.data.get(ms), undefined); + assert.strictEqual(histogram.data.get(expected), 1); + }); + }); + + describe('percentile', function() { + function range(a, b) { + var result = []; + + for (; a < b; a++) { + result.push(a); + } + + return result; + } + + it('should return the nth percentile', function() { + range(100, 201).forEach(function(value) { + histogram.add(value * 1000); + }); + + assert.strictEqual(histogram.percentile(100), 200000); + assert.strictEqual(histogram.percentile(101), 200000); + assert.strictEqual(histogram.percentile(99), 199000); + assert.strictEqual(histogram.percentile(1), 101000); + }); + + it('should return the min value if unable to determine', function() { + assert.strictEqual(histogram.percentile(99), MIN_VALUE); + }); + }); +}); From 88e75e1d05f6bb282f2cc72f089bc85cfacf53f5 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 28 Jul 2017 15:10:27 -0400 Subject: [PATCH 42/67] removed unused deps & lint --- packages/pubsub/package.json | 4 ---- packages/pubsub/src/connection-pool.js | 16 ++++++---------- packages/pubsub/src/index.js | 4 +--- packages/pubsub/src/publisher.js | 9 +++------ packages/pubsub/src/subscription.js | 6 ++---- packages/pubsub/src/topic.js | 2 +- packages/pubsub/system-test/pubsub.js | 3 +-- packages/pubsub/test/connection-pool.js | 4 ++-- packages/pubsub/test/index.js | 25 ++++++++++++------------- packages/pubsub/test/subscription.js | 6 +++--- packages/pubsub/test/topic.js | 1 - 11 files changed, 31 insertions(+), 49 deletions(-) diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index 0d1233acf3c..7f71b150193 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -59,10 +59,6 @@ "google-gax": "^0.13.0", "google-proto-files": "^0.12.0", "is": "^3.0.1", - "modelo": "^4.2.0", - "propprop": "^0.3.1", - "stream-events": "^1.0.2", - "through2": "^2.0.3", "uuid": "^3.0.1" }, "devDependencies": { diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index dca32e83f04..fb6b7bcd667 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -165,14 +165,6 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { var pt = resp.message.publishTime; var milliseconds = parseInt(pt.nanos, 10) / 1e6; - function ack() { - self.subscription.ack_(this); - } - - function nack() { - self.subscription.nack_(this); - } - return { connectionId: connectionId, ackId: resp.ackId, @@ -181,8 +173,12 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { attributes: resp.message.attributes, publishTime: new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds), received: Date.now(), - ack: ack, - nack: nack + ack: function() { + self.subscription.ack_(this); + }, + nack: function() { + self.subscription.nack_(this); + } }; }; diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index b33295553fe..ddd13a3d011 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -24,8 +24,6 @@ var common = require('@google-cloud/common'); var extend = require('extend'); var googleAuth = require('google-auto-auth'); var is = require('is'); -var streamEvents = require('stream-events'); -var through = require('through2'); var v1 = require('./v1'); @@ -603,7 +601,7 @@ PubSub.prototype.getTopics = function(options, callback) { project: 'projects/' + this.projectId }, options); - delete reqOpts.gaxOpts + delete reqOpts.gaxOpts; this.request({ client: 'publisherClient', diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 651db59bcf6..6692eb6eae2 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -24,7 +24,6 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); var extend = require('extend'); var is = require('is'); -var prop = require('propprop'); /** * @@ -107,11 +106,6 @@ Publisher.prototype.publish = function(data, attrs, callback) { * This should never be called directly. */ Publisher.prototype.publish_ = function() { - var self = this; - - clearTimeout(this.timeoutHandle_); - this.timeoutHandle_ = null; - var callbacks = this.inventory_.callbacks; var messages = this.inventory_.queued; @@ -119,6 +113,9 @@ Publisher.prototype.publish_ = function() { this.inventory_.queued = []; this.inventory_.bytes = 0; + clearTimeout(this.timeoutHandle_); + this.timeoutHandle_ = null; + var reqOpts = { topic: this.topic.name, messages: messages diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 306f915e7f5..6d29a3c5c3e 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -20,13 +20,11 @@ 'use strict'; -var arrify = require('arrify'); var common = require('@google-cloud/common'); var events = require('events'); var extend = require('extend'); var is = require('is'); var os = require('os'); -var prop = require('propprop'); var util = require('util'); /** @@ -282,7 +280,7 @@ Subscription.prototype.breakLease_ = function(message) { clearTimeout(this.leaseTimeoutHandle_); this.leaseTimeoutHandle_ = null; } -} +}; /** * @@ -625,7 +623,7 @@ Subscription.prototype.nack_ = function(message) { modifyDeadlineSeconds: [0] }); }); -} +}; /** * @private diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index 27169e46561..e0e9d170818 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -50,7 +50,7 @@ var Publisher = require('./publisher.js'); * @example * var topic = pubsub.topic('my-topic'); */ -function Topic(pubsub, name, options) { +function Topic(pubsub, name) { this.name = Topic.formatName_(pubsub.projectId, name); this.pubsub = pubsub; this.request = pubsub.request.bind(pubsub); diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index e2dc942c728..284009db8c6 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -196,7 +196,6 @@ describe('pubsub', function() { var TOPIC_NAME = generateTopicName(); var topic = pubsub.topic(TOPIC_NAME); var publisher = topic.publisher(); - var subscription; var SUB_NAMES = [ generateSubName(), @@ -464,7 +463,7 @@ describe('pubsub', function() { return new Promise(function(resolve) { setTimeout(resolve, milliseconds); }); - } + }; } before(function() { diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index 4fa7a1db75e..47a5a5487b0 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -41,7 +41,7 @@ describe('ConnectionPool', function() { '@google-cloud/common': { util: fakeUtil }, - 'uuid': fakeUuid + uuid: fakeUuid }); }); @@ -102,7 +102,7 @@ describe('ConnectionPool', function() { done(); }; - var pool = new ConnectionPool(SUBSCRIPTION); + new ConnectionPool(SUBSCRIPTION); }); }); diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 13f26659841..3eac76466f9 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -19,7 +19,6 @@ var arrify = require('arrify'); var assert = require('assert'); var extend = require('extend'); -var path = require('path'); var proxyquire = require('proxyquire'); var util = require('@google-cloud/common').util; @@ -186,7 +185,7 @@ describe('PubSub', function() { called = true; }; - var pubsub = new PubSub({}); + new PubSub({}); assert(called); }); @@ -358,7 +357,7 @@ describe('PubSub', function() { }; }; - pubsub.request = function(config, callback) { + pubsub.request = function(config) { assert.notStrictEqual(config.reqOpts, options); assert.deepEqual(config.reqOpts, expectedBody); done(); @@ -411,9 +410,9 @@ describe('PubSub', function() { callback({ code: 6 }, apiResponse); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, subscription) { + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, sub) { assert.ifError(err); - assert.strictEqual(subscription, SUBSCRIPTION); + assert.strictEqual(sub, SUBSCRIPTION); done(); }); }); @@ -423,12 +422,14 @@ describe('PubSub', function() { callback(error, apiResponse); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { + function callback(err, sub, resp) { assert.strictEqual(err, error); assert.strictEqual(sub, null); assert.strictEqual(resp, apiResponse); done(); - }); + } + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, callback); }); }); @@ -452,12 +453,14 @@ describe('PubSub', function() { callback(null, apiResponse); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, function(err, sub, resp) { + function callback(err, sub, resp) { assert.ifError(err); assert.strictEqual(sub, subscription); assert.strictEqual(resp, apiResponse); done(); - }); + } + + pubsub.createSubscription(TOPIC_NAME, SUB_NAME, callback); }); }); }); @@ -571,8 +574,6 @@ describe('PubSub', function() { }); it('should remove slashes from the baseUrl', function() { - var expectedBaseUrl = 'localhost:8080'; - setHost('localhost:8080/'); pubsub.determineBaseUrl_(); assert.strictEqual(pubsub.options.servicePath, 'localhost'); @@ -630,7 +631,6 @@ describe('PubSub', function() { it('should build the right request', function(done) { var options = { a: 'b', c: 'd', gaxOpts: {} }; - var originalOptions = extend({}, options); var expectedOptions = extend({}, options, { project: 'projects/' + pubsub.projectId }); @@ -777,7 +777,6 @@ describe('PubSub', function() { it('should build the right request', function(done) { var options = { a: 'b', c: 'd', gaxOpts: {} }; - var originalOptions = extend({}, options); var expectedOptions = extend({}, options, { project: 'projects/' + pubsub.projectId }); diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index d746eb40d0a..43a64dccddb 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -208,7 +208,7 @@ describe('Subscription', function() { called = true; }; - var subscription = new Subscription(PUBSUB, SUB_NAME); + new Subscription(PUBSUB, SUB_NAME); assert(called); }); }); @@ -299,7 +299,7 @@ describe('Subscription', function() { subscription.ack_(MESSAGE); }); - it('should emit an error when unable to get a connection', function(done) { + it('should emit an error when unable to get a conn', function(done) { var error = new Error('err'); subscription.connectionPool = { @@ -1078,7 +1078,7 @@ describe('Subscription', function() { subscription.nack_(MESSAGE); }); - it('should emit an error when unable to get a connection', function(done) { + it('should emit an error when unable to get a conn', function(done) { var error = new Error('err'); subscription.connectionPool = { diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index abd8ba97a38..6ea63beb372 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -18,7 +18,6 @@ var assert = require('assert'); var extend = require('extend'); -var nodeutil = require('util'); var proxyquire = require('proxyquire'); var util = require('@google-cloud/common').util; From 768bb8691528a0cf472c08f8b7e52c0eeff3eefa Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Sun, 30 Jul 2017 21:27:44 -0400 Subject: [PATCH 43/67] update top level object documentation --- packages/pubsub/src/index.js | 128 ++++++++++------------------------ packages/pubsub/test/index.js | 6 +- 2 files changed, 40 insertions(+), 94 deletions(-) diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index ddd13a3d011..fefd22bed35 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -81,21 +81,28 @@ function PubSub(options) { /** * Create a subscription to a topic. * - * All generated subscription names share a common prefix, `autogenerated-`. - * * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} * * @throws {Error} If a Topic instance or topic name is not provided. - * @throws {Error} If a subName is not provided. + * @throws {Error} If a subscription name is not provided. * * @param {module:pubsub/topic|string} topic - The Topic to create a * subscription to. - * @param {string=} subName - The name of the subscription. If a name is not - * provided, a random subscription name will be generated and created. + * @param {string=} name - The name of the subscription. * @param {object=} options - See a * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadlineSeconds - The maximum time after receiving - * a message that you must ack a message before it is redelivered. + * @param {number} options.ackDeadline - The maximum time after receiving a + * message that you must ack a message before it is redelivered. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number|date} options.messageRetentionDuration - Set this to override * the default duration of 7 days. This value is expected in seconds. * Acceptable values are in the range of 10 minutes and 7 days. @@ -121,18 +128,10 @@ function PubSub(options) { * pubsub.createSubscription(topic, name, callback); * * //- - * // Omit the name to have one generated automatically. All generated names - * // share a common prefix, `autogenerated-`. - * //- - * pubsub.createSubscription(topic, function(err, subscription, apiResponse) { - * // subscription.name = The generated name. - * }); - * - * //- * // Customize the subscription. * //- * pubsub.createSubscription(topic, name, { - * ackDeadlineSeconds: 90 + * ackDeadline: 90000 // 90 seconds * }, callback); * * //- @@ -172,6 +171,11 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { delete reqOpts.gaxOpts; + if (options.ackDeadline) { + reqOpts.ackDeadlineSeconds = options.ackDeadline / 1000; + delete reqOpts.ackDeadline; + } + if (options.messageRetentionDuration) { reqOpts.retainAckedMessages = true; @@ -211,6 +215,8 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { * @resource [Topics: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/create} * * @param {string} name - Name of the topic. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function=} callback - The callback function. * @param {?error} callback.err - An error from the API call, may be null. * @param {module:pubsub/topic} callback.topic - The newly created topic. @@ -292,10 +298,8 @@ PubSub.prototype.determineBaseUrl_ = function() { * Get a list of snapshots. * * @param {object=} options - Configuration object. - * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: true. - * @param {number} options.maxApiCalls - Maximum number of API calls to make. - * @param {number} options.maxResults - Maximum number of results to return. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. * @param {string} options.pageToken - Page token. * @param {function} callback - The callback function. @@ -311,21 +315,6 @@ PubSub.prototype.determineBaseUrl_ = function() { * }); * * //- - * // To control how many API requests are made and page through the results - * // manually, set `autoPaginate` to `false`. - * //- - * var callback = function(err, snapshots, nextQuery, apiResponse) { - * if (nextQuery) { - * // More results exist. - * pubsub.getSnapshots(nextQuery, callback); - * } - * }; - * - * pubsub.getSnapshots({ - * autoPaginate: false - * }, callback); - * - * //- * // If the callback is omitted, we'll return a Promise. * //- * pubsub.getSnapshots().then(function(data) { @@ -409,10 +398,8 @@ PubSub.prototype.getSnapshotsStream = * @resource [Subscriptions: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/list} * * @param {object=} options - Configuration object. - * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: true. - * @param {number} options.maxApiCalls - Maximum number of API calls to make. - * @param {number} options.maxResults - Maximum number of results to return. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. * @param {string} options.pageToken - Page token. * @param {string|module:pubsub/topic} options.topic - The name of the topic to @@ -432,21 +419,6 @@ PubSub.prototype.getSnapshotsStream = * }); * * //- - * // To control how many API requests are made and page through the results - * // manually, set `autoPaginate` to `false`. - * //- - * var callback = function(err, subscriptions, nextQuery, apiResponse) { - * if (nextQuery) { - * // More results exist. - * pubsub.getSubscriptions(nextQuery, callback); - * } - * }; - * - * pubsub.getSubscriptions({ - * autoPaginate: false - * }, callback); - * - * //- * // If the callback is omitted, we'll return a Promise. * //- * pubsub.getSubscriptions().then(function(data) { @@ -487,15 +459,8 @@ PubSub.prototype.getSubscriptions = function(options, callback) { if (subscriptions) { arguments[1] = subscriptions.map(function(sub) { - // Depending on if we're using a subscriptions.list or - // topics.subscriptions.list API endpoint, we will get back a - // Subscription resource or just the name of the subscription. - var subscriptionInstance = self.subscription(sub.name || sub); - - if (sub.name) { - subscriptionInstance.metadata = sub; - } - + var subscriptionInstance = self.subscription(sub.name); + subscriptionInstance.metadata = sub; return subscriptionInstance; }); } @@ -541,10 +506,8 @@ PubSub.prototype.getSubscriptionsStream = * @resource [Topics: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list} * * @param {object=} query - Query object. - * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: true. - * @param {number} options.maxApiCalls - Maximum number of API calls to make. - * @param {number} options.maxResults - Maximum number of results to return. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} query.pageSize - Max number of results to return. * @param {string} query.pageToken - Page token. * @param {function} callback - The callback function. @@ -568,21 +531,6 @@ PubSub.prototype.getSubscriptionsStream = * }, function(err, topics) {}); * * //- - * // To control how many API requests are made and page through the results - * // manually, set `autoPaginate` to `false`. - * //- - * var callback = function(err, rows, nextQuery, apiResponse) { - * if (nextQuery) { - * // More results exist. - * pubsub.getTopics(nextQuery, callback); - * } - * }; - * - * pubsub.getTopics({ - * autoPaginate: false - * }, callback); - * - * //- * // If the callback is omitted, we'll return a Promise. * //- * pubsub.getTopics().then(function(data) { @@ -655,10 +603,13 @@ PubSub.prototype.getTopicsStream = common.paginator.streamify('getTopics'); /** * Funnel all API requests through this method, to be sure we have a project ID. * + * @private + * * @param {object} config - Configuration object. * @param {object} config.gaxOpts - GAX options. * @param {function} config.method - The gax method to call. * @param {object} config.reqOpts - Request options. + * @param {boolean} config.returnFn - Return function as opposed to calling it. * @param {function=} callback - The callback function. */ PubSub.prototype.request = function(config, callback) { @@ -739,13 +690,10 @@ PubSub.prototype.snapshot = function(name) { * requests. You will receive a {module:pubsub/subscription} object, * which will allow you to interact with a subscription. * - * All generated names share a common prefix, `autogenerated-`. + * @throws {Error} If subscription name is omitted. * - * @param {string=} name - The name of the subscription. If a name is not - * provided, a random subscription name will be generated. + * @param {string} name - Name of the subscription. * @param {object=} options - Configuration object. - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') * @return {module:pubsub/subscription} * * @example @@ -757,7 +705,7 @@ PubSub.prototype.snapshot = function(name) { * // message.id = ID of the message. * // message.ackId = ID used to acknowledge the message receival. * // message.data = Contents of the message. - * // message.attrs = Attributes of the message. + * // message.attributes = Attributes of the message. * // message.publishTime = Timestamp when Pub/Sub received the message. * }); */ @@ -779,10 +727,6 @@ PubSub.prototype.subscription = function(name, options) { * * @example * var topic = pubsub.topic('my-topic'); - * - * topic.publish({ - * data: 'New message!' - * }, function(err) {}); */ PubSub.prototype.topic = function(name, options) { if (!name) { diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 3eac76466f9..d4b290c623c 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -329,7 +329,7 @@ describe('PubSub', function() { it('should pass options to the api request', function(done) { var options = { - ackDeadlineSeconds: 90, + ackDeadline: 90000, retainAckedMessages: true, pushEndpoint: 'https://domain/push', }; @@ -340,9 +340,11 @@ describe('PubSub', function() { }, options, { pushConfig: { pushEndpoint: options.pushEndpoint - } + }, + ackDeadlineSeconds: options.ackDeadline / 1000 }); + delete expectedBody.ackDeadline; delete expectedBody.pushEndpoint; pubsub.topic = function() { From 46b9cd96b360ee32b24d4a9c17b6a4afdf8afbd0 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Sun, 30 Jul 2017 21:28:15 -0400 Subject: [PATCH 44/67] update topic documentation --- packages/pubsub/src/topic.js | 101 +++++++++++++++++----------------- packages/pubsub/test/topic.js | 7 ++- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index e0e9d170818..ab897202864 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -111,7 +111,8 @@ Topic.formatName_ = function(projectId, name) { /** * Create a topic. * - * @param {object=} config - See {module:pubsub#createTopic}. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * * @example * topic.create(function(err, topic, apiResponse) { @@ -128,31 +129,32 @@ Topic.formatName_ = function(projectId, name) { * var apiResponse = data[1]; * }); */ -Topic.prototype.create = function(callback) { - this.pubsub.createTopic(this.name, callback); +Topic.prototype.create = function(gaxOpts, callback) { + this.pubsub.createTopic(this.name, gaxOpts, callback); }; /** * Create a subscription to this topic. * - * All generated subscription names share a common prefix, `autogenerated-`. - * * @resource [Subscriptions: create API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/create} * - * @param {string=} subName - The name of the subscription. If a name is not - * provided, a random subscription name will be generated and created. + * @throws {Error} If subscription name is omitted. + * + * @param {string=} name - The name of the subscription. * @param {object=} options - See a * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadlineSeconds - The maximum time after - * receiving a message that you must ack a message before it is redelivered. - * @param {boolean=} options.autoAck - Automatically acknowledge the message - * once it's pulled. (default: false) - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') - * @param {number} options.interval - Interval in milliseconds to check for new - * messages. (default: 10) - * @param {number} options.maxInProgress - Maximum messages to consume - * simultaneously. + * @param {number} options.ackDeadline - The maximum time after receiving a + * message that you must ack a message before it is redelivered. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number|date} options.messageRetentionDuration - Set this to override * the default duration of 7 days. This value is expected in seconds. * Acceptable values are in the range of 10 minutes and 7 days. @@ -161,10 +163,10 @@ Topic.prototype.create = function(callback) { * @param {boolean} options.retainAckedMessages - If set, acked messages are * retained in the subscription's backlog for 7 days (unless overriden by * `options.messageRetentionDuration`). Default: `false` - * @param {number} options.timeout - Set a maximum amount of time in - * milliseconds on an HTTP request to pull new messages to wait for a - * response before the connection is broken. * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request + * @param {module:pubsub/subscription} callback.subscription - The subscription. + * @param {object} callback.apiResponse - The full API response. * * @example * var callback = function(err, subscription, apiResponse) {}; @@ -172,17 +174,9 @@ Topic.prototype.create = function(callback) { * // Without specifying any options. * topic.createSubscription('newMessages', callback); * - * //- - * // Omit the name to have one generated automatically. All generated names - * // share a common prefix, `autogenerated-`. - * //- - * topic.createSubscription(function(err, subscription, apiResponse) { - * // subscription.name = The generated name. - * }); - * * // With options. * topic.createSubscription('newMessages', { - * ackDeadlineSeconds: 90 + * ackDeadline: 90000 // 90 seconds * }, callback); * * //- @@ -202,7 +196,9 @@ Topic.prototype.createSubscription = function(name, options, callback) { * * @resource [Topics: delete API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/delete} * - * @param {function=} callback - The callback function. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {function} callback - The callback function. * * @example * topic.delete(function(err, apiResponse) {}); @@ -237,6 +233,8 @@ Topic.prototype.delete = function(gaxOpts, callback) { * * @resource [Topics: get API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/get} * + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function} callback - The callback function. * @param {?error} callback.err - An error returned while making this * request. @@ -282,10 +280,8 @@ Topic.prototype.getMetadata = function(gaxOpts, callback) { * @resource [Subscriptions: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics.subscriptions/list} * * @param {object=} options - Configuration object. - * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: true. - * @param {number} options.maxApiCalls - Maximum number of API calls to make. - * @param {number} options.maxResults - Maximum number of results to return. + * @param {object} options.gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. * @param {string} options.pageToken - Page token. * @param {function} callback - The callback function. @@ -301,21 +297,6 @@ Topic.prototype.getMetadata = function(gaxOpts, callback) { * }, callback); * * //- - * // To control how many API requests are made and page through the results - * // manually, set `autoPaginate` to `false`. - * //- - * function callback(err, subscriptions, nextQuery, apiResponse) { - * if (nextQuery) { - * // More results exist. - * topic.getSubscriptions(nextQuery, callback); - * } - * } - * - * topic.getSubscriptions({ - * autoPaginate: false - * }, callback); - * - * //- * // If the callback is omitted, we'll return a Promise. * //- * topic.getSubscriptions().then(function(data) { @@ -386,7 +367,25 @@ Topic.prototype.getSubscriptionsStream = common.paginator.streamify('getSubscriptions'); /** + * Creates a Publisher object that allows you to publish messages to this topic. + * + * @param {object=} options - Configuration object. + * @param {object} options.batching - Batching settings. + * @param {number} options.batching.maxBytes - The maximum number of bytes to + * buffer before sending a payload. + * @param {number} options.batching.maxMessages - The maximum number of messages + * to buffer before sending a payload. + * @param {number} options.batching.maxMilliseconds - The maximum duration to + * wait before sending a payload. * + * @example + * var publisher = topic.publisher(); + * + * publisher.publish(new Buffer('Hello, world!'), function(err, messageId) { + * if (err) { + * // Error handling omitted. + * } + * }); */ Topic.prototype.publisher = function(options) { return new Publisher(this, options); @@ -397,6 +396,8 @@ Topic.prototype.publisher = function(options) { * requests. You will receive a {module:pubsub/subscription} object, * which will allow you to interact with a subscription. * + * @throws {Error} If subscription name is omitted. + * * @param {string} name - Name of the subscription. * @param {object=} options - Configuration object. * @return {module:pubsub/subscription} @@ -410,7 +411,7 @@ Topic.prototype.publisher = function(options) { * // message.id = ID of the message. * // message.ackId = ID used to acknowledge the message receival. * // message.data = Contents of the message. - * // message.attrs = Attributes of the message. + * // message.attributes = Attributes of the message. * // message.publishTime = Timestamp when Pub/Sub received the message. * }); */ diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index 6ea63beb372..81723c0578b 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -151,12 +151,15 @@ describe('Topic', function() { describe('create', function() { it('should call the parent createTopic method', function(done) { - PUBSUB.createTopic = function(name, callback) { + var options_ = {}; + + PUBSUB.createTopic = function(name, options, callback) { assert.strictEqual(name, topic.name); + assert.strictEqual(options, options_); callback(); // the done fn }; - topic.create(done); + topic.create(options_, done); }); }); From 15273fd0bb0d800a752216f43cba5e2d83d4256e Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Sun, 30 Jul 2017 21:28:38 -0400 Subject: [PATCH 45/67] update subscription documentation --- packages/pubsub/src/subscription.js | 177 +++++++++++++++++++++++++-- packages/pubsub/test/subscription.js | 24 ++++ 2 files changed, 189 insertions(+), 12 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 6d29a3c5c3e..36432a9cb1d 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -54,10 +54,19 @@ var Snapshot = require('./snapshot.js'); /*! Developer Documentation * * @param {module:pubsub} pubsub - PubSub object. - * @param {object} options - Configuration object. - * @param {string} options.encoding - When pulling for messages, this type is - * used when converting a message's data to a string. (default: 'utf-8') - * @param {string} options.name - Name of the subscription. + * @param {string=} name - The name of the subscription. + * @param {object=} options - See a + * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + * @param {number} options.ackDeadline - The maximum time after receiving a + * message that you must ack a message before it is redelivered. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. */ /** * A Subscription object will give you access to your Cloud Pub/Sub @@ -98,10 +107,10 @@ var Snapshot = require('./snapshot.js'); * }); * * //- - * // From {module:pubsub/topic#subscribe}: + * // From {module:pubsub/topic#createSubscription}: * //- * var topic = pubsub.topic('my-topic'); - * topic.subscribe('new-subscription', function(err, subscription) { + * topic.createSubscription('new-subscription', function(err, subscription) { * // `subscription` is a Subscription object. * }); * @@ -131,12 +140,11 @@ var Snapshot = require('./snapshot.js'); * // message.timestamp = Timestamp when Pub/Sub received the message. * * // Ack the message: - * // message.ack(callback); + * // message.ack(); * - * // Skip the message. This is useful with `maxInProgress` option when - * // creating your subscription. This doesn't ack the message, but allows - * // more messages to be retrieved if your limit was hit. - * // message.skip(); + * // This doesn't ack the message, but allows more messages to be retrieved + * // if your limit was hit or if you don't want to ack the message. + * // message.nack(); * } * subscription.on('message', onMessage); * @@ -237,7 +245,13 @@ Subscription.formatName_ = function(projectId, name) { }; /** + * Acks the provided message. If the connection pool is absent, it will be + * placed in an internal queue and sent out after 1 second or if the pool is + * re-opened before the timeout hits. * + * @private + * + * @param {object} message - The message object. */ Subscription.prototype.ack_ = function(message) { this.breakLease_(message); @@ -262,7 +276,15 @@ Subscription.prototype.ack_ = function(message) { }; /** + * Breaks the lease on a message. Essentially this means we no longer treat the + * message as being un-acked and count it towards the flow control limits. + * + * If the pool was previously paused and we freed up space, we'll continue to + * recieve messages. + * + * @private * + * @param {object} message - The message object. */ Subscription.prototype.breakLease_ = function(message) { var messageIndex = this.inventory_.lease.indexOf(message.ackId); @@ -283,7 +305,24 @@ Subscription.prototype.breakLease_ = function(message) { }; /** + * Closes the subscription, once this is called you will no longer receive + * message events unless you add a new message listener. * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while closing the + * Subscription. + * + * @example + * subscription.close(function(err) { + * if (err) { + * // Error handling omitted. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.close().then(function() {}); */ Subscription.prototype.close = function(callback) { this.userClosed_ = true; @@ -299,7 +338,12 @@ Subscription.prototype.close = function(callback) { }; /** + * Closes the connection pool. + * + * @private * + * @param {function=} callback - The callback function. + * @param {?error} err - An error returned from this request. */ Subscription.prototype.closeConnection_ = function(callback) { if (this.connectionPool) { @@ -314,6 +358,8 @@ Subscription.prototype.closeConnection_ = function(callback) { * Create a snapshot with the given name. * * @param {string} name - Name of the snapshot. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function=} callback - The callback function. * @param {?error} callback.err - An error from the API call, may be null. * @param {module:pubsub/snapshot} callback.snapshot - The newly created @@ -379,6 +425,8 @@ Subscription.prototype.createSnapshot = function(name, gaxOpts, callback) { * * @resource [Subscriptions: delete API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/delete} * + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function=} callback - The callback function. * @param {?error} callback.err - An error returned while making this * request. @@ -422,7 +470,15 @@ Subscription.prototype.delete = function(gaxOpts, callback) { }; /** + * Flushes internal queues. These can build up if a user attempts to ack/nack + * while there is no connection pool (e.g. after they called close). * + * Typically this will only be called either after a timeout or when a + * connection is re-opened. + * + * Any errors that occur will be emitted via `error` events. + * + * @private */ Subscription.prototype.flushQueues_ = function() { var self = this; @@ -498,10 +554,26 @@ Subscription.prototype.flushQueues_ = function() { /** + * Fetches the subscriptions metadata. + * + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function} callback - The callback function. * @param {?error} callback.err - An error returned while making this * request. * @param {object} callback.apiResponse - Raw API response. + * + * @example + * subscription.getMetadata(function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.getMetadata().then(function(apiResponse) {}); */ Subscription.prototype.getMetadata = function(gaxOpts, callback) { if (is.fn(gaxOpts)) { @@ -522,7 +594,12 @@ Subscription.prototype.getMetadata = function(gaxOpts, callback) { }; /** + * Checks to see if this Subscription has hit any of the flow control + * thresholds. + * + * @private * + * @return {boolean} */ Subscription.prototype.hasMaxMessages_ = function() { return this.inventory_.lease.length >= this.flowControl.maxMessages || @@ -530,7 +607,13 @@ Subscription.prototype.hasMaxMessages_ = function() { }; /** + * Leases a message. This will add the message to our inventory list and then + * modifiy the ack deadline for the user if they exceed the specified ack + * deadline. + * + * @private * + * @param {object} message - The message object. */ Subscription.prototype.leaseMessage_ = function(message) { this.inventory_.lease.push(message.ackId); @@ -575,9 +658,36 @@ Subscription.prototype.listenForEvents_ = function() { }; /** + * Modify the push config for the subscription. + * * @param {object} config - The push config. * @param {string} config.pushEndpoint * @param {object} config.attributes + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error from the API call. + * @param {object} callback.apiResponse - The full API response from the + * service. + * + * @example + * var pushConfig = { + * pushEndpoint: 'https://mydomain.com/push', + * attributes: { + * key: 'value' + * } + * }; + * + * subscription.modifyPushConfig(pushConfig, function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.modifyPushConfig(pushConfig).then(function(apiResponse) {}); */ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { if (is.fn(gaxOpts)) { @@ -599,7 +709,13 @@ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { }; /** + * Nacks the provided message. If the connection pool is absent, it will be + * placed in an internal queue and sent out after 1 second or if the pool is + * re-opened before the timeout hits. + * + * @private * + * @param {object} message - The message object. */ Subscription.prototype.nack_ = function(message) { this.breakLease_(message); @@ -626,6 +742,8 @@ Subscription.prototype.nack_ = function(message) { }; /** + * Opens the ConnectionPool. + * * @private */ Subscription.prototype.openConnection_ = function() { @@ -643,7 +761,7 @@ Subscription.prototype.openConnection_ = function() { pool.on('message', function(message) { self.emit('message', self.leaseMessage_(message)); - if (self.hasMaxMessages_()) { + if (self.hasMaxMessages_() && !pool.isPaused) { pool.pause(); } }); @@ -656,7 +774,10 @@ Subscription.prototype.openConnection_ = function() { }; /** + * Modifies the ack deadline on messages that have yet to be acked. We update + * the ack deadline to the 99th percentile of known ack times. * + * @private */ Subscription.prototype.renewLeases_ = function() { var self = this; @@ -707,6 +828,8 @@ Subscription.prototype.renewLeases_ = function() { * * @param {string|date} snapshot - The point to seek to. This will accept the * name of the snapshot or a Date object. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function} callback - The callback function. * @param {?error} callback.err - An error from the API call, may be null. * @param {object} callback.apiResponse - The full API response from the @@ -756,7 +879,10 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { }; /** + * Sets a timeout to flush any acks/nacks that have been made since the pool has + * closed. * + * @private */ Subscription.prototype.setFlushTimeout_ = function() { if (!this.flushTimeoutHandle_) { @@ -765,7 +891,10 @@ Subscription.prototype.setFlushTimeout_ = function() { }; /** + * Sets a timeout to modify the ack deadlines for any unacked/unnacked messages, + * renewing their lease. * + * @private */ Subscription.prototype.setLeaseTimeout_ = function() { if (this.leaseTimeoutHandle_) { @@ -777,7 +906,31 @@ Subscription.prototype.setLeaseTimeout_ = function() { }; /** + * Update the subscription object. + * + * @param {object} metadata - The subscription metadata. + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error from the API call. + * @param {object} callback.apiResponse - The full API response from the + * service. + * + * @example + * var metadata = { + * key: 'value' + * }; + * + * subscription.setMetadata(metadata, function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * }); * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.setMetadata(metadata).then(function(apiResponse) {}); */ Subscription.prototype.setMetadata = function(metadata, gaxOpts, callback) { if (is.fn(gaxOpts)) { diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index 43a64dccddb..88a069135bf 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -1153,10 +1153,34 @@ describe('Subscription', function() { }; subscription.openConnection_(); + subscription.connectionPool.isPaused = false; subscription.connectionPool.pause = done; subscription.connectionPool.emit('message', message); }); + it('should not re-pause the pool', function(done) { + var message = {}; + var leasedMessage = {}; + + subscription.leaseMessage_ = function() { + return leasedMessage; + }; + + subscription.hasMaxMessages_ = function() { + return true; + }; + + subscription.openConnection_(); + subscription.connectionPool.isPaused = true; + + subscription.connectionPool.pause = function() { + done(new Error('Should not have been called.')); + }; + + subscription.connectionPool.emit('message', message); + done(); + }); + it('should flush the queue when connected', function(done) { subscription.flushQueues_ = function() { assert.strictEqual(subscription.flushTimeoutHandle_, null); From 485392311a7f331f789a3fac880786777042f920 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Sun, 30 Jul 2017 21:28:53 -0400 Subject: [PATCH 46/67] update snapshot documentation --- packages/pubsub/src/snapshot.js | 8 ++++++++ packages/pubsub/test/snapshot.js | 34 +++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index c4c36085abd..897f3c2d99a 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -20,6 +20,7 @@ 'use strict'; +var common = require('@google-cloud/common'); var is = require('is'); /** @@ -199,4 +200,11 @@ Snapshot.prototype.delete = function(callback) { }, callback); }; +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +common.util.promisifyAll(Snapshot); + module.exports = Snapshot; diff --git a/packages/pubsub/test/snapshot.js b/packages/pubsub/test/snapshot.js index b6701f1164c..d74684a5300 100644 --- a/packages/pubsub/test/snapshot.js +++ b/packages/pubsub/test/snapshot.js @@ -17,9 +17,21 @@ 'use strict'; var assert = require('assert'); +var common = require('@google-cloud/common'); +var extend = require('extend'); +var proxyquire = require('proxyquire'); + +var promisified = false; +var fakeUtil = extend({}, common.util, { + promisifyAll: function(Class) { + if (Class.name === 'Snapshot') { + promisified = true; + } + } +}); describe('Snapshot', function() { - var Snapshot = require('../src/snapshot.js'); + var Snapshot; var snapshot; var SNAPSHOT_NAME = 'a'; @@ -32,6 +44,18 @@ describe('Snapshot', function() { seek: function() {} }; + before(function() { + Snapshot = proxyquire('../src/snapshot.js', { + '@google-cloud/common': { + util: fakeUtil + } + }); + }); + + beforeEach(function() { + snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); + }); + describe('initialization', function() { var FULL_SNAPSHOT_NAME = 'a/b/c/d'; var formatName_; @@ -43,14 +67,14 @@ describe('Snapshot', function() { }; }); - beforeEach(function() { - snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); - }); - after(function() { Snapshot.formatName_ = formatName_; }); + it('should promisify all the things', function() { + assert(promisified); + }); + it('should localize the parent', function() { assert.strictEqual(snapshot.parent, SUBSCRIPTION); }); From e1848e9cb08b0fcd10fc37273c237729a872c763 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Sun, 30 Jul 2017 21:29:08 -0400 Subject: [PATCH 47/67] add docs for new classes --- packages/pubsub/src/connection-pool.js | 35 +++++++++++-- packages/pubsub/src/histogram.js | 9 +++- packages/pubsub/src/publisher.js | 68 +++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index fb6b7bcd667..c0f8d9cc3f6 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -30,7 +30,16 @@ var util = require('util'); var uuid = require('uuid'); /** + * ConnectionPool is used to manage the stream connections created via + * StreamingPull rpc. * + * @param {module:pubsub/subscription} subscription - The subscription to create + * connections for. + * @param {object=} options - Pool options. + * @param {number} options.maxConnections - Number of connections to create. + * Default: 5. + * @param {number} options.ackDeadline - The ack deadline to send when + * creating a connection. */ function ConnectionPool(subscription, options) { this.subscription = subscription; @@ -51,7 +60,15 @@ function ConnectionPool(subscription, options) { util.inherits(ConnectionPool, events.EventEmitter); /** + * Acquires a connection from the pool. Optionally you can specify an id for a + * specific connection, but if it is no longer available it will return the + * first available connection. * + * @param {string=} id - The id of the connection to retrieve. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while acquiring a + * connection. + * @param {stream} callback.connection - A duplex stream. */ ConnectionPool.prototype.acquire = function(id, callback) { var self = this; @@ -89,7 +106,11 @@ ConnectionPool.prototype.acquire = function(id, callback) { }; /** + * Ends each connection in the pool and closes the pool, preventing new + * connections from being created. * + * @param {function} callback - The callback function. + * @param {?error} callback.error - An error returned while closing the pool. */ ConnectionPool.prototype.close = function(callback) { var connections = Array.from(this.connections.values()); @@ -103,7 +124,8 @@ ConnectionPool.prototype.close = function(callback) { }; /** - * + * Creates a connection. This is async but instead of providing a callback + * a `connected` event will fire once the connection is ready. */ ConnectionPool.prototype.createConnection = function() { var self = this; @@ -157,7 +179,12 @@ ConnectionPool.prototype.createConnection = function() { }; /** + * Creates a message object for the user. * + * @param {string} connectionId - The connection id that the message was + * received on. + * @param {object} resp - The message response data from StreamingPull. + * @return {object} message - The message object. */ ConnectionPool.prototype.createMessage = function(connectionId, resp) { var self = this; @@ -183,7 +210,7 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { }; /** - * + * Creates specified number of connections and puts pool in open state. */ ConnectionPool.prototype.open = function() { for (var i = 0; i < this.settings.maxConnections; i++) { @@ -194,7 +221,7 @@ ConnectionPool.prototype.open = function() { }; /** - * + * Pauses each of the connections, causing `message` events to stop firing. */ ConnectionPool.prototype.pause = function() { this.isPaused = true; @@ -205,7 +232,7 @@ ConnectionPool.prototype.pause = function() { }; /** - * + * Calls resume on each connection, allowing `message` events to fire off again. */ ConnectionPool.prototype.resume = function() { this.isPaused = false; diff --git a/packages/pubsub/src/histogram.js b/packages/pubsub/src/histogram.js index d2b3d031267..e0674851721 100644 --- a/packages/pubsub/src/histogram.js +++ b/packages/pubsub/src/histogram.js @@ -24,7 +24,9 @@ var MIN_VALUE = 10000; var MAX_VALUE = 600000; /** - * + * The Histogram class is used to capture the lifespan of messages within the + * the client. These durations are then used to calculate the 99th percentile + * of ack deadlines for future messages. */ function Histogram() { this.data = new Map(); @@ -32,7 +34,9 @@ function Histogram() { } /** + * Adds a value to the histogram. * + * @param {numnber} value - The value in milliseconds. */ Histogram.prototype.add = function(value) { value = Math.max(value, MIN_VALUE); @@ -49,7 +53,10 @@ Histogram.prototype.add = function(value) { }; /** + * Retrieves the nth percentile of recorded values. * + * @param {number} percent - The requested percentage. + * @return {number} */ Histogram.prototype.percentile = function(percent) { percent = Math.min(percent, 100); diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 6692eb6eae2..05d6f6da5ab 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -25,8 +25,30 @@ var common = require('@google-cloud/common'); var extend = require('extend'); var is = require('is'); +/*! Developer Documentation. + * + * @param {module:pubsub/topic} topic - The topic associated with this + * publisher. + * @param {object=} options - Configuration object. + * @param {object} options.batching - Batching settings. + * @param {number} options.batching.maxBytes - The maximum number of bytes to + * buffer before sending a payload. + * @param {number} options.batching.maxMessages - The maximum number of messages + * to buffer before sending a payload. + * @param {number} options.batching.maxMilliseconds - The maximum duration to + * wait before sending a payload. + */ /** + * A Publisher object allows you to publish messages to a specific topic. + * + * @constructor + * @alias module:pubsub/publisher + * + * @resource [Topics: publish API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish} * + * @example + * var topic = pubsub.topic('my-topic'); + * var publisher = topic.publisher(); */ function Publisher(topic, options) { options = extend(true, { @@ -62,7 +84,43 @@ function Publisher(topic, options) { } /** + * Publish the provided message. + * + * @throws {TypeError} If data is not a Buffer object. + * + * @param {buffer} data - The message data. This must come in the form of a + * Buffer object. + * @param {object=} attributes - Optional attributes for this message. + * @param {function=} callback - The callback function. If omitted a Promise + * will be returned. + * @param {?error} callback.error - An error returned while making this request. + * @param {string} callback.messageId - The id for the message. + * + * @example + * var data = new Buffer('Hello, world!'); * + * var callback = function(err, messageId) { + * if (err) { + * // Error handling omitted. + * } + * }; + * + * publisher.publish(data, callback); + * + * //- + * // Optionally you can provide an object containing attributes for the + * // message. + * //- + * var attributes = { + * key: 'value' + * }; + * + * publisher.publish(data, attributes, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * publisher.publish(data).then(function(messageId) {}); */ Publisher.prototype.publish = function(data, attrs, callback) { if (!(data instanceof Buffer)) { @@ -103,7 +161,9 @@ Publisher.prototype.publish = function(data, attrs, callback) { }; /** - * This should never be called directly. + * This publishes a batch of messages and should never be called directly. + * + * @private */ Publisher.prototype.publish_ = function() { var callbacks = this.inventory_.callbacks; @@ -135,7 +195,13 @@ Publisher.prototype.publish_ = function() { }; /** + * Queues message to be sent to the server. + * + * @private * + * @param {buffer} data - The message data. + * @param {object} attributes - The message attributes. + * @param {function} callback - The callback function. */ Publisher.prototype.queue_ = function(data, attrs, callback) { this.inventory_.queued.push({ From 00759797a549ee9ee92b4d386150dbff0038c2b6 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 31 Jul 2017 10:19:06 -0400 Subject: [PATCH 48/67] update PubSub examples in READMEs --- README.md | 7 +++++-- packages/pubsub/README.md | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6ae24b037ac..98b660a8cd8 100644 --- a/README.md +++ b/README.md @@ -847,10 +847,13 @@ var pubsubClient = pubsub({ var topic = pubsubClient.topic('my-topic'); // Publish a message to the topic. -topic.publish('New message!', function(err) {}); +var publisher = topic.publisher(); +var message = new Buffer('New message!'); + +publisher.publish('New message!', function(err, messageId) {}); // Subscribe to the topic. -topic.subscribe('subscription-name', function(err, subscription) { +topic.createSubscription('subscription-name', function(err, subscription) { // Register listeners to start pulling for messages. function onError(err) {} function onMessage(message) {} diff --git a/packages/pubsub/README.md b/packages/pubsub/README.md index eae849cf6eb..4244d2bb77c 100644 --- a/packages/pubsub/README.md +++ b/packages/pubsub/README.md @@ -20,10 +20,13 @@ var pubsub = require('@google-cloud/pubsub')({ var topic = pubsub.topic('my-topic'); // Publish a message to the topic. -topic.publish('New message!', function(err) {}); +var publisher = topic.publisher(); +var message = new Buffer('New message!'); + +publisher.publish(message, function(err, messageId) {}); // Subscribe to the topic. -topic.subscribe('subscription-name', function(err, subscription) { +topic.createSubscription('subscription-name', function(err, subscription) { // Register listeners to start pulling for messages. function onError(err) {} function onMessage(message) {} @@ -36,7 +39,7 @@ topic.subscribe('subscription-name', function(err, subscription) { }); // Promises are also supported by omitting callbacks. -topic.publish('New message!').then(function(data) { +publisher.publish(message).then(function(data) { var messageIds = data[0]; }); From cc6ca622b06c1b3cffcc6d51b42e9a524cd39197 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 31 Jul 2017 10:21:41 -0400 Subject: [PATCH 49/67] update docs script to ignore internal pubsub classes --- scripts/docs/config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/docs/config.js b/scripts/docs/config.js index fbb4974401e..a840daeb23f 100644 --- a/scripts/docs/config.js +++ b/scripts/docs/config.js @@ -32,6 +32,8 @@ module.exports = { 'error-reporting/src/error-router.js', 'error-reporting/src/logger.js', 'logging/src/metadata.js', + 'pubsub/src/connection-pool.js', + 'pubsub/src/histogram.js', 'pubsub/src/iam.js', 'spanner/src/codec.js', 'spanner/src/partial-result-stream.js', From fb068d1101d90efdb23673e2b2371b7b65172693 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 31 Jul 2017 10:43:57 -0400 Subject: [PATCH 50/67] fix documentation typos/errors --- README.md | 2 +- packages/pubsub/src/index.js | 4 ++-- packages/pubsub/src/subscription.js | 14 ++++++++++---- packages/pubsub/src/topic.js | 16 +++++++++++----- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 98b660a8cd8..6fb83e07c77 100644 --- a/README.md +++ b/README.md @@ -850,7 +850,7 @@ var topic = pubsubClient.topic('my-topic'); var publisher = topic.publisher(); var message = new Buffer('New message!'); -publisher.publish('New message!', function(err, messageId) {}); +publisher.publish(message, function(err, messageId) {}); // Subscribe to the topic. topic.createSubscription('subscription-name', function(err, subscription) { diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index fefd22bed35..35d83e8f4fd 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -118,7 +118,7 @@ function PubSub(options) { * * @example * //- - * // Subscribe to a topic. (Also see {module:pubsub/topic#subscribe}). + * // Subscribe to a topic. (Also see {module:pubsub/topic#createSubscription}). * //- * var topic = 'messageCenter'; * var name = 'newMessages'; @@ -506,7 +506,7 @@ PubSub.prototype.getSubscriptionsStream = * @resource [Topics: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list} * * @param {object=} query - Query object. - * @param {object} options.gaxOpts - Request configuration options, outlined + * @param {object} query.gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} query.pageSize - Max number of results to return. * @param {string} query.pageToken - Page token. diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 36432a9cb1d..3a475548593 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -76,7 +76,7 @@ var Snapshot = require('./snapshot.js'); * * - {module:pubsub#getSubscriptions} * - {module:pubsub/topic#getSubscriptions} - * - {module:pubsub/topic#subscribe} + * - {module:pubsub/topic#createSubscription} * * Subscription objects may be created directly with: * @@ -573,7 +573,9 @@ Subscription.prototype.flushQueues_ = function() { * //- * // If the callback is omitted, we'll return a Promise. * //- - * subscription.getMetadata().then(function(apiResponse) {}); + * subscription.getMetadata().then(function(data) { + * var apiResponse = data[0]; + * }); */ Subscription.prototype.getMetadata = function(gaxOpts, callback) { if (is.fn(gaxOpts)) { @@ -687,7 +689,9 @@ Subscription.prototype.listenForEvents_ = function() { * //- * // If the callback is omitted, we'll return a Promise. * //- - * subscription.modifyPushConfig(pushConfig).then(function(apiResponse) {}); + * subscription.modifyPushConfig(pushConfig).then(function(data) { + * var apiResponse = data[0]; + * }); */ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { if (is.fn(gaxOpts)) { @@ -930,7 +934,9 @@ Subscription.prototype.setLeaseTimeout_ = function() { * //- * // If the callback is omitted, we'll return a Promise. * //- - * subscription.setMetadata(metadata).then(function(apiResponse) {}); + * subscription.setMetadata(metadata).then(function(data) { + * var apiResponse = data[0]; + * }); */ Subscription.prototype.setMetadata = function(metadata, gaxOpts, callback) { if (is.fn(gaxOpts)) { diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index ab897202864..4a967e178b0 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -113,6 +113,10 @@ Topic.formatName_ = function(projectId, name) { * * @param {object=} gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:pubsub/topic} callback.topic - The topic. + * @param {object} callback.apiResponse - The full API response. * * @example * topic.create(function(err, topic, apiResponse) { @@ -164,7 +168,7 @@ Topic.prototype.create = function(gaxOpts, callback) { * retained in the subscription's backlog for 7 days (unless overriden by * `options.messageRetentionDuration`). Default: `false` * @param {function} callback - The callback function. - * @param {?error} callback.err - An error returned while making this request + * @param {?error} callback.err - An error returned while making this request. * @param {module:pubsub/subscription} callback.subscription - The subscription. * @param {object} callback.apiResponse - The full API response. * @@ -238,18 +242,16 @@ Topic.prototype.delete = function(gaxOpts, callback) { * @param {function} callback - The callback function. * @param {?error} callback.err - An error returned while making this * request. - * @param {object} callback.metadata - The metadata of the Topic. * @param {object} callback.apiResponse - The full API response. * * @example - * topic.getMetadata(function(err, metadata, apiResponse) {}); + * topic.getMetadata(function(err, apiResponse) {}); * * //- * // If the callback is omitted, we'll return a Promise. * //- * topic.getMetadata().then(function(data) { - * var metadata = data[0]; - * var apiResponse = data[1]; + * var apiResponse = data[0]; * }); */ Topic.prototype.getMetadata = function(gaxOpts, callback) { @@ -285,6 +287,10 @@ Topic.prototype.getMetadata = function(gaxOpts, callback) { * @param {number} options.pageSize - Maximum number of results to return. * @param {string} options.pageToken - Page token. * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {module:pubsub/subscription[]} callback.subscriptions - List of + * subscriptions. * * @example * topic.getSubscriptions(function(err, subscriptions) { From 3a367fe263040aed17f3e2961d8b3678a87b6117 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 3 Aug 2017 13:11:46 -0400 Subject: [PATCH 51/67] tweaks per PR feedback & full test coverage --- packages/pubsub/package.json | 1 + packages/pubsub/src/connection-pool.js | 13 ++- packages/pubsub/src/iam.js | 2 +- packages/pubsub/src/publisher.js | 8 +- packages/pubsub/src/subscription.js | 50 ++++++----- packages/pubsub/src/topic.js | 4 +- packages/pubsub/system-test/pubsub.js | 9 +- packages/pubsub/test/connection-pool.js | 14 ++- packages/pubsub/test/histogram.js | 2 +- packages/pubsub/test/iam.js | 1 - packages/pubsub/test/index.js | 79 +++++++++++++---- packages/pubsub/test/publisher.js | 17 ++++ packages/pubsub/test/snapshot.js | 21 +++++ packages/pubsub/test/subscription.js | 111 ++++++++++++++---------- packages/pubsub/test/topic.js | 14 ++- 15 files changed, 235 insertions(+), 111 deletions(-) diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index 7f71b150193..36e14de2610 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -56,6 +56,7 @@ "arrify": "^1.0.0", "async-each": "^1.0.1", "extend": "^3.0.0", + "google-auto-auth": "^0.7.1", "google-gax": "^0.13.0", "google-proto-files": "^0.12.0", "is": "^3.0.1", diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index c0f8d9cc3f6..038c2cef00e 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -24,7 +24,6 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); var each = require('async-each'); var events = require('events'); -var extend = require('extend'); var is = require('is'); var util = require('util'); var uuid = require('uuid'); @@ -41,16 +40,16 @@ var uuid = require('uuid'); * @param {number} options.ackDeadline - The ack deadline to send when * creating a connection. */ -function ConnectionPool(subscription, options) { +function ConnectionPool(subscription) { this.subscription = subscription; this.connections = new Map(); this.isPaused = false; this.isOpen = false; - this.settings = extend({ - maxConnections: 5, - ackDeadline: 10000 - }, options); + this.settings = { + maxConnections: subscription.maxConnections || 5, + ackDeadline: subscription.ackDeadline || 10000 + }; events.EventEmitter.call(this); @@ -79,7 +78,7 @@ ConnectionPool.prototype.acquire = function(id, callback) { } if (!this.isOpen) { - callback(new Error('Connection pool is closed.')); + callback(new Error('No connections available to make request.')); return; } diff --git a/packages/pubsub/src/iam.js b/packages/pubsub/src/iam.js index 914c43fbb41..a64a32fb8d8 100644 --- a/packages/pubsub/src/iam.js +++ b/packages/pubsub/src/iam.js @@ -1,5 +1,5 @@ /*! - * Copyright 2017 Google Inc. All Rights Reserved. + * Copyright 2014 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 05d6f6da5ab..0162fb615b1 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -22,6 +22,7 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); +var each = require('async-each'); var extend = require('extend'); var is = require('is'); @@ -188,8 +189,11 @@ Publisher.prototype.publish_ = function() { }, function(err, resp) { var messageIds = arrify(resp && resp.messageIds); - callbacks.forEach(function(callback, i) { - callback(err, messageIds[i]); + each(callbacks, function(callback, next) { + var messageId = messageIds[callbacks.indexOf(callback)]; + + callback(err, messageId); + next(); }); }); }; diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 3a475548593..04fa6a984eb 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -52,21 +52,8 @@ var IAM = require('./iam.js'); var Snapshot = require('./snapshot.js'); /*! Developer Documentation - * * @param {module:pubsub} pubsub - PubSub object. * @param {string=} name - The name of the subscription. - * @param {object=} options - See a - * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadline - The maximum time after receiving a - * message that you must ack a message before it is redelivered. - * @param {object} options.flowControl - Flow control configurations for - * receiving messages. - * @param {number} options.flowControl.maxBytes - The maximum number of bytes - * in un-acked messages to allow before the subscription pauses incoming - * messages. Defaults to 20% of free memory. - * @param {number} options.flowControl.maxMessages - The maximum number of - * un-acked messages to allow before the subscription pauses incoming - * messages. Default: Infinity. */ /** * A Subscription object will give you access to your Cloud Pub/Sub @@ -90,6 +77,21 @@ var Snapshot = require('./snapshot.js'); * @alias module:pubsub/subscription * @constructor * + * @param {object=} options - See a + * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + * @param {number} options.ackDeadline - The maximum time after receiving a + * message that you must ack a message before it is redelivered. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. + * @param {number} options.maxConnections - Use this to limit the number of + * connections to be used when sending and receiving messages. Default: 5. + * * @example * //- * // From {module:pubsub#getSubscriptions}: @@ -450,6 +452,8 @@ Subscription.prototype.delete = function(gaxOpts, callback) { gaxOpts = {}; } + callback = callback || common.util.noop; + var reqOpts = { subscription: this.name }; @@ -461,7 +465,7 @@ Subscription.prototype.delete = function(gaxOpts, callback) { gaxOpts: gaxOpts }, function(err, resp) { if (!err) { - self.removeAllListeners('message'); + self.removeAllListeners(); self.close(); } @@ -663,8 +667,10 @@ Subscription.prototype.listenForEvents_ = function() { * Modify the push config for the subscription. * * @param {object} config - The push config. - * @param {string} config.pushEndpoint - * @param {object} config.attributes + * @param {string} config.pushEndpoint - A URL locating the endpoint to which + * messages should be published. + * @param {object} config.attributes - A set of API supported attributes that + * can be used to control different aspects of the message delivery. * @param {object=} gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function} callback - The callback function. @@ -752,11 +758,7 @@ Subscription.prototype.nack_ = function(message) { */ Subscription.prototype.openConnection_ = function() { var self = this; - - var pool = this.connectionPool = new ConnectionPool(this, { - ackDeadline: this.ackDeadline, - maxConnections: this.maxConnections - }); + var pool = this.connectionPool = new ConnectionPool(this); pool.on('error', function(err) { self.emit('error', err); @@ -889,9 +891,11 @@ Subscription.prototype.seek = function(snapshot, gaxOpts, callback) { * @private */ Subscription.prototype.setFlushTimeout_ = function() { - if (!this.flushTimeoutHandle_) { - this.flushTimeoutHandle_ = setTimeout(this.flushQueues_.bind(this), 1000); + if (this.flushTimeoutHandle_) { + return; } + + this.flushTimeoutHandle_ = setTimeout(this.flushQueues_.bind(this), 1000); }; /** diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index 4a967e178b0..a645d8005a0 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -202,7 +202,7 @@ Topic.prototype.createSubscription = function(name, options, callback) { * * @param {object=} gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. - * @param {function} callback - The callback function. + * @param {function=} callback - The callback function. * * @example * topic.delete(function(err, apiResponse) {}); @@ -220,6 +220,8 @@ Topic.prototype.delete = function(gaxOpts, callback) { gaxOpts = {}; } + callback = callback || common.util.noop; + var reqOpts = { topic: this.name }; diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 284009db8c6..be161bafe26 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -138,6 +138,7 @@ describe('pubsub', function() { it('should allow manual paging', function(done) { pubsub.getTopics({ + autoPaginate: false, pageSize: TOPIC_NAMES.length - 1 }, function(err, topics) { assert.ifError(err); @@ -474,17 +475,15 @@ describe('pubsub', function() { return deleteAllSnapshots() .then(wait(2500)) - .then(subscription.create.bind(subscription)); + .then(subscription.create.bind(subscription)) + .then(snapshot.create.bind(snapshot)) + .then(wait(2500)); }); after(function() { return deleteAllSnapshots(); }); - it('should create a snapshot', function(done) { - snapshot.create(done); - }); - it('should get a list of snapshots', function(done) { pubsub.getSnapshots(function(err, snapshots) { assert.ifError(err); diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index 47a5a5487b0..e88af046c05 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -85,9 +85,12 @@ describe('ConnectionPool', function() { ackDeadline: 100 }; - var pool = new ConnectionPool(SUBSCRIPTION, options); + var subscription = extend({}, SUBSCRIPTION, options); + var subscriptionCopy = extend({}, subscription); + var pool = new ConnectionPool(subscription); assert.deepEqual(pool.settings, options); + assert.deepEqual(subscription, subscriptionCopy); }); it('should inherit from EventEmitter', function() { @@ -108,11 +111,13 @@ describe('ConnectionPool', function() { describe('acquire', function() { it('should return an error if the pool is closed', function(done) { + var expectedErr = 'No connections available to make request.'; + pool.isOpen = false; pool.acquire(function(err) { assert(err instanceof Error); - assert.strictEqual(err.message, 'Connection pool is closed.'); + assert.strictEqual(err.message, expectedErr); done(); }); }); @@ -420,14 +425,17 @@ describe('ConnectionPool', function() { describe('open', function() { it('should make the specified number of connections', function() { + var expectedCount = 5; var connectionCount = 0; pool.createConnection = function() { connectionCount += 1; }; + pool.settings.maxConnections = expectedCount; pool.open(); - assert.strictEqual(pool.settings.maxConnections, connectionCount); + + assert.strictEqual(expectedCount, connectionCount); }); it('should set the isOpen flag to true', function() { diff --git a/packages/pubsub/test/histogram.js b/packages/pubsub/test/histogram.js index 30afcc854e6..eee2693c194 100644 --- a/packages/pubsub/test/histogram.js +++ b/packages/pubsub/test/histogram.js @@ -41,7 +41,7 @@ describe('Histogram', function() { }); describe('add', function() { - it('increment a value', function() { + it('should increment a value', function() { histogram.data.set(MIN_VALUE, 1); histogram.add(MIN_VALUE); diff --git a/packages/pubsub/test/iam.js b/packages/pubsub/test/iam.js index 8c346a02761..25dde7ed8fc 100644 --- a/packages/pubsub/test/iam.js +++ b/packages/pubsub/test/iam.js @@ -45,7 +45,6 @@ describe('IAM', function() { var iam; var PUBSUB = { - defaultBaseUrl_: 'base-url', options: {} }; var ID = 'id'; diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index d4b290c623c..057923dc355 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -27,8 +27,6 @@ var v1 = require('../src/v1/index.js'); var SubscriptionCached = require('../src/subscription.js'); var SubscriptionOverride; -var Topic = require('../src/topic.js'); - function Subscription(a, b, c) { var OverrideFn = SubscriptionOverride || SubscriptionCached; return new OverrideFn(a, b, c); @@ -55,6 +53,10 @@ function FakeSnapshot() { this.calledWith_ = arguments; } +function FakeTopic() { + this.calledWith_ = arguments; +} + var extended = false; var fakePaginator = { extend: function(Class, methods) { @@ -121,7 +123,7 @@ describe('PubSub', function() { 'google-auto-auth': fakeGoogleAutoAuth, './snapshot.js': FakeSnapshot, './subscription.js': Subscription, - './topic.js': Topic, + './topic.js': FakeTopic, './v1': fakeV1, './v1/publisher_client_config.json': GAX_CONFIG.Publisher, './v1/subscriber_client_config.json': GAX_CONFIG.Subscriber @@ -230,9 +232,9 @@ describe('PubSub', function() { describe('createSubscription', function() { var TOPIC_NAME = 'topic'; - var TOPIC = { + var TOPIC = extend(new FakeTopic(), { name: 'projects/' + PROJECT_ID + '/topics/' + TOPIC_NAME - }; + }); var SUB_NAME = 'subscription'; var SUBSCRIPTION = { @@ -260,7 +262,7 @@ describe('PubSub', function() { callback(null, apiResponse); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, done); + pubsub.createSubscription(TOPIC, SUB_NAME, done); }); it('should allow undefined/optional configuration options', function(done) { @@ -268,7 +270,7 @@ describe('PubSub', function() { callback(null, apiResponse); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, undefined, done); + pubsub.createSubscription(TOPIC, SUB_NAME, undefined, done); }); it('should create a Subscription', function(done) { @@ -283,7 +285,7 @@ describe('PubSub', function() { return SUBSCRIPTION; }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, opts, assert.ifError); + pubsub.createSubscription(TOPIC, SUB_NAME, opts, assert.ifError); }); it('should create a Topic object from a string', function(done) { @@ -318,13 +320,13 @@ describe('PubSub', function() { pubsub.request = function(config) { assert.strictEqual(config.client, 'subscriberClient'); assert.strictEqual(config.method, 'createSubscription'); - assert.strictEqual(config.reqOpts.topic, TOPIC_NAME); + assert.strictEqual(config.reqOpts.topic, TOPIC.name); assert.strictEqual(config.reqOpts.name, SUB_NAME); assert.strictEqual(config.gaxOpts, options.gaxOpts); done(); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, options, assert.ifError); + pubsub.createSubscription(TOPIC, SUB_NAME, options, assert.ifError); }); it('should pass options to the api request', function(done) { @@ -335,7 +337,7 @@ describe('PubSub', function() { }; var expectedBody = extend({ - topic: TOPIC_NAME, + topic: TOPIC.name, name: SUB_NAME }, options, { pushConfig: { @@ -365,7 +367,7 @@ describe('PubSub', function() { done(); }; - pubsub.createSubscription(TOPIC_NAME, SUB_NAME, options, assert.ifError); + pubsub.createSubscription(TOPIC, SUB_NAME, options, assert.ifError); }); describe('message retention', function() { @@ -668,7 +670,7 @@ describe('PubSub', function() { it('should pass back all parameters', function(done) { var err_ = new Error('abc'); - var snapshots_ = []; + var snapshots_ = null; var nextQuery_ = {}; var apiResponse_ = {}; @@ -741,7 +743,7 @@ describe('PubSub', function() { it('should pass back all params', function(done) { var err_ = new Error('err'); - var subs_ = []; + var subs_ = false; var nextQuery_ = {}; var apiResponse_ = {}; @@ -757,6 +759,45 @@ describe('PubSub', function() { done(); }); }); + + describe('with topic', function() { + var TOPIC_NAME = 'topic-name'; + + it('should call topic.getSubscriptions', function(done) { + var topic = new FakeTopic(); + + var opts = { + topic: topic + }; + + topic.getSubscriptions = function(options, callback) { + assert.strictEqual(options, opts); + callback(); // the done fn + }; + + pubsub.getSubscriptions(opts, done); + }); + + it('should create a topic instance from a name', function(done) { + var opts = { + topic: TOPIC_NAME + }; + + var fakeTopic = { + getSubscriptions: function(options, callback) { + assert.strictEqual(options, opts); + callback(); // the done fn + } + }; + + pubsub.topic = function(name) { + assert.strictEqual(name, TOPIC_NAME); + return fakeTopic; + }; + + pubsub.getSubscriptions(opts, done); + }); + }); }); describe('getTopics', function() { @@ -814,7 +855,7 @@ describe('PubSub', function() { it('should pass back all params', function(done) { var err_ = new Error('err'); - var topics_ = []; + var topics_ = false; var nextQuery_ = {}; var apiResponse_ = {}; @@ -937,9 +978,9 @@ describe('PubSub', function() { }); it('should do nothing if sandbox env var is set', function(done) { - process.env.GCLOUD_SANDBOX_ENV = true; + global.GCLOUD_SANDBOX_ENV = true; pubsub.request(CONFIG, done); // should not fire done - process.evn.GCLOUD_SANDBOX_ENV = false; + global.GCLOUD_SANDBOX_ENV = false; done(); }); }); @@ -964,7 +1005,7 @@ describe('PubSub', function() { describe('subscription', function() { var SUB_NAME = 'new-sub-name'; - var CONFIG = { autoAck: true, interval: 90 }; + var CONFIG = {}; it('should return a Subscription object', function() { SubscriptionOverride = function() {}; @@ -1003,7 +1044,7 @@ describe('PubSub', function() { }); it('should return a Topic object', function() { - assert(pubsub.topic('new-topic') instanceof Topic); + assert(pubsub.topic('new-topic') instanceof FakeTopic); }); }); }); diff --git a/packages/pubsub/test/publisher.js b/packages/pubsub/test/publisher.js index d1ee6918767..4b5e3db2a2d 100644 --- a/packages/pubsub/test/publisher.js +++ b/packages/pubsub/test/publisher.js @@ -87,12 +87,14 @@ describe('Publisher', function() { maxMessages: 11, maxMilliseconds: 12 }; + var optionsCopy = extend({}, options); var publisher = new Publisher(TOPIC, { batching: options }); assert.deepEqual(publisher.settings.batching, options); + assert.deepEqual(options, optionsCopy); }); it('should cap maxBytes', function() { @@ -119,6 +121,21 @@ describe('Publisher', function() { var DATA = new Buffer('hello'); var ATTRS = { a: 'a' }; + var globalSetTimeout; + + before(function() { + globalSetTimeout = global.setTimeout; + }); + + beforeEach(function() { + publisher.publish_ = fakeUtil.noop; + global.setTimeout = fakeUtil.noop; + }); + + after(function() { + global.setTimeout = globalSetTimeout; + }); + it('should throw an error when data is not a buffer', function() { assert.throws(function() { publisher.publish('hello', {}, fakeUtil.noop); diff --git a/packages/pubsub/test/snapshot.js b/packages/pubsub/test/snapshot.js index d74684a5300..8e04caa2639 100644 --- a/packages/pubsub/test/snapshot.js +++ b/packages/pubsub/test/snapshot.js @@ -37,7 +37,12 @@ describe('Snapshot', function() { var SNAPSHOT_NAME = 'a'; var PROJECT_ID = 'grape-spaceship-123'; + var PUBSUB = { + projectId: PROJECT_ID + }; + var SUBSCRIPTION = { + pubsub: PUBSUB, projectId: PROJECT_ID, api: {}, createSnapshot: function() {}, @@ -124,6 +129,22 @@ describe('Snapshot', function() { snapshot.seek(done); }); }); + + describe('with PubSub parent', function() { + var snapshot; + + beforeEach(function() { + snapshot = new Snapshot(PUBSUB, SNAPSHOT_NAME); + }); + + it('should not include the create method', function() { + assert.strictEqual(snapshot.create, undefined); + }); + + it('should not include a seek method', function() { + assert.strictEqual(snapshot.seek, undefined); + }); + }); }); describe('formatName_', function() { diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index 88a069135bf..c0b726b04e4 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -82,7 +82,7 @@ describe('Subscription', function() { }); beforeEach(function() { - PUBSUB.request = fakeUtil.noop; + PUBSUB.request = fakeUtil.noop = function() {}; subscription = new Subscription(PUBSUB, SUB_NAME); }); @@ -233,6 +233,7 @@ describe('Subscription', function() { }; beforeEach(function() { + subscription.setFlushTimeout_ = fakeUtil.noop; subscription.breakLease_ = fakeUtil.noop; subscription.histogram.add = fakeUtil.noop; }); @@ -459,7 +460,7 @@ describe('Subscription', function() { it('should use a noop when callback is absent', function(done) { fakeUtil.noop = done; - subscription.closeConnection_(done); + subscription.closeConnection_(); assert.strictEqual(subscription.connectionPool, null); }); }); @@ -523,13 +524,16 @@ describe('Subscription', function() { it('should pass back any errors to the callback', function(done) { var error = new Error('err'); + var apiResponse = {}; subscription.request = function(config, callback) { - callback(error); + callback(error, apiResponse); }; - subscription.createSnapshot(SNAPSHOT_NAME, function(err) { + subscription.createSnapshot(SNAPSHOT_NAME, function(err, snapshot, resp) { assert.strictEqual(err, error); + assert.strictEqual(snapshot, null); + assert.strictEqual(resp, apiResponse); done(); }); }); @@ -588,6 +592,16 @@ describe('Subscription', function() { }; }); + it('should optionally accept a callback', function(done) { + fakeUtil.noop = function(err, resp) { + assert.ifError(err); + assert.strictEqual(resp, apiResponse); + done(); + }; + + subscription.delete(); + }); + it('should return the api response', function(done) { subscription.delete(function(err, resp) { assert.ifError(err); @@ -599,8 +613,7 @@ describe('Subscription', function() { it('should remove all message listeners', function(done) { var called = false; - subscription.removeAllListeners = function(name) { - assert.strictEqual(name, 'message'); + subscription.removeAllListeners = function() { called = true; }; @@ -670,6 +683,19 @@ describe('Subscription', function() { subscription.inventory_.nack = ['ghi', 'jkl']; }); + it('should do nothing if theres nothing to ack/nack', function() { + subscription.inventory_.ack = []; + subscription.inventory_.nack = []; + + subscription.connectionPool = { + acquire: function() { + throw new Error('Should not be called.'); + } + }; + + subscription.flushQueues_(); + }); + describe('with connection pool', function() { var fakeConnection; @@ -685,17 +711,6 @@ describe('Subscription', function() { }; }); - it('should do nothing if theres nothing to ack/nack', function() { - subscription.inventory_.ack = []; - subscription.inventory_.nack = []; - - subscription.connectionPool.acquire = function() { - throw new Error('Should not be called.'); - }; - - subscription.flushQueues_(); - }); - it('should emit any connection acquiring errors', function(done) { var error = new Error('err'); @@ -713,20 +728,26 @@ describe('Subscription', function() { it('should write the acks to the connection', function(done) { fakeConnection.write = function(reqOpts) { - assert.deepEqual(reqOpts.ackIds, ['abc', 'def']); + assert.deepEqual(reqOpts, { + ackIds: ['abc', 'def'] + }); done(); }; + subscription.inventory_.nack = []; subscription.flushQueues_(); }); it('should write the nacks to the connection', function(done) { fakeConnection.write = function(reqOpts) { - assert.deepEqual(reqOpts.modifyDeadlineAckIds, ['ghi', 'jkl']); - assert.deepEqual(reqOpts.modifyDeadlineSeconds, [0, 0]); + assert.deepEqual(reqOpts, { + modifyDeadlineAckIds: ['ghi', 'jkl'], + modifyDeadlineSeconds: [0, 0] + }); done(); }; + subscription.inventory_.ack = []; subscription.flushQueues_(); }); @@ -739,17 +760,6 @@ describe('Subscription', function() { }); describe('without connection pool', function() { - it('should do nothing if theres nothing to ack/nack', function() { - subscription.inventory_.ack = []; - subscription.inventory_.nack = []; - - subscription.request = function() { - throw new Error('Should not be called.'); - }; - - subscription.flushQueues_(); - }); - describe('acking', function() { beforeEach(function() { subscription.inventory_.nack = []; @@ -899,6 +909,10 @@ describe('Subscription', function() { data: new Buffer('hello') }; + beforeEach(function() { + subscription.setLeaseTimeout_ = fakeUtil.noop; + }); + it('should add the ackId to the inventory', function() { subscription.leaseMessage_(MESSAGE); assert.deepEqual(subscription.inventory_.lease, [MESSAGE.ackId]); @@ -907,7 +921,7 @@ describe('Subscription', function() { it('should update the byte count', function() { assert.strictEqual(subscription.inventory_.bytes, 0); subscription.leaseMessage_(MESSAGE); - assert.strictEqual(subscription.inventory_.bytes, 5); + assert.strictEqual(subscription.inventory_.bytes, MESSAGE.data.length); }); it('should begin auto-leasing', function(done) { @@ -1027,6 +1041,7 @@ describe('Subscription', function() { }; beforeEach(function() { + subscription.setFlushTimeout_ = fakeUtil.noop; subscription.breakLease_ = fakeUtil.noop; }); @@ -1104,10 +1119,6 @@ describe('Subscription', function() { var args = subscription.connectionPool.calledWith_; assert.strictEqual(args[0], subscription); - assert.deepEqual(args[1], { - ackDeadline: subscription.ackDeadline, - maxConnections: subscription.maxConnections - }); }); it('should emit pool errors', function(done) { @@ -1198,6 +1209,7 @@ describe('Subscription', function() { beforeEach(function() { subscription.inventory_.lease = ['abc', 'def']; + subscription.setLeaseTimeout_ = fakeUtil.noop; subscription.histogram.percentile = function() { return fakeDeadline; @@ -1222,6 +1234,17 @@ describe('Subscription', function() { subscription.renewLeases_(); }); + it('should not renew leases if inventory is empty', function() { + subscription.connectionPool = { + acquire: function() { + throw new Error('Should not have been called.'); + } + }; + + subscription.inventory_.lease = []; + subscription.renewLeases_(); + }); + describe('with connection pool', function() { var fakeConnection; @@ -1237,15 +1260,6 @@ describe('Subscription', function() { }; }); - it('should not renew leases if inventory is empty', function() { - subscription.connectionPool.acquire = function() { - throw new Error('Should not have been called.'); - }; - - subscription.inventory_.lease = []; - subscription.renewLeases_(); - }); - it('should emit any pool acquiring errors', function(done) { var error = new Error('err'); @@ -1276,7 +1290,7 @@ describe('Subscription', function() { describe('without connection pool', function() { it('should make the correct request', function(done) { - subscription.request = function(config) { + subscription.request = function(config, callback) { assert.strictEqual(config.client, 'subscriberClient'); assert.strictEqual(config.method, 'modifyAckDeadline'); assert.deepEqual(config.reqOpts, { @@ -1284,9 +1298,14 @@ describe('Subscription', function() { ackIds: ['abc', 'def'], ackDeadlineSeconds: fakeDeadline / 1000 }); + callback(); done(); }; + subscription.on('error', function(err) { + done(err); + }); + subscription.renewLeases_(); }); diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index 81723c0578b..9d206b789aa 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -196,11 +196,21 @@ describe('Topic', function() { topic.request = function(config, callback) { assert.strictEqual(config.gaxOpts, options); - callback(); + callback(); // the done fn }; topic.delete(options, done); }); + + it('should optionally accept a callback', function(done) { + fakeUtil.noop = done; + + topic.request = function(config, callback) { + callback(); // the done fn + }; + + topic.delete(); + }); }); describe('getMetadata', function() { @@ -288,7 +298,7 @@ describe('Topic', function() { it('should pass all params to the callback', function(done) { var err_ = new Error('err'); - var subs_ = []; + var subs_ = false; var nextQuery_ = {}; var apiResponse_ = {}; From fe1d0304d0c981859b408964d2e62f46bc4c789d Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 3 Aug 2017 14:19:31 -0400 Subject: [PATCH 52/67] stub out os module --- packages/pubsub/test/subscription.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index c0b726b04e4..c58117d6210 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -21,10 +21,16 @@ var common = require('@google-cloud/common'); var events = require('events'); var extend = require('extend'); var is = require('is'); -var os = require('os'); var proxyquire = require('proxyquire'); var util = require('util'); +var FAKE_FREE_MEM = 168222720; +var fakeOs = { + freemem: function() { + return FAKE_FREE_MEM; + } +}; + var promisified = false; var fakeUtil = extend({}, common.util, { promisifyAll: function(Class, options) { @@ -74,6 +80,7 @@ describe('Subscription', function() { '@google-cloud/common': { util: fakeUtil }, + os: fakeOs, './connection-pool.js': FakeConnectionPool, './histogram.js': FakeHistogram, './iam.js': FakeIAM, @@ -157,7 +164,7 @@ describe('Subscription', function() { assert.strictEqual(subscription.messageListeners, 0); assert.deepEqual(subscription.flowControl, { - maxBytes: os.freemem() * 0.2, + maxBytes: FAKE_FREE_MEM * 0.2, maxMessages: Infinity }); }); From c00f6b4aeb2baec1c39e20223471a439f2456546 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 3 Aug 2017 14:27:10 -0400 Subject: [PATCH 53/67] add link for push config attributes --- packages/pubsub/src/subscription.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 04fa6a984eb..691bce115a8 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -669,8 +669,7 @@ Subscription.prototype.listenForEvents_ = function() { * @param {object} config - The push config. * @param {string} config.pushEndpoint - A URL locating the endpoint to which * messages should be published. - * @param {object} config.attributes - A set of API supported attributes that - * can be used to control different aspects of the message delivery. + * @param {object} config.attributes - [PushConfig attributes](https://cloud.google.com/pubsub/docs/reference/rpc/google.pubsub.v1#google.pubsub.v1.PushConfig). * @param {object=} gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {function} callback - The callback function. From 09ef328f701b436720a3fa59eff01ef85e29a7bb Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 3 Aug 2017 14:57:58 -0400 Subject: [PATCH 54/67] test tweaks --- packages/pubsub/system-test/pubsub.js | 4 ++-- packages/pubsub/test/connection-pool.js | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index be161bafe26..dc6c4284c72 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -138,8 +138,8 @@ describe('pubsub', function() { it('should allow manual paging', function(done) { pubsub.getTopics({ - autoPaginate: false, - pageSize: TOPIC_NAMES.length - 1 + pageSize: TOPIC_NAMES.length - 1, + gaxOpts: { autoPaginate: false } }, function(err, topics) { assert.ifError(err); assert(topics.length, TOPIC_NAMES.length - 1); diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index e88af046c05..74658a35ea6 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -363,8 +363,10 @@ describe('ConnectionPool', function() { describe('createMessage', function() { var message; + var globalDateNow; var CONNECTION_ID = 'abc'; + var FAKE_DATE_NOW = Date.now(); var PT = { seconds: 6838383, @@ -383,10 +385,21 @@ describe('ConnectionPool', function() { } }; + before(function() { + globalDateNow = global.Date.now; + global.Date.now = function() { + return FAKE_DATE_NOW; + }; + }); + beforeEach(function() { message = pool.createMessage(CONNECTION_ID, RESP); }); + after(function() { + global.Date.now = globalDateNow; + }); + it('should capture the connection id', function() { assert.strictEqual(message.connectionId, CONNECTION_ID); }); @@ -394,14 +407,13 @@ describe('ConnectionPool', function() { it('should capture the message data', function() { var expectedPublishTime = new Date( parseInt(PT.seconds, 10) * 1000 + parseInt(PT.nanos, 10) / 1e6); - var dateNowValue = Date.now(); assert.strictEqual(message.ackId, RESP.ackId); assert.strictEqual(message.id, RESP.message.messageId); assert.strictEqual(message.data, RESP.message.data); assert.strictEqual(message.attributes, RESP.message.attributes); assert.deepEqual(message.publishTime, expectedPublishTime); - assert.strictEqual(message.received, dateNowValue); + assert.strictEqual(message.received, FAKE_DATE_NOW); }); it('should create an ack method', function(done) { From e6cf7204a4eebbb51e076aa173b445639dd51e6c Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 7 Aug 2017 08:58:23 -0400 Subject: [PATCH 55/67] add top level autoPaginate support --- packages/pubsub/src/index.js | 27 +++++++++++++++++--- packages/pubsub/src/topic.js | 9 ++++++- packages/pubsub/test/index.js | 48 ++++++++++++++++++++++++++++++----- packages/pubsub/test/topic.js | 16 +++++++++--- 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index 35d83e8f4fd..c6f51fb111e 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -298,6 +298,8 @@ PubSub.prototype.determineBaseUrl_ = function() { * Get a list of snapshots. * * @param {object=} options - Configuration object. + * @param {boolean} options.autoPaginate - Have pagination handled + * automatically. Default: true. * @param {object} options.gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. @@ -334,12 +336,17 @@ PubSub.prototype.getSnapshots = function(options, callback) { }, options); delete reqOpts.gaxOpts; + delete reqOpts.autoPaginate; + + var gaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); this.request({ client: 'subscriberClient', method: 'listSnapshots', reqOpts: reqOpts, - gaxOpts: options.gaxOpts + gaxOpts: gaxOpts }, function() { var snapshots = arguments[1]; @@ -398,6 +405,8 @@ PubSub.prototype.getSnapshotsStream = * @resource [Subscriptions: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/list} * * @param {object=} options - Configuration object. + * @param {boolean} options.autoPaginate - Have pagination handled + * automatically. Default: true. * @param {object} options.gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. @@ -448,12 +457,17 @@ PubSub.prototype.getSubscriptions = function(options, callback) { reqOpts.project = 'projects/' + this.projectId; delete reqOpts.gaxOpts; + delete reqOpts.autoPaginate; + + var gaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); this.request({ client: 'subscriberClient', method: 'listSubscriptions', reqOpts: reqOpts, - gaxOpts: options.gaxOpts + gaxOpts: gaxOpts }, function() { var subscriptions = arguments[1]; @@ -506,6 +520,8 @@ PubSub.prototype.getSubscriptionsStream = * @resource [Topics: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list} * * @param {object=} query - Query object. + * @param {boolean} query.autoPaginate - Have pagination handled + * automatically. Default: true. * @param {object} query.gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} query.pageSize - Max number of results to return. @@ -550,12 +566,17 @@ PubSub.prototype.getTopics = function(options, callback) { }, options); delete reqOpts.gaxOpts; + delete reqOpts.autoPaginate; + + var gaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); this.request({ client: 'publisherClient', method: 'listTopics', reqOpts: reqOpts, - gaxOpts: options.gaxOpts + gaxOpts: gaxOpts }, function() { var topics = arguments[1]; diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index a645d8005a0..cd18b773ec2 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -284,6 +284,8 @@ Topic.prototype.getMetadata = function(gaxOpts, callback) { * @resource [Subscriptions: list API Documentation]{@link https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics.subscriptions/list} * * @param {object=} options - Configuration object. + * @param {boolean} options.autoPaginate - Have pagination handled + * automatically. Default: true. * @param {object} options.gaxOpts - Request configuration options, outlined * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. * @param {number} options.pageSize - Maximum number of results to return. @@ -324,12 +326,17 @@ Topic.prototype.getSubscriptions = function(options, callback) { }, options); delete reqOpts.gaxOpts; + delete reqOpts.autoPaginate; + + var gaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); this.request({ client: 'publisherClient', method: 'listTopicSubscriptions', reqOpts: reqOpts, - gaxOpts: options.gaxOpts + gaxOpts: gaxOpts }, function() { var subscriptions = arguments[1]; diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 057923dc355..5bbceffdee3 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -634,18 +634,31 @@ describe('PubSub', function() { }); it('should build the right request', function(done) { - var options = { a: 'b', c: 'd', gaxOpts: {} }; + var options = { + a: 'b', + c: 'd', + gaxOpts: { + e: 'f' + }, + autoPaginate: false + }; + var expectedOptions = extend({}, options, { project: 'projects/' + pubsub.projectId }); + var expectedGaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); + delete expectedOptions.gaxOpts; + delete expectedOptions.autoPaginate; pubsub.request = function(config) { assert.strictEqual(config.client, 'subscriberClient'); assert.strictEqual(config.method, 'listSnapshots'); assert.deepEqual(config.reqOpts, expectedOptions); - assert.deepEqual(config.gaxOpts, options.gaxOpts); + assert.deepEqual(config.gaxOpts, expectedGaxOpts); done(); }; @@ -706,14 +719,24 @@ describe('PubSub', function() { }); it('should pass the correct arguments to the API', function(done) { - var options = { gaxOpts: {} }; + var options = { + gaxOpts: { + a: 'b' + }, + autoPaginate: false + }; + + var expectedGaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); + var project = 'projects/' + pubsub.projectId; pubsub.request = function(config) { assert.strictEqual(config.client, 'subscriberClient'); assert.strictEqual(config.method, 'listSubscriptions'); assert.deepEqual(config.reqOpts, { project: project }); - assert.strictEqual(config.gaxOpts, options.gaxOpts); + assert.deepEqual(config.gaxOpts, expectedGaxOpts); done(); }; @@ -819,18 +842,31 @@ describe('PubSub', function() { }); it('should build the right request', function(done) { - var options = { a: 'b', c: 'd', gaxOpts: {} }; + var options = { + a: 'b', + c: 'd', + gaxOpts: { + e: 'f' + }, + autoPaginate: false + }; + var expectedOptions = extend({}, options, { project: 'projects/' + pubsub.projectId }); + var expectedGaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); + delete expectedOptions.gaxOpts; + delete expectedOptions.autoPaginate; pubsub.request = function(config) { assert.strictEqual(config.client, 'publisherClient'); assert.strictEqual(config.method, 'listTopics'); assert.deepEqual(config.reqOpts, expectedOptions); - assert.strictEqual(config.gaxOpts, options.gaxOpts); + assert.deepEqual(config.gaxOpts, expectedGaxOpts); done(); }; diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index 9d206b789aa..c23ca5f97b4 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -240,22 +240,30 @@ describe('Topic', function() { describe('getSubscriptions', function() { it('should make the correct request', function(done) { var options = { - gaxOpts: {}, a: 'a', - b: 'b' + b: 'b', + gaxOpts: { + e: 'f' + }, + autoPaginate: false }; var expectedOptions = extend({ topic: topic.name }, options); + var expectedGaxOpts = extend({ + autoPaginate: options.autoPaginate + }, options.gaxOpts); + delete expectedOptions.gaxOpts; + delete expectedOptions.autoPaginate; topic.request = function(config) { assert.strictEqual(config.client, 'publisherClient'); assert.strictEqual(config.method, 'listTopicSubscriptions'); assert.deepEqual(config.reqOpts, expectedOptions); - assert.strictEqual(config.gaxOpts, options.gaxOpts); + assert.deepEqual(config.gaxOpts, expectedGaxOpts); done(); }; @@ -265,7 +273,7 @@ describe('Topic', function() { it('should accept only a callback', function(done) { topic.request = function(config) { assert.deepEqual(config.reqOpts, { topic: topic.name }); - assert.strictEqual(config.gaxOpts, undefined); + assert.deepEqual(config.gaxOpts, { autoPaginate: undefined }); done(); }; From 79cf70170f13003893adf9e0017208c847ea2f1e Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 7 Aug 2017 09:47:58 -0400 Subject: [PATCH 56/67] made Snapshot#delete cb optional --- packages/pubsub/src/snapshot.js | 2 ++ packages/pubsub/src/subscription.js | 2 +- packages/pubsub/test/snapshot.js | 11 +++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/pubsub/src/snapshot.js b/packages/pubsub/src/snapshot.js index 897f3c2d99a..ff25e0f1340 100644 --- a/packages/pubsub/src/snapshot.js +++ b/packages/pubsub/src/snapshot.js @@ -193,6 +193,8 @@ Snapshot.prototype.delete = function(callback) { snapshot: this.name }; + callback = callback || common.util.noop; + this.parent.request({ client: 'subscriberClient', method: 'deleteSnapshot', diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index 691bce115a8..e49b0186ccd 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -1,5 +1,5 @@ /*! - * Copyright 2017 Google Inc. All Rights Reserved. + * Copyright 2014 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/pubsub/test/snapshot.js b/packages/pubsub/test/snapshot.js index 8e04caa2639..0e9f1d711eb 100644 --- a/packages/pubsub/test/snapshot.js +++ b/packages/pubsub/test/snapshot.js @@ -58,6 +58,7 @@ describe('Snapshot', function() { }); beforeEach(function() { + fakeUtil.noop = function() {}; snapshot = new Snapshot(SUBSCRIPTION, SNAPSHOT_NAME); }); @@ -174,5 +175,15 @@ describe('Snapshot', function() { snapshot.delete(done); }); + + it('should optionally accept a callback', function(done) { + fakeUtil.noop = done; + + snapshot.parent.request = function(config, callback) { + callback(); // the done fn + }; + + snapshot.delete(); + }); }); }); From 73a025cee49fae638eea74a98dd1ed97de6e2752 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Tue, 15 Aug 2017 14:33:52 -0400 Subject: [PATCH 57/67] tweaks based on PR feedback --- packages/pubsub/src/connection-pool.js | 12 +++++++-- packages/pubsub/src/index.js | 15 ++++------- packages/pubsub/src/publisher.js | 5 ++-- packages/pubsub/src/subscription.js | 7 +++-- packages/pubsub/test/connection-pool.js | 11 ++++++++ packages/pubsub/test/index.js | 36 ++++++++++++++++++++++--- packages/pubsub/test/publisher.js | 10 ++----- packages/pubsub/test/subscription.js | 2 -- 8 files changed, 65 insertions(+), 33 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 038c2cef00e..c3576e0344c 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -112,6 +112,7 @@ ConnectionPool.prototype.acquire = function(id, callback) { * @param {?error} callback.error - An error returned while closing the pool. */ ConnectionPool.prototype.close = function(callback) { + var self = this; var connections = Array.from(this.connections.values()); this.isOpen = false; @@ -119,7 +120,10 @@ ConnectionPool.prototype.close = function(callback) { each(connections, function(connection, onEndCallback) { connection.end(onEndCallback); - }, callback); + }, function(err) { + self.connections.clear(); + callback(err); + }); }; /** @@ -190,15 +194,19 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { var pt = resp.message.publishTime; var milliseconds = parseInt(pt.nanos, 10) / 1e6; + var data = resp.message.data; return { connectionId: connectionId, ackId: resp.ackId, id: resp.message.messageId, - data: resp.message.data, attributes: resp.message.attributes, publishTime: new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds), received: Date.now(), + // using get here to prevent user from overwriting data + get data() { + return data; + }, ack: function() { self.subscription.ack_(this); }, diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index c6f51fb111e..d48f1226969 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -91,10 +91,9 @@ function PubSub(options) { * @param {string=} name - The name of the subscription. * @param {object=} options - See a * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadline - The maximum time after receiving a - * message that you must ack a message before it is redelivered. * @param {object} options.flowControl - Flow control configurations for - * receiving messages. + * receiving messages. Note that these options do not persist across + * subscription instances. * @param {number} options.flowControl.maxBytes - The maximum number of bytes * in un-acked messages to allow before the subscription pauses incoming * messages. Defaults to 20% of free memory. @@ -109,8 +108,8 @@ function PubSub(options) { * @param {string} options.pushEndpoint - A URL to a custom endpoint that * messages should be pushed to. * @param {boolean} options.retainAckedMessages - If set, acked messages are - * retained in the subscription's backlog for 7 days (unless overriden by - * `options.messageRetentionDuration`). Default: `false` + * retained in the subscription's backlog for the length of time specified + * by `options.messageRetentionDuration`. Default: `false` * @param {function} callback - The callback function. * @param {?error} callback.err - An error returned while making this request * @param {module:pubsub/subscription} callback.subscription - The subscription. @@ -170,11 +169,7 @@ PubSub.prototype.createSubscription = function(topic, name, options, callback) { }, options); delete reqOpts.gaxOpts; - - if (options.ackDeadline) { - reqOpts.ackDeadlineSeconds = options.ackDeadline / 1000; - delete reqOpts.ackDeadline; - } + delete reqOpts.flowControl; if (options.messageRetentionDuration) { reqOpts.retainAckedMessages = true; diff --git a/packages/pubsub/src/publisher.js b/packages/pubsub/src/publisher.js index 0162fb615b1..b84f634a590 100644 --- a/packages/pubsub/src/publisher.js +++ b/packages/pubsub/src/publisher.js @@ -134,11 +134,10 @@ Publisher.prototype.publish = function(data, attrs, callback) { } var opts = this.settings.batching; - var newPayloadSize = this.inventory_.bytes + data.length; // if this message puts us over the maxBytes option, then let's ship // what we have and add it to the next batch - if (newPayloadSize > opts.maxBytes) { + if ((this.inventory_.bytes + data.length) > opts.maxBytes) { this.publish_(); } @@ -149,7 +148,7 @@ Publisher.prototype.publish = function(data, attrs, callback) { // magically hit the max byte limit var hasMaxMessages = this.inventory_.queued.length === opts.maxMessages; - if (newPayloadSize === opts.maxBytes || hasMaxMessages) { + if (this.inventory_.bytes === opts.maxBytes || hasMaxMessages) { this.publish_(); return; } diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index e49b0186ccd..fe4cac321a7 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -79,10 +79,9 @@ var Snapshot = require('./snapshot.js'); * * @param {object=} options - See a * [Subscription resource](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - * @param {number} options.ackDeadline - The maximum time after receiving a - * message that you must ack a message before it is redelivered. * @param {object} options.flowControl - Flow control configurations for - * receiving messages. + * receiving messages. Note that these options do not persist across + * subscription instances. * @param {number} options.flowControl.maxBytes - The maximum number of bytes * in un-acked messages to allow before the subscription pauses incoming * messages. Defaults to 20% of free memory. @@ -164,7 +163,7 @@ function Subscription(pubsub, name, options) { this.name = Subscription.formatName_(pubsub.projectId, name); this.connectionPool = null; - this.ackDeadline = options.ackDeadline || 10000; + this.ackDeadline = 10000; this.maxConnections = options.maxConnections || 5; this.inventory_ = { diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index 74658a35ea6..6f67984124d 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -201,6 +201,11 @@ describe('ConnectionPool', function() { assert.strictEqual(b.endCalled, true); }); + it('should clear the connections map', function(done) { + pool.connections.clear = done; + pool.close(); + }); + it('should exec a callback when finished closing', function(done) { pool.close(done); }); @@ -433,6 +438,12 @@ describe('ConnectionPool', function() { message.nack(); }); + + it('should not allow data to be overridden', function() { + assert.throws(function() { + message.data = 'hihihi'; + }); + }); }); describe('open', function() { diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 5bbceffdee3..6dfd37eaad0 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -331,7 +331,6 @@ describe('PubSub', function() { it('should pass options to the api request', function(done) { var options = { - ackDeadline: 90000, retainAckedMessages: true, pushEndpoint: 'https://domain/push', }; @@ -342,11 +341,9 @@ describe('PubSub', function() { }, options, { pushConfig: { pushEndpoint: options.pushEndpoint - }, - ackDeadlineSeconds: options.ackDeadline / 1000 + } }); - delete expectedBody.ackDeadline; delete expectedBody.pushEndpoint; pubsub.topic = function() { @@ -370,6 +367,37 @@ describe('PubSub', function() { pubsub.createSubscription(TOPIC, SUB_NAME, options, assert.ifError); }); + it('should discard flow control options', function(done) { + var options = { + flowControl: {} + }; + + var expectedBody = { + topic: TOPIC.name, + name: SUB_NAME + }; + + pubsub.topic = function() { + return { + name: TOPIC_NAME + }; + }; + + pubsub.subscription = function() { + return { + name: SUB_NAME + }; + }; + + pubsub.request = function(config) { + assert.notStrictEqual(config.reqOpts, options); + assert.deepEqual(config.reqOpts, expectedBody); + done(); + }; + + pubsub.createSubscription(TOPIC, SUB_NAME, options, assert.ifError); + }); + describe('message retention', function() { it('should accept a number', function(done) { var threeDaysInSeconds = 3 * 24 * 60 * 60; diff --git a/packages/pubsub/test/publisher.js b/packages/pubsub/test/publisher.js index 4b5e3db2a2d..d5e0487b24a 100644 --- a/packages/pubsub/test/publisher.js +++ b/packages/pubsub/test/publisher.js @@ -181,17 +181,11 @@ describe('Publisher', function() { }); it('should publish if data puts payload at size cap', function(done) { - var queueCalled = false; - publisher.queue_ = function() { - queueCalled = true; - }; - - publisher.publish_ = function() { - assert(queueCalled); - done(); + publisher.inventory_.bytes += DATA.length; }; + publisher.publish_ = done; publisher.inventory_.bytes = batchOpts.maxBytes - DATA.length; publisher.publish(DATA, fakeUtil.noop); }); diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index c58117d6210..b4bbb36ea06 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -138,7 +138,6 @@ describe('Subscription', function() { it('should honor configuration settings', function() { var options = { - ackDeadline: 5000, maxConnections: 2, flowControl: { maxBytes: 5, @@ -148,7 +147,6 @@ describe('Subscription', function() { var subscription = new Subscription(PUBSUB, SUB_NAME, options); - assert.strictEqual(subscription.ackDeadline, options.ackDeadline); assert.strictEqual(subscription.maxConnections, options.maxConnections); assert.deepEqual(subscription.flowControl, { From bbef9992032dccab1daa70470674976b10d46ff2 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 16 Aug 2017 15:34:45 -0400 Subject: [PATCH 58/67] increase grpc max recieve message size --- packages/pubsub/src/index.js | 3 ++- packages/pubsub/test/index.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index d48f1226969..48e51f1df35 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -68,7 +68,8 @@ function PubSub(options) { } this.options = extend({ - scopes: v1.ALL_SCOPES + scopes: v1.ALL_SCOPES, + 'grpc.max_receive_message_length': 20000001 }, options); this.determineBaseUrl_(); diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 6dfd37eaad0..4a70ee234b6 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -215,7 +215,8 @@ describe('PubSub', function() { it('should localize the options provided', function() { assert.deepEqual(pubsub.options, extend({ - scopes: v1.ALL_SCOPES + scopes: v1.ALL_SCOPES, + 'grpc.max_receive_message_length': 20000001 }, OPTIONS)); }); From 403f8fe47d2646ff5eba27bddb1014806544b102 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 16 Aug 2017 18:53:50 -0400 Subject: [PATCH 59/67] refactoring connection pool error handling --- packages/pubsub/src/connection-pool.js | 135 ++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 14 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index c3576e0344c..6a4c03219bf 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -28,6 +28,21 @@ var is = require('is'); var util = require('util'); var uuid = require('uuid'); +var CONFIG = require('./v1/subscriber_client_config.json') + .interfaces['google.pubsub.v1.Subscriber']; + +// deadline applied to streams +var STREAM_TIMEOUT = CONFIG.methods.StreamingPull.timeout_millis; + +// codes to retry streams +var RETRY_CODES = [ + 1, // canceled + 4, // deadline exceeded + 8, // resource exhausted + 13, // internal error + 14 // unavailable +]; + /** * ConnectionPool is used to manage the stream connections created via * StreamingPull rpc. @@ -43,14 +58,19 @@ var uuid = require('uuid'); function ConnectionPool(subscription) { this.subscription = subscription; this.connections = new Map(); + this.isPaused = false; this.isOpen = false; + this.failedConnectionAttempts = 0; + this.lastKnownConnection = Date.now(); + this.settings = { maxConnections: subscription.maxConnections || 5, ackDeadline: subscription.ackDeadline || 10000 }; + this.queue = Promise.resolve(); events.EventEmitter.call(this); this.open(); @@ -119,6 +139,7 @@ ConnectionPool.prototype.close = function(callback) { callback = callback || common.util.noop; each(connections, function(connection, onEndCallback) { + connection.removeAllListeners(); connection.end(onEndCallback); }, function(err) { self.connections.clear(); @@ -130,9 +151,16 @@ ConnectionPool.prototype.close = function(callback) { * Creates a connection. This is async but instead of providing a callback * a `connected` event will fire once the connection is ready. */ -ConnectionPool.prototype.createConnection = function() { +ConnectionPool.prototype.createConnection = function(callback) { var self = this; + callback = callback || common.util.noop; + + if (!this.isOpen) { + callback(new Error('Pool is closed.')); + return; + } + this.subscription.request({ client: 'subscriberClient', method: 'streamingPull', @@ -145,29 +173,53 @@ ConnectionPool.prototype.createConnection = function() { var id = uuid.v4(); var connection = requestFn(); + var errorImmediateHandle; connection.on('error', function(err) { - self.emit('error', err); + // since this is a bidi stream it's possible that we recieve errors from + // reads or writes. We also want to try and cut down on the number of + // errors that we emit if other connections are still open. So by using + // setImmediate we're able to cancel the error message if it gets passed + // to the `status` event where we can check if the connection should be + // re-opened or if we should send the error to the user + errorImmediateHandle = setImmediate(self.emit.bind(self), 'error', err); }); - connection.on('data', function(data) { - arrify(data.receivedMessages).forEach(function(message) { - self.emit('message', self.createMessage(id, message)); - }); + connection.on('metadata', function(metadata) { + if (metadata.get('date').length) { + self.lastKnownConnection = Date.now(); + self.failedConnectionAttempts = 0; + connection.isConnected = true; + } }); - connection.once('metadata', function() { - self.emit('connected', connection); - }); + connection.on('status', function(status) { + clearImmediate(errorImmediateHandle); + + if (!connection.isConnected) { + self.failedConnectionAttempts += 1; + } + + connection.removeAllListeners(); + connection.end(); - connection.once('close', function() { self.connections.delete(id); - if (self.isOpen) { - self.createConnection(); + if (self.shouldReconnect(status)) { + self.queueConnection(); + } else if (self.isOpen && !self.connections.size) { + var error = new Error(status.details); + error.code = status.code; + self.emit('error', error); } }); + connection.on('data', function(data) { + arrify(data.receivedMessages).forEach(function(message) { + self.emit('message', self.createMessage(id, message)); + }); + }); + if (self.isPaused) { connection.pause(); } @@ -178,6 +230,7 @@ ConnectionPool.prototype.createConnection = function() { }); self.connections.set(id, connection); + callback(null); }); }; @@ -220,8 +273,11 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { * Creates specified number of connections and puts pool in open state. */ ConnectionPool.prototype.open = function() { - for (var i = 0; i < this.settings.maxConnections; i++) { - this.createConnection(); + var existing = this.connections.size; + var max = this.settings.maxConnections; + + for (; existing < max; existing++) { + this.queueConnection(); } this.isOpen = true; @@ -238,6 +294,26 @@ ConnectionPool.prototype.pause = function() { }); }; +/** + * Queues a connection to be created. If any previous connections have failed, + * it will apply a back off based on the number of failures. + */ +ConnectionPool.prototype.queueConnection = function() { + var self = this; + + var attempts = this.failedConnectionAttempts / this.settings.maxConnections; + var delay = (Math.pow(2, Math.ceil(attempts)) * 1000) + + (Math.floor(Math.random() * 1000)); + + this.queue = this.queue.then(function() { + return common.util.promisify(self.createConnection).call(self); + }).then(function() { + return new Promise(function(resolve) { + setTimeout(resolve, delay); + }); + }); +}; + /** * Calls resume on each connection, allowing `message` events to fire off again. */ @@ -249,4 +325,35 @@ ConnectionPool.prototype.resume = function() { }); }; +/** + * Inspects a status object to determine whether or not we should try and + * reconnect. + * + * @param {object} status - The gRPC status object. + * @return {boolean} + */ +ConnectionPool.prototype.shouldReconnect = function(status) { + // If the pool was closed, we should definitely not reconnect + if (!this.isOpen) { + return false; + } + + // We should check to see if the status code is a non-recoverable error + if (RETRY_CODES.indexOf(status.code) === -1) { + return false; + } + + // deadline exceeded errors are tricky because gax applies a deadline no + // matter what, so it might not be an error at all, the stream could have just + // been closed. That being said we need to check if it is a deadline error + // and if it is only stop retrying if we've failed to make a connection + var elapsed = Date.now() - this.lastKnownConnection; + + if (status.code === 4 && elapsed > (STREAM_TIMEOUT + 300000)) { + return false; + } + + return true; +}; + module.exports = ConnectionPool; From 8cea20931e619f24076dad8414746244edf0172c Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 17 Aug 2017 17:09:36 -0400 Subject: [PATCH 60/67] small cleanup --- packages/pubsub/src/connection-pool.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 6a4c03219bf..970e64f2a20 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -136,15 +136,12 @@ ConnectionPool.prototype.close = function(callback) { var connections = Array.from(this.connections.values()); this.isOpen = false; + self.connections.clear(); callback = callback || common.util.noop; each(connections, function(connection, onEndCallback) { - connection.removeAllListeners(); connection.end(onEndCallback); - }, function(err) { - self.connections.clear(); - callback(err); - }); + }, callback); }; /** @@ -157,7 +154,7 @@ ConnectionPool.prototype.createConnection = function(callback) { callback = callback || common.util.noop; if (!this.isOpen) { - callback(new Error('Pool is closed.')); + callback(); return; } @@ -200,9 +197,7 @@ ConnectionPool.prototype.createConnection = function(callback) { self.failedConnectionAttempts += 1; } - connection.removeAllListeners(); connection.end(); - self.connections.delete(id); if (self.shouldReconnect(status)) { @@ -230,7 +225,7 @@ ConnectionPool.prototype.createConnection = function(callback) { }); self.connections.set(id, connection); - callback(null); + callback(); }); }; From 7fa8118cc59c4f04b306b6d78b72ea3db962c62e Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Fri, 18 Aug 2017 05:46:46 -0400 Subject: [PATCH 61/67] bypass gax for streamingPull rpc --- packages/pubsub/src/connection-pool.js | 98 +++++++++++++++++--------- packages/pubsub/src/index.js | 56 ++++----------- 2 files changed, 78 insertions(+), 76 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 970e64f2a20..31c8a0267f7 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -24,10 +24,13 @@ var arrify = require('arrify'); var common = require('@google-cloud/common'); var each = require('async-each'); var events = require('events'); +var grpc = require('grpc'); var is = require('is'); var util = require('util'); var uuid = require('uuid'); +var v1 = require('./v1'); + var CONFIG = require('./v1/subscriber_client_config.json') .interfaces['google.pubsub.v1.Subscriber']; @@ -63,14 +66,14 @@ function ConnectionPool(subscription) { this.isOpen = false; this.failedConnectionAttempts = 0; - this.lastKnownConnection = Date.now(); + this.noConnectionsTime = Date.now(); this.settings = { maxConnections: subscription.maxConnections || 5, ackDeadline: subscription.ackDeadline || 10000 }; - this.queue = Promise.resolve(); + this.queue = []; events.EventEmitter.call(this); this.open(); @@ -135,9 +138,13 @@ ConnectionPool.prototype.close = function(callback) { var self = this; var connections = Array.from(this.connections.values()); + callback = callback || common.util.noop; + this.isOpen = false; self.connections.clear(); - callback = callback || common.util.noop; + + this.queue.forEach(clearTimeout); + this.queue.length = 0; each(connections, function(connection, onEndCallback) { connection.end(onEndCallback); @@ -148,28 +155,17 @@ ConnectionPool.prototype.close = function(callback) { * Creates a connection. This is async but instead of providing a callback * a `connected` event will fire once the connection is ready. */ -ConnectionPool.prototype.createConnection = function(callback) { +ConnectionPool.prototype.createConnection = function() { var self = this; - callback = callback || common.util.noop; - - if (!this.isOpen) { - callback(); - return; - } - - this.subscription.request({ - client: 'subscriberClient', - method: 'streamingPull', - returnFn: true - }, function(err, requestFn) { + this.getClient(function(err, client) { if (err) { self.emit('error', err); return; } var id = uuid.v4(); - var connection = requestFn(); + var connection = client.streamingPull(); var errorImmediateHandle; connection.on('error', function(err) { @@ -184,21 +180,25 @@ ConnectionPool.prototype.createConnection = function(callback) { connection.on('metadata', function(metadata) { if (metadata.get('date').length) { - self.lastKnownConnection = Date.now(); - self.failedConnectionAttempts = 0; connection.isConnected = true; + self.noConnectionsTime = 0; + self.failedConnectionAttempts = 0; } }); connection.on('status', function(status) { clearImmediate(errorImmediateHandle); + connection.end(); + self.connections.delete(id); + if (!connection.isConnected) { self.failedConnectionAttempts += 1; } - connection.end(); - self.connections.delete(id); + if (!self.connections.size) { + self.noConnectionsTime = Date.now(); + } if (self.shouldReconnect(status)) { self.queueConnection(); @@ -225,7 +225,6 @@ ConnectionPool.prototype.createConnection = function(callback) { }); self.connections.set(id, connection); - callback(); }); }; @@ -264,6 +263,37 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { }; }; +ConnectionPool.prototype.getClient = function(callback) { + if (this.client) { + callback(null, this.client); + return; + } + + var self = this; + var pubsub = this.subscription.pubsub; + + pubsub.auth.getAuthClient(function(err, authClient) { + if (err) { + callback(err); + return; + } + + var credentials = grpc.credentials.combineChannelCredentials( + grpc.credentials.createSsl(), + grpc.credentials.createFromGoogleCredential(authClient) + ); + + var Subscriber = v1(pubsub.options).Subscriber; + + self.client = new Subscriber(v1.SERVICE_ADDRESS, credentials, { + 'grpc.max_send_message_length': -1, // unlimited + 'grpc.max_receive_message_length': -1 // unlimited + }); + + callback(null, self.client); + }); +}; + /** * Creates specified number of connections and puts pool in open state. */ @@ -300,13 +330,13 @@ ConnectionPool.prototype.queueConnection = function() { var delay = (Math.pow(2, Math.ceil(attempts)) * 1000) + (Math.floor(Math.random() * 1000)); - this.queue = this.queue.then(function() { - return common.util.promisify(self.createConnection).call(self); - }).then(function() { - return new Promise(function(resolve) { - setTimeout(resolve, delay); - }); - }); + var timeoutHandle = setTimeout(createConnection, delay); + this.queue.push(timeoutHandle); + + function createConnection() { + self.createConnection(); + self.queue.splice(self.queue.indexOf(timeoutHandle), 1); + } }; /** @@ -338,13 +368,11 @@ ConnectionPool.prototype.shouldReconnect = function(status) { return false; } - // deadline exceeded errors are tricky because gax applies a deadline no - // matter what, so it might not be an error at all, the stream could have just - // been closed. That being said we need to check if it is a deadline error - // and if it is only stop retrying if we've failed to make a connection - var elapsed = Date.now() - this.lastKnownConnection; + var hasNoConnections = !this.connections.size; + var exceededRetryLimit = this.noConnectionsTime && + Date.now() - this.noConnectionsTime > 300000; - if (status.code === 4 && elapsed > (STREAM_TIMEOUT + 300000)) { + if (hasNoConnections && exceededRetryLimit) { return false; } diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index 48e51f1df35..d0c1e0374c4 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -626,60 +626,34 @@ PubSub.prototype.getTopicsStream = common.paginator.streamify('getTopics'); * @param {object} config.gaxOpts - GAX options. * @param {function} config.method - The gax method to call. * @param {object} config.reqOpts - Request options. - * @param {boolean} config.returnFn - Return function as opposed to calling it. * @param {function=} callback - The callback function. */ PubSub.prototype.request = function(config, callback) { var self = this; - if (config.returnFn) { - prepareGaxRequest(callback); - } else { - makeRequestCallback(); + if (global.GCLOUD_SANDBOX_ENV) { + return; } - function prepareGaxRequest(callback) { - if (global.GCLOUD_SANDBOX_ENV) { + self.auth.getProjectId(function(err, projectId) { + if (err) { + callback(err); return; } - self.auth.getProjectId(function(err, projectId) { - if (err) { - callback(err); - return; - } - - var gaxClient = self.api[config.client]; - - if (!gaxClient) { - // Lazily instantiate client. - gaxClient = v1(self.options)[config.client](self.options); - self.api[config.client] = gaxClient; - } - - var reqOpts = extend(true, {}, config.reqOpts); - reqOpts = common.util.replaceProjectIdToken(reqOpts, projectId); + var gaxClient = self.api[config.client]; - var requestFn = gaxClient[config.method].bind( - gaxClient, - reqOpts, - config.gaxOpts - ); - - callback(null, requestFn); - }); - } + if (!gaxClient) { + // Lazily instantiate client. + gaxClient = v1(self.options)[config.client](self.options); + self.api[config.client] = gaxClient; + } - function makeRequestCallback() { - prepareGaxRequest(function(err, requestFn) { - if (err) { - callback(err); - return; - } + var reqOpts = extend(true, {}, config.reqOpts); + reqOpts = common.util.replaceProjectIdToken(reqOpts, projectId); - requestFn(callback); - }); - } + gaxClient[config.method](reqOpts, config.gaxOpts, callback); + }); }; /** From f95b8c407b07fc24c61c77fef482effad520cf1a Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 21 Aug 2017 22:16:00 -0400 Subject: [PATCH 62/67] tweaks/refactors based on pr feedback + stream retry strategy --- packages/pubsub/src/connection-pool.js | 76 ++- packages/pubsub/src/index.js | 16 +- packages/pubsub/src/subscription.js | 8 +- packages/pubsub/src/topic.js | 11 + packages/pubsub/system-test/pubsub.js | 8 +- packages/pubsub/test/connection-pool.js | 704 +++++++++++++++++++----- packages/pubsub/test/index.js | 31 +- packages/pubsub/test/subscription.js | 86 ++- 8 files changed, 740 insertions(+), 200 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index 31c8a0267f7..f9562d1b4a2 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -29,16 +29,12 @@ var is = require('is'); var util = require('util'); var uuid = require('uuid'); +var PKG = require('../package.json'); var v1 = require('./v1'); -var CONFIG = require('./v1/subscriber_client_config.json') - .interfaces['google.pubsub.v1.Subscriber']; - -// deadline applied to streams -var STREAM_TIMEOUT = CONFIG.methods.StreamingPull.timeout_millis; - // codes to retry streams var RETRY_CODES = [ + 0, // ok 1, // canceled 4, // deadline exceeded 8, // resource exhausted @@ -76,6 +72,15 @@ function ConnectionPool(subscription) { this.queue = []; events.EventEmitter.call(this); + // grpc related fields we need since we're bypassing gax + this.metadata_ = new grpc.Metadata(); + + this.metadata_.add('x-goog-api-client', [ + 'gl-node/' + process.versions.node, + 'gccl/' + PKG.version, + 'grpc/' + require('grpc/package.json').version + ].join(' ')); + this.open(); } @@ -165,7 +170,7 @@ ConnectionPool.prototype.createConnection = function() { } var id = uuid.v4(); - var connection = client.streamingPull(); + var connection = client.streamingPull(self.metadata_); var errorImmediateHandle; connection.on('error', function(err) { @@ -179,11 +184,14 @@ ConnectionPool.prototype.createConnection = function() { }); connection.on('metadata', function(metadata) { - if (metadata.get('date').length) { - connection.isConnected = true; - self.noConnectionsTime = 0; - self.failedConnectionAttempts = 0; + if (!metadata.get('date').length) { + return; } + + connection.isConnected = true; + self.noConnectionsTime = 0; + self.failedConnectionAttempts = 0; + self.emit('connected', connection); }); connection.on('status', function(status) { @@ -196,7 +204,7 @@ ConnectionPool.prototype.createConnection = function() { self.failedConnectionAttempts += 1; } - if (!self.connections.size) { + if (!self.isConnected() && !self.noConnectionsTime) { self.noConnectionsTime = Date.now(); } @@ -241,7 +249,7 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { var pt = resp.message.publishTime; var milliseconds = parseInt(pt.nanos, 10) / 1e6; - var data = resp.message.data; + var originalDataLength = resp.message.data.length; return { connectionId: connectionId, @@ -250,9 +258,10 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { attributes: resp.message.attributes, publishTime: new Date(parseInt(pt.seconds, 10) * 1000 + milliseconds), received: Date.now(), + data: resp.message.data, // using get here to prevent user from overwriting data - get data() { - return data; + get length() { + return originalDataLength; }, ack: function() { self.subscription.ack_(this); @@ -263,6 +272,14 @@ ConnectionPool.prototype.createMessage = function(connectionId, resp) { }; }; +/** + * Gets the Subscriber client. We need to bypass GAX until they allow deadlines + * to be optional. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error occurred while getting the client. + * @param {object} callback.client - The Subscriber client. + */ ConnectionPool.prototype.getClient = function(callback) { if (this.client) { callback(null, this.client); @@ -286,14 +303,33 @@ ConnectionPool.prototype.getClient = function(callback) { var Subscriber = v1(pubsub.options).Subscriber; self.client = new Subscriber(v1.SERVICE_ADDRESS, credentials, { - 'grpc.max_send_message_length': -1, // unlimited - 'grpc.max_receive_message_length': -1 // unlimited + 'grpc.max_receive_message_length': 20000001, + 'grpc.primary_user_agent': common.util.getUserAgentFromPackageJson(PKG) }); callback(null, self.client); }); }; +/** + * Check to see if at least one stream in the pool is connected. + * @return {boolean} + */ +ConnectionPool.prototype.isConnected = function() { + var interator = this.connections.values(); + var connection = interator.next().value; + + while (connection) { + if (connection.isConnected) { + return true; + } + + connection = interator.next().value; + } + + return false; +}; + /** * Creates specified number of connections and puts pool in open state. */ @@ -326,8 +362,7 @@ ConnectionPool.prototype.pause = function() { ConnectionPool.prototype.queueConnection = function() { var self = this; - var attempts = this.failedConnectionAttempts / this.settings.maxConnections; - var delay = (Math.pow(2, Math.ceil(attempts)) * 1000) + + var delay = (Math.pow(2, this.failedConnectionAttempts) * 1000) + (Math.floor(Math.random() * 1000)); var timeoutHandle = setTimeout(createConnection, delay); @@ -368,11 +403,10 @@ ConnectionPool.prototype.shouldReconnect = function(status) { return false; } - var hasNoConnections = !this.connections.size; var exceededRetryLimit = this.noConnectionsTime && Date.now() - this.noConnectionsTime > 300000; - if (hasNoConnections && exceededRetryLimit) { + if (exceededRetryLimit) { return false; } diff --git a/packages/pubsub/src/index.js b/packages/pubsub/src/index.js index d0c1e0374c4..e70bc171d63 100644 --- a/packages/pubsub/src/index.js +++ b/packages/pubsub/src/index.js @@ -25,6 +25,7 @@ var extend = require('extend'); var googleAuth = require('google-auto-auth'); var is = require('is'); +var PKG = require('../package.json'); var v1 = require('./v1'); /** @@ -69,7 +70,9 @@ function PubSub(options) { this.options = extend({ scopes: v1.ALL_SCOPES, - 'grpc.max_receive_message_length': 20000001 + 'grpc.max_receive_message_length': 20000001, + libName: 'gccl', + libVersion: PKG.version }, options); this.determineBaseUrl_(); @@ -685,6 +688,17 @@ PubSub.prototype.snapshot = function(name) { * * @param {string} name - Name of the subscription. * @param {object=} options - Configuration object. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. Note that these options do not persist across + * subscription instances. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. + * @param {number} options.maxConnections - Use this to limit the number of + * connections to be used when sending and receiving messages. Default: 5. * @return {module:pubsub/subscription} * * @example diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index fe4cac321a7..dfd8a39a419 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -258,7 +258,7 @@ Subscription.prototype.ack_ = function(message) { this.breakLease_(message); this.histogram.add(Date.now() - message.received); - if (!this.connectionPool) { + if (!this.connectionPool || !this.connectionPool.isConnected()) { this.inventory_.ack.push(message.ackId); this.setFlushTimeout_(); return; @@ -291,7 +291,7 @@ Subscription.prototype.breakLease_ = function(message) { var messageIndex = this.inventory_.lease.indexOf(message.ackId); this.inventory_.lease.splice(messageIndex, 1); - this.inventory_.bytes -= message.data.length; + this.inventory_.bytes -= message.length; if (this.connectionPool) { if (this.connectionPool.isPaused && !this.hasMaxMessages_()) { @@ -622,7 +622,7 @@ Subscription.prototype.hasMaxMessages_ = function() { */ Subscription.prototype.leaseMessage_ = function(message) { this.inventory_.lease.push(message.ackId); - this.inventory_.bytes += message.data.length; + this.inventory_.bytes += message.length; this.setLeaseTimeout_(); return message; @@ -728,7 +728,7 @@ Subscription.prototype.modifyPushConfig = function(config, gaxOpts, callback) { Subscription.prototype.nack_ = function(message) { this.breakLease_(message); - if (!this.connectionPool) { + if (!this.connectionPool || !this.connectionPool.isConnected()) { this.inventory_.nack.push(message.ackId); this.setFlushTimeout_(); return; diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index cd18b773ec2..0dcaafe4eb1 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -415,6 +415,17 @@ Topic.prototype.publisher = function(options) { * * @param {string} name - Name of the subscription. * @param {object=} options - Configuration object. + * @param {object} options.flowControl - Flow control configurations for + * receiving messages. Note that these options do not persist across + * subscription instances. + * @param {number} options.flowControl.maxBytes - The maximum number of bytes + * in un-acked messages to allow before the subscription pauses incoming + * messages. Defaults to 20% of free memory. + * @param {number} options.flowControl.maxMessages - The maximum number of + * un-acked messages to allow before the subscription pauses incoming + * messages. Default: Infinity. + * @param {number} options.maxConnections - Use this to limit the number of + * connections to be used when sending and receiving messages. Default: 5. * @return {module:pubsub/subscription} * * @example diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index dc6c4284c72..450e1dce683 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -532,7 +532,9 @@ describe('pubsub', function() { subscription.on('error', done); subscription.on('message', function(message) { - assert.strictEqual(message.id, messageId); + if (message.id !== messageId) { + return; + } message.ack(); @@ -554,7 +556,9 @@ describe('pubsub', function() { subscription.on('error', done); subscription.on('message', function(message) { - assert.strictEqual(message.id, messageId); + if (message.id !== messageId) { + return; + } message.ack(); diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index 6f67984124d..94f45bdc490 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -20,19 +20,64 @@ var assert = require('assert'); var common = require('@google-cloud/common'); var events = require('events'); var extend = require('extend'); +var grpc = require('grpc'); var proxyquire = require('proxyquire'); var uuid = require('uuid'); +var util = require('util'); + +var PKG = require('../package.json'); +var v1 = require('../src/v1'); var fakeUtil = extend({}, common.util); var fakeUuid = extend({}, uuid); +var fakeGrpc = extend({}, grpc); + +var v1Override; +function fakeV1(options) { + return (v1Override || v1)(options); +} + +function FakeConnection() { + this.isConnected = false; + this.isPaused = false; + this.ended = false; + + events.EventEmitter.call(this); +} + +util.inherits(FakeConnection, events.EventEmitter); + +FakeConnection.prototype.write = function() {}; + +FakeConnection.prototype.end = function() { + this.ended = true; +}; + +FakeConnection.prototype.pause = function() { + this.isPaused = true; +}; + +FakeConnection.prototype.resume = function() { + this.isPaused = false; +}; describe('ConnectionPool', function() { var ConnectionPool; var pool; + var FAKE_PUBSUB_OPTIONS = {}; + + var PUBSUB = { + auth: { + getAuthClient: fakeUtil.noop + }, + options: FAKE_PUBSUB_OPTIONS + }; + var SUB_NAME = 'test-subscription'; var SUBSCRIPTION = { name: SUB_NAME, + pubsub: PUBSUB, request: fakeUtil.noop }; @@ -41,42 +86,53 @@ describe('ConnectionPool', function() { '@google-cloud/common': { util: fakeUtil }, - uuid: fakeUuid + grpc: fakeGrpc, + uuid: fakeUuid, + './v1': fakeV1 }); }); beforeEach(function() { SUBSCRIPTION.request = fakeUtil.noop; + PUBSUB.auth.getAuthClient = fakeUtil.noop; + pool = new ConnectionPool(SUBSCRIPTION); }); - describe('initialization', function() { - it('should localize the subscription', function() { - assert.strictEqual(pool.subscription, SUBSCRIPTION); - }); - - it('should create a map for the connections', function() { - assert(pool.connections instanceof Map); - }); - - it('should set isPaused to false', function() { - assert.strictEqual(pool.isPaused, false); - }); + afterEach(function() { + if (pool.isOpen) { + pool.close(); + } + }); - it('should set isOpen to false', function() { + describe('initialization', function() { + it('should initialize internally used properties', function() { var open = ConnectionPool.prototype.open; - - ConnectionPool.prototype.open = function() { - ConnectionPool.prototype.open = open; - }; + ConnectionPool.prototype.open = fakeUtil.noop; var pool = new ConnectionPool(SUBSCRIPTION); - assert.strictEqual(pool.isOpen, false); - }); - it('should provide default settings', function() { + assert.strictEqual(pool.subscription, SUBSCRIPTION); + assert(pool.connections instanceof Map); + assert.strictEqual(pool.isPaused, false); + assert.strictEqual(pool.isOpen, false); assert.strictEqual(pool.settings.maxConnections, 5); assert.strictEqual(pool.settings.ackDeadline, 10000); + assert.deepEqual(pool.queue, []); + + ConnectionPool.prototype.open = open; + }); + + it('should create grpc metadata', function() { + assert(pool.metadata_ instanceof grpc.Metadata); + + assert.deepEqual(pool.metadata_.get('x-goog-api-client'), [ + [ + 'gl-node/' + process.versions.node, + 'gccl/' + PKG.version, + 'grpc/' + require('grpc/package.json').version + ].join(' ') + ]); }); it('should respect user specified settings', function() { @@ -124,10 +180,10 @@ describe('ConnectionPool', function() { it('should return a specified connection', function(done) { var id = 'a'; - var fakeConnection = {}; + var fakeConnection = new FakeConnection(); pool.connections.set(id, fakeConnection); - pool.connections.set('b', {}); + pool.connections.set('b', new FakeConnection()); pool.acquire(id, function(err, connection) { assert.ifError(err); @@ -137,7 +193,7 @@ describe('ConnectionPool', function() { }); it('should return any conn when the specified is missing', function(done) { - var fakeConnection = {}; + var fakeConnection = new FakeConnection(); pool.connections.set('a', fakeConnection); @@ -149,7 +205,7 @@ describe('ConnectionPool', function() { }); it('should return any connection when id is missing', function(done) { - var fakeConnection = {}; + var fakeConnection = new FakeConnection(); pool.connections.set('a', fakeConnection); @@ -161,7 +217,7 @@ describe('ConnectionPool', function() { }); it('should listen for connected event if no conn is ready', function(done) { - var fakeConnection = {}; + var fakeConnection = new FakeConnection(); pool.acquire(function(err, connection) { assert.ifError(err); @@ -180,15 +236,6 @@ describe('ConnectionPool', function() { }); it('should call end on all active connections', function() { - function FakeConnection() { - this.endCalled = false; - } - - FakeConnection.prototype.end = function(cb) { - this.endCalled = true; - cb(); - }; - var a = new FakeConnection(); var b = new FakeConnection(); @@ -197,8 +244,8 @@ describe('ConnectionPool', function() { pool.close(); - assert.strictEqual(a.endCalled, true); - assert.strictEqual(b.endCalled, true); + assert.strictEqual(a.ended, true); + assert.strictEqual(b.ended, true); }); it('should clear the connections map', function(done) { @@ -206,6 +253,25 @@ describe('ConnectionPool', function() { pool.close(); }); + it('should clear any timeouts in the queue', function() { + var _clearTimeout = global.clearTimeout; + var clearCalls = 0; + + var fakeHandles = ['a', 'b', 'c', 'd']; + + global.clearTimeout = function(handle) { + assert.strictEqual(handle, fakeHandles[clearCalls++]); + }; + + pool.queue = Array.from(fakeHandles); + pool.close(); + + assert.strictEqual(clearCalls, fakeHandles.length); + assert.strictEqual(pool.queue.length, 0); + + global.clearTimeout = _clearTimeout; + }); + it('should exec a callback when finished closing', function(done) { pool.close(done); }); @@ -221,21 +287,37 @@ describe('ConnectionPool', function() { }); describe('createConnection', function() { + var fakeClient; + var fakeConnection; + + beforeEach(function() { + fakeConnection = new FakeConnection(); + + fakeClient = { + streamingPull: function() { + return fakeConnection; + } + }; + + pool.getClient = function(callback) { + callback(null, fakeClient); + }; + }); + it('should make the correct request', function(done) { - pool.subscription.request = function(config) { - assert.strictEqual(config.client, 'subscriberClient'); - assert.strictEqual(config.method, 'streamingPull'); - assert(config.returnFn); - done(); + fakeClient.streamingPull = function(metadata) { + assert.strictEqual(metadata, pool.metadata_); + setImmediate(done); + return fakeConnection; }; pool.createConnection(); }); - it('should emit any error that occurs', function(done) { + it('should emit any errors that occur when getting client', function(done) { var error = new Error('err'); - pool.subscription.request = function(config, callback) { + pool.getClient = function(callback) { callback(error); }; @@ -248,24 +330,14 @@ describe('ConnectionPool', function() { }); describe('connection', function() { - var fakeConnection; var fakeId; beforeEach(function() { - fakeConnection = new events.EventEmitter(); - fakeConnection.write = fakeUtil.noop; - fakeId = uuid.v4(); fakeUuid.v4 = function() { return fakeId; }; - - pool.subscription.request = function(config, callback) { - callback(null, function() { - return fakeConnection; - }); - }; }); it('should create a connection', function(done) { @@ -285,83 +357,202 @@ describe('ConnectionPool', function() { pool.createConnection(); }); - it('should emit errors to the pool', function(done) { - var error = new Error('err'); + it('should pause the connection if the pool is paused', function(done) { + fakeConnection.pause = done; + pool.isPaused = true; + pool.createConnection(); + }); - pool.on('error', function(err) { - assert.strictEqual(err, error); - done(); - }); + describe('error events', function() { + it('should emit errors to the pool', function(done) { + var error = new Error('err'); - pool.createConnection(); - fakeConnection.emit('error', error); + pool.on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('error', error); + }); }); - it('should emit messages', function(done) { - var fakeResp = {}; - var fakeMessage = {}; + describe('metadata events', function() { + it('should do nothing if the metadata is empty', function(done) { + var metadata = new grpc.Metadata(); - pool.createMessage = function(id, resp) { - assert.strictEqual(id, fakeId); - assert.strictEqual(resp, fakeResp); - return fakeMessage; - }; + pool.on('connected', done); // should not fire + pool.createConnection(); - pool.on('message', function(message) { - assert.strictEqual(message, fakeMessage); + fakeConnection.emit('metadata', metadata); done(); }); - pool.createConnection(); - fakeConnection.emit('data', { receivedMessages: [fakeResp] }); + it('should reset counters and fire connected', function(done) { + var metadata = new grpc.Metadata(); + + metadata.set('date', 'abc'); + + pool.on('connected', function(connection) { + assert.strictEqual(connection, fakeConnection); + assert(fakeConnection.isConnected); + assert.strictEqual(pool.noConnectionsTime, 0); + assert.strictEqual(pool.failedConnectionAttempts, 0); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('metadata', metadata); + }); }); - it('should emit connected when ready', function(done) { - pool.on('connected', function(connection) { - assert.strictEqual(connection, fakeConnection); + describe('status events', function() { + beforeEach(function() { + pool.connections.set('a', new FakeConnection()); + }); + + it('should cancel any error events', function(done) { + var fakeError = { code: 4 }; + + pool.on('error', done); // should not fire + pool.createConnection(); + + fakeConnection.emit('error', fakeError); + fakeConnection.emit('status', fakeError); + done(); }); - pool.createConnection(); - fakeConnection.emit('metadata'); - }); + it('should close and delete the connection', function(done) { + var endCalled = false; - it('should create a new connection if the pool is open', function(done) { - var deleted = false; + pool.createConnection(); - pool.connections.delete = function(id) { - assert.strictEqual(id, fakeId); - deleted = true; - }; + fakeConnection.end = function() { + endCalled = true; + }; - pool.createConnection(); + pool.connections.delete = function(id) { + assert.strictEqual(id, fakeId); + done(); + }; - pool.createConnection = function() { - assert(deleted); - done(); - }; + fakeConnection.emit('status', {}); + }); - pool.isOpen = true; - fakeConnection.emit('close'); - }); + it('should increment the failed connection counter', function() { + pool.failedConnectionAttempts = 0; + fakeConnection.isConnected = false; - it('should not create a conn if the pool is closed', function(done) { - pool.createConnection(); + pool.createConnection(); + fakeConnection.emit('status', {}); - pool.connections.delete = function() { - done(); - }; + assert.strictEqual(pool.failedConnectionAttempts, 1); + }); + + it('should not incr. the failed connection counter', function() { + pool.failedConnectionAttempts = 0; + fakeConnection.isConnected = true; + + pool.createConnection(); + fakeConnection.emit('status', {}); + + assert.strictEqual(pool.failedConnectionAttempts, 0); + }); + + it('should capture the date when no connections are found', function() { + pool.noConnectionsTime = 0; + pool.isConnected = function() { + return false; + }; + + pool.createConnection(); + fakeConnection.emit('status', {}); + + assert.strictEqual(pool.noConnectionsTime, Date.now()); + }); + + it('should not capture the date when already set', function() { + pool.noConnectionsTime = 123; + pool.isConnected = function() { + return false; + }; + + pool.createConnection(); + fakeConnection.emit('status', {}); + + assert.strictEqual(pool.noConnectionsTime, 123); + }); + + it('should not capture the date if a conn. is found', function() { + pool.noConnectionsTime = 0; + pool.isConnected = function() { + return true; + }; - pool.createConnection = done; // should not be called + pool.createConnection(); + fakeConnection.emit('status', {}); - pool.isOpen = false; - fakeConnection.emit('close'); + assert.strictEqual(pool.noConnectionsTime, 0); + }); + + it('should queue a connection if status is retryable', function(done) { + var fakeStatus = {}; + + pool.shouldReconnect = function(status) { + assert.strictEqual(status, fakeStatus); + return true; + }; + + pool.queueConnection = done; + + pool.createConnection(); + fakeConnection.emit('status', fakeStatus); + }); + + it('should emit error if no pending conn. are found', function(done) { + var error = { + code: 4, + details: 'Deadline Exceeded' + }; + + pool.shouldReconnect = function() { + return false; + }; + + // will only emit status errors if pool is empty + pool.connections = new Map(); + + pool.on('error', function(err) { + assert.strictEqual(err.code, error.code); + assert.strictEqual(err.message, error.details); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('status', error); + }); }); - it('should pause the connection if the pool is paused', function(done) { - fakeConnection.pause = done; - pool.isPaused = true; - pool.createConnection(); + describe('data events', function() { + it('should emit messages', function(done) { + var fakeResp = {}; + var fakeMessage = {}; + + pool.createMessage = function(id, resp) { + assert.strictEqual(id, fakeId); + assert.strictEqual(resp, fakeResp); + return fakeMessage; + }; + + pool.on('message', function(message) { + assert.strictEqual(message, fakeMessage); + done(); + }); + + pool.createConnection(); + fakeConnection.emit('data', { receivedMessages: [fakeResp] }); + }); }); }); }); @@ -421,6 +612,14 @@ describe('ConnectionPool', function() { assert.strictEqual(message.received, FAKE_DATE_NOW); }); + it('should create a read-only message length property', function() { + assert.strictEqual(message.length, RESP.message.data.length); + + assert.throws(function() { + message.length = 3; + }); + }); + it('should create an ack method', function(done) { SUBSCRIPTION.ack_ = function(message_) { assert.strictEqual(message_, message); @@ -438,20 +637,165 @@ describe('ConnectionPool', function() { message.nack(); }); + }); - it('should not allow data to be overridden', function() { - assert.throws(function() { - message.data = 'hihihi'; + describe('getClient', function() { + var fakeAuthClient = {}; + + function FakeSubscriber(address, creds, options) { + this.address = address; + this.creds = creds; + this.options = options; + } + + beforeEach(function() { + PUBSUB.auth.getAuthClient = function(callback) { + callback(null, fakeAuthClient); + }; + + v1Override = function() { + return { + Subscriber: FakeSubscriber + }; + }; + }); + + it('should return the cached client when available', function(done) { + var fakeClient = pool.client = {}; + + pool.getClient(function(err, client) { + assert.ifError(err); + assert.strictEqual(client, fakeClient); + done(); + }); + }); + + it('should return any auth errors', function(done) { + var error = new Error('err'); + + PUBSUB.auth.getAuthClient = function(callback) { + callback(error); + }; + + pool.getClient(function(err, client) { + assert.strictEqual(err, error); + assert.strictEqual(client, undefined); + done(); + }); + }); + + it('should create/use grpc credentials', function(done) { + var fakeSslCreds = {}; + var fakeGoogCreds = {}; + var fakeCombinedCreds = {}; + + fakeGrpc.credentials = { + createSsl: function() { + return fakeSslCreds; + }, + createFromGoogleCredential: function(authClient) { + assert.strictEqual(authClient, fakeAuthClient); + return fakeGoogCreds; + }, + combineChannelCredentials: function(sslCreds, googCreds) { + assert.strictEqual(sslCreds, fakeSslCreds); + assert.strictEqual(googCreds, fakeGoogCreds); + return fakeCombinedCreds; + } + }; + + pool.getClient(function(err, client) { + assert.ifError(err); + assert(client instanceof FakeSubscriber); + assert.strictEqual(client.creds, fakeCombinedCreds); + done(); + }); + }); + + it('should pass the pubsub options into the gax fn', function(done) { + v1Override = function(options) { + assert.strictEqual(options, FAKE_PUBSUB_OPTIONS); + setImmediate(done); + + return { + Subscriber: FakeSubscriber + }; + }; + + pool.getClient(assert.ifError); + }); + + it('should pass in the correct the args to the Subscriber', function(done) { + var fakeCreds = {}; + fakeGrpc.credentials.combineChannelCredentials = function() { + return fakeCreds; + }; + + var fakeAddress = 'a.b.c'; + fakeV1.SERVICE_ADDRESS = fakeAddress; + + var fakeUserAgent = 'a-b-c'; + fakeUtil.getUserAgentFromPackageJson = function(packageJson) { + assert.deepEqual(packageJson, PKG); + return fakeUserAgent; + }; + + pool.getClient(function(err, client) { + assert.ifError(err); + assert(client instanceof FakeSubscriber); + assert.strictEqual(client.address, fakeAddress); + assert.strictEqual(client.creds, fakeCreds); + + assert.deepEqual(client.options, { + 'grpc.max_receive_message_length': 20000001, + 'grpc.primary_user_agent': fakeUserAgent + }); + + done(); }); }); }); + describe('isConnected', function() { + it('should return true when at least one stream is connected', function() { + var connections = pool.connections = new Map(); + + connections.set('a', new FakeConnection()); + connections.set('b', new FakeConnection()); + connections.set('c', new FakeConnection()); + connections.set('d', new FakeConnection()); + + var conn = new FakeConnection(); + conn.isConnected = true; + connections.set('e', conn); + + assert(pool.isConnected()); + }); + + it('should return false when there is no connection', function() { + var connections = pool.connections = new Map(); + + connections.set('a', new FakeConnection()); + connections.set('b', new FakeConnection()); + connections.set('c', new FakeConnection()); + connections.set('d', new FakeConnection()); + connections.set('e', new FakeConnection()); + + assert(!pool.isConnected()); + }); + + it('should return false when the map is empty', function() { + pool.connections = new Map(); + assert(!pool.isConnected()); + }); + }); + describe('open', function() { it('should make the specified number of connections', function() { var expectedCount = 5; var connectionCount = 0; - pool.createConnection = function() { + pool.queueConnection = function() { connectionCount += 1; }; @@ -474,14 +818,6 @@ describe('ConnectionPool', function() { }); it('should pause all the connections', function() { - function FakeConnection() { - this.isPaused = false; - } - - FakeConnection.prototype.pause = function() { - this.isPaused = true; - }; - var a = new FakeConnection(); var b = new FakeConnection(); @@ -495,6 +831,80 @@ describe('ConnectionPool', function() { }); }); + describe('queueConnection', function() { + var fakeTimeoutHandle = 123; + + var _setTimeout; + var _random; + var _open; + + before(function() { + _setTimeout = global.setTimeout; + _random = global.Math.random; + + _open = ConnectionPool.prototype.open; + // prevent open from calling queueConnection + ConnectionPool.prototype.open = fakeUtil.noop; + }); + + beforeEach(function() { + Math.random = function() { + return 1; + }; + + global.setTimeout = function(cb) { + cb(); + return fakeTimeoutHandle; + }; + + pool.failedConnectionAttempts = 0; + pool.createConnection = fakeUtil.noop; + }); + + after(function() { + global.setTimeout = _setTimeout; + global.Math.random = _random; + ConnectionPool.prototype.open = _open; + }); + + it('should set a timeout to create the connection', function(done) { + pool.createConnection = done; + + global.setTimeout = function(cb, delay) { + assert.strictEqual(delay, 2000); + cb(); // should call the done fn + }; + + pool.queueConnection(); + }); + + it('should factor in the number of failed requests', function(done) { + pool.createConnection = done; + pool.failedConnectionAttempts = 3; + + global.setTimeout = function(cb, delay) { + assert.strictEqual(delay, 9000); + cb(); // should call the done fn + }; + + pool.queueConnection(); + }); + + it('should capture the timeout handle', function() { + pool.queueConnection(); + assert.deepEqual(pool.queue, [fakeTimeoutHandle]); + }); + + it('should remove the timeout handle once it fires', function(done) { + pool.createConnection = function() { + assert.strictEqual(pool.queue.length, 0); + done(); + }; + + pool.queueConnection(); + }); + }); + describe('resume', function() { it('should set the isPaused flag to false', function() { pool.resume(); @@ -502,14 +912,6 @@ describe('ConnectionPool', function() { }); it('should resume all the connections', function() { - function FakeConnection() { - this.isPaused = true; - } - - FakeConnection.prototype.resume = function() { - this.isPaused = false; - }; - var a = new FakeConnection(); var b = new FakeConnection(); @@ -522,4 +924,54 @@ describe('ConnectionPool', function() { assert.strictEqual(b.isPaused, false); }); }); + + describe('shouldReconnect', function() { + it('should not reconnect if the pool is closed', function() { + pool.isOpen = false; + assert.strictEqual(pool.shouldReconnect({}), false); + }); + + it('should return true for retryable errors', function() { + assert(pool.shouldReconnect({ code: 0 })); // OK + assert(pool.shouldReconnect({ code: 1 })); // Canceled + assert(pool.shouldReconnect({ code: 4 })); // DeadlineExceeded + assert(pool.shouldReconnect({ code: 8 })); // ResourceExhausted + assert(pool.shouldReconnect({ code: 13 })); // Internal + assert(pool.shouldReconnect({ code: 14 })); // Unavailable + }); + + it('should return false for non-retryable errors', function() { + assert(!pool.shouldReconnect({ code: 2 })); // Unknown + assert(!pool.shouldReconnect({ code: 3 })); // InvalidArgument + assert(!pool.shouldReconnect({ code: 5 })); // NotFound + assert(!pool.shouldReconnect({ code: 6 })); // AlreadyExists + assert(!pool.shouldReconnect({ code: 7 })); // PermissionDenied + assert(!pool.shouldReconnect({ code: 9 })); // FailedPrecondition + assert(!pool.shouldReconnect({ code: 10 })); // Aborted + assert(!pool.shouldReconnect({ code: 11 })); // OutOfRange + assert(!pool.shouldReconnect({ code: 12 })); // Unimplemented + assert(!pool.shouldReconnect({ code: 15 })); // DataLoss + assert(!pool.shouldReconnect({ code: 16 })); // Unauthenticated + }); + + it('should not retry if no connection can be made', function() { + var fakeStatus = { + code: 4 + }; + + pool.noConnectionsTime = Date.now() - 300001; + + assert.strictEqual(pool.shouldReconnect(fakeStatus), false); + }); + + it('should return true if all conditions are met', function() { + var fakeStatus = { + code: 4 + }; + + pool.noConnectionsTime = 0; + + assert.strictEqual(pool.shouldReconnect(fakeStatus), true); + }); + }); }); diff --git a/packages/pubsub/test/index.js b/packages/pubsub/test/index.js index 4a70ee234b6..a7fc25c9d67 100644 --- a/packages/pubsub/test/index.js +++ b/packages/pubsub/test/index.js @@ -22,6 +22,7 @@ var extend = require('extend'); var proxyquire = require('proxyquire'); var util = require('@google-cloud/common').util; +var PKG = require('../package.json'); var v1 = require('../src/v1/index.js'); var SubscriptionCached = require('../src/subscription.js'); @@ -204,7 +205,10 @@ describe('PubSub', function() { googleAutoAuthOverride = function(options_) { assert.deepEqual(options_, extend({ - scopes: v1.ALL_SCOPES + scopes: v1.ALL_SCOPES, + 'grpc.max_receive_message_length': 20000001, + libName: 'gccl', + libVersion: PKG.version }, options)); return fakeGoogleAutoAuthInstance; }; @@ -216,7 +220,9 @@ describe('PubSub', function() { it('should localize the options provided', function() { assert.deepEqual(pubsub.options, extend({ scopes: v1.ALL_SCOPES, - 'grpc.max_receive_message_length': 20000001 + 'grpc.max_receive_message_length': 20000001, + libName: 'gccl', + libVersion: PKG.version }, OPTIONS)); }); @@ -998,16 +1004,6 @@ describe('PubSub', function() { pubsub.request(CONFIG, assert.ifError); }); - it('should call the specified method', function(done) { - pubsub.api.fakeClient.fakeMethod = function(reqOpts, gaxOpts, callback) { - assert.deepEqual(reqOpts, CONFIG.reqOpts); - assert.strictEqual(gaxOpts, CONFIG.gaxOpts); - callback(); // the done function - }; - - pubsub.request(CONFIG, done); - }); - it('should instantiate the client lazily', function(done) { var fakeClientInstance = { fakeMethod: function(reqOpts, gaxOpts, callback) { @@ -1031,17 +1027,6 @@ describe('PubSub', function() { pubsub.request(CONFIG, done); }); - it('should return the rpc function when returnFn is set', function(done) { - var config = extend({ - returnFn: true - }, CONFIG); - - pubsub.request(config, function(err, requestFn) { - assert.ifError(err); - requestFn(done); - }); - }); - it('should do nothing if sandbox env var is set', function(done) { global.GCLOUD_SANDBOX_ENV = true; pubsub.request(CONFIG, done); // should not fire done diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index b4bbb36ea06..1d0b2f6c610 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -281,10 +281,33 @@ describe('Subscription', function() { }); describe('with connection pool', function() { + var pool; + beforeEach(function() { subscription.setFlushTimeout_ = function() { throw new Error('Should not be called.'); }; + + pool = { + isConnected: function() { + return true; + } + }; + + subscription.connectionPool = pool; + }); + + it('should set a timeout if pool is not connected', function(done) { + subscription.setFlushTimeout_ = function() { + assert.deepEqual(subscription.inventory_.ack, [MESSAGE.ackId]); + done(); + }; + + pool.isConnected = function() { + return false; + }; + + subscription.ack_(MESSAGE); }); it('should write to the connection it came in on', function(done) { @@ -295,11 +318,9 @@ describe('Subscription', function() { } }; - subscription.connectionPool = { - acquire: function(connectionId, callback) { - assert.strictEqual(connectionId, MESSAGE.connectionId); - callback(null, fakeConnection); - } + pool.acquire = function(connectionId, callback) { + assert.strictEqual(connectionId, MESSAGE.connectionId); + callback(null, fakeConnection); }; subscription.ack_(MESSAGE); @@ -308,10 +329,8 @@ describe('Subscription', function() { it('should emit an error when unable to get a conn', function(done) { var error = new Error('err'); - subscription.connectionPool = { - acquire: function(connectionId, callback) { - callback(error); - } + pool.acquire = function(connectionId, callback) { + callback(error); }; subscription.on('error', function(err) { @@ -327,17 +346,18 @@ describe('Subscription', function() { describe('breakLease_', function() { var MESSAGE = { ackId: 'abc', - data: new Buffer('hello') + data: new Buffer('hello'), + length: 5 }; beforeEach(function() { subscription.inventory_.lease.push(MESSAGE.ackId); - subscription.inventory_.bytes += MESSAGE.data.length; + subscription.inventory_.bytes += MESSAGE.length; }); it('should remove the message from the lease array', function() { assert.strictEqual(subscription.inventory_.lease.length, 1); - assert.strictEqual(subscription.inventory_.bytes, MESSAGE.data.length); + assert.strictEqual(subscription.inventory_.bytes, MESSAGE.length); subscription.breakLease_(MESSAGE); @@ -911,7 +931,8 @@ describe('Subscription', function() { describe('leaseMessage_', function() { var MESSAGE = { ackId: 'abc', - data: new Buffer('hello') + data: new Buffer('hello'), + length: 5 }; beforeEach(function() { @@ -926,7 +947,7 @@ describe('Subscription', function() { it('should update the byte count', function() { assert.strictEqual(subscription.inventory_.bytes, 0); subscription.leaseMessage_(MESSAGE); - assert.strictEqual(subscription.inventory_.bytes, MESSAGE.data.length); + assert.strictEqual(subscription.inventory_.bytes, MESSAGE.length); }); it('should begin auto-leasing', function(done) { @@ -1071,10 +1092,33 @@ describe('Subscription', function() { }); describe('with connection pool', function() { + var pool; + beforeEach(function() { subscription.setFlushTimeout_ = function() { throw new Error('Should not be called.'); }; + + pool = { + isConnected: function() { + return true; + } + }; + + subscription.connectionPool = pool; + }); + + it('should not write to the pool if not connected', function(done) { + subscription.setFlushTimeout_ = function() { + assert.deepEqual(subscription.inventory_.nack, [MESSAGE.ackId]); + done(); + }; + + pool.isConnected = function() { + return false; + }; + + subscription.nack_(MESSAGE); }); it('should write to the connection it came in on', function(done) { @@ -1088,11 +1132,9 @@ describe('Subscription', function() { } }; - subscription.connectionPool = { - acquire: function(connectionId, callback) { - assert.strictEqual(connectionId, MESSAGE.connectionId); - callback(null, fakeConnection); - } + pool.acquire = function(connectionId, callback) { + assert.strictEqual(connectionId, MESSAGE.connectionId); + callback(null, fakeConnection); }; subscription.nack_(MESSAGE); @@ -1101,10 +1143,8 @@ describe('Subscription', function() { it('should emit an error when unable to get a conn', function(done) { var error = new Error('err'); - subscription.connectionPool = { - acquire: function(connectionId, callback) { - callback(error); - } + pool.acquire = function(connectionId, callback) { + callback(error); }; subscription.on('error', function(err) { From 194a03e7b85f5530d211179b9a4aaba5c329bcb1 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 23 Aug 2017 19:34:00 -0400 Subject: [PATCH 63/67] adjustments per pr feedback --- packages/pubsub/src/connection-pool.js | 15 +++++++++---- packages/pubsub/test/connection-pool.js | 30 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/pubsub/src/connection-pool.js b/packages/pubsub/src/connection-pool.js index f9562d1b4a2..4899f953054 100644 --- a/packages/pubsub/src/connection-pool.js +++ b/packages/pubsub/src/connection-pool.js @@ -62,7 +62,7 @@ function ConnectionPool(subscription) { this.isOpen = false; this.failedConnectionAttempts = 0; - this.noConnectionsTime = Date.now(); + this.noConnectionsTime = 0; this.settings = { maxConnections: subscription.maxConnections || 5, @@ -147,9 +147,11 @@ ConnectionPool.prototype.close = function(callback) { this.isOpen = false; self.connections.clear(); - this.queue.forEach(clearTimeout); + this.queue.length = 0; + this.failedConnectionAttempts = 0; + this.noConnectionsTime = 0; each(connections, function(connection, onEndCallback) { connection.end(onEndCallback); @@ -342,6 +344,8 @@ ConnectionPool.prototype.open = function() { } this.isOpen = true; + this.failedConnectionAttempts = 0; + this.noConnectionsTime = Date.now(); }; /** @@ -361,9 +365,12 @@ ConnectionPool.prototype.pause = function() { */ ConnectionPool.prototype.queueConnection = function() { var self = this; + var delay = 0; - var delay = (Math.pow(2, this.failedConnectionAttempts) * 1000) + - (Math.floor(Math.random() * 1000)); + if (this.failedConnectionAttempts > 0) { + delay = (Math.pow(2, this.failedConnectionAttempts) * 1000) + + (Math.floor(Math.random() * 1000)); + } var timeoutHandle = setTimeout(createConnection, delay); this.queue.push(timeoutHandle); diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index 94f45bdc490..e2101e96c81 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -97,6 +97,8 @@ describe('ConnectionPool', function() { PUBSUB.auth.getAuthClient = fakeUtil.noop; pool = new ConnectionPool(SUBSCRIPTION); + pool.queue.forEach(clearTimeout); + pool.queue.length = 0; }); afterEach(function() { @@ -116,6 +118,8 @@ describe('ConnectionPool', function() { assert(pool.connections instanceof Map); assert.strictEqual(pool.isPaused, false); assert.strictEqual(pool.isOpen, false); + assert.strictEqual(pool.failedConnectionAttempts, 0); + assert.strictEqual(pool.noConnectionsTime, 0); assert.strictEqual(pool.settings.maxConnections, 5); assert.strictEqual(pool.settings.ackDeadline, 10000); assert.deepEqual(pool.queue, []); @@ -272,6 +276,16 @@ describe('ConnectionPool', function() { global.clearTimeout = _clearTimeout; }); + it('should reset internally used props', function() { + pool.failedConnectionAttempts = 100; + pool.noConnectionsTime = Date.now(); + + pool.close(); + + assert.strictEqual(pool.failedConnectionAttempts, 0); + assert.strictEqual(pool.noConnectionsTime, 0); + }); + it('should exec a callback when finished closing', function(done) { pool.close(done); }); @@ -791,6 +805,10 @@ describe('ConnectionPool', function() { }); describe('open', function() { + beforeEach(function() { + pool.queueConnection = fakeUtil.noop; + }); + it('should make the specified number of connections', function() { var expectedCount = 5; var connectionCount = 0; @@ -809,6 +827,16 @@ describe('ConnectionPool', function() { pool.open(); assert(pool.isOpen); }); + + it('should reset internal used props', function() { + pool.failedConnectionAttempts = 100; + pool.noConnectionsTime = 0; + + pool.open(); + + assert.strictEqual(pool.failedConnectionAttempts, 0); + assert.strictEqual(pool.noConnectionsTime, Date.now()); + }); }); describe('pause', function() { @@ -871,7 +899,7 @@ describe('ConnectionPool', function() { pool.createConnection = done; global.setTimeout = function(cb, delay) { - assert.strictEqual(delay, 2000); + assert.strictEqual(delay, 0); cb(); // should call the done fn }; From 9e9bdfa9c2ea59ea133c79d4e501ff67ca38d7ad Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Wed, 23 Aug 2017 22:37:43 -0400 Subject: [PATCH 64/67] add get and exists methods --- packages/pubsub/src/subscription.js | 90 ++++++++++- packages/pubsub/src/topic.js | 91 +++++++++++- packages/pubsub/system-test/pubsub.js | 61 +++++++- packages/pubsub/test/subscription.js | 205 +++++++++++++++++++++++++- packages/pubsub/test/topic.js | 183 ++++++++++++++++++++++- 5 files changed, 615 insertions(+), 15 deletions(-) diff --git a/packages/pubsub/src/subscription.js b/packages/pubsub/src/subscription.js index dfd8a39a419..42683f15c35 100644 --- a/packages/pubsub/src/subscription.js +++ b/packages/pubsub/src/subscription.js @@ -472,6 +472,40 @@ Subscription.prototype.delete = function(gaxOpts, callback) { }); }; +/** + * Check if a subscription exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.exists - Whether the subscription exists or not. + * + * @example + * subscription.exists(function(err, exists) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.exists().then(function(data) { + * var exists = data[0]; + * }); + */ +Subscription.prototype.exists = function(callback) { + this.getMetadata(function(err) { + if (!err) { + callback(null, true); + return; + } + + if (err.code === 5) { + callback(null, false); + return; + } + + callback(err); + }); +}; + /** * Flushes internal queues. These can build up if a user attempts to ack/nack * while there is no connection pool (e.g. after they called close). @@ -555,6 +589,52 @@ Subscription.prototype.flushQueues_ = function() { } }; +/** + * Get a subscription if it exists. + * + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {boolean} gaxOpts.autoCreate - Automatically create the subscription + * does not already exist. Default: false. + * + * @example + * subscription.get(function(err, subscription, apiResponse) { + * // The `subscription` data has been populated. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * subscription.get().then(function(data) { + * var subscription = data[0]; + * var apiResponse = data[1]; + * }); + */ +Subscription.prototype.get = function(gaxOpts, callback) { + var self = this; + + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + + var autoCreate = !!gaxOpts.autoCreate && is.fn(this.create); + delete gaxOpts.autoCreate; + + this.getMetadata(gaxOpts, function(err, apiResponse) { + if (!err) { + callback(null, self, apiResponse); + return; + } + + if (err.code !== 5 || !autoCreate) { + callback(err, null, apiResponse); + return; + } + + self.create(gaxOpts, callback); + }); +}; /** * Fetches the subscriptions metadata. @@ -581,6 +661,8 @@ Subscription.prototype.flushQueues_ = function() { * }); */ Subscription.prototype.getMetadata = function(gaxOpts, callback) { + var self = this; + if (is.fn(gaxOpts)) { callback = gaxOpts; gaxOpts = {}; @@ -595,7 +677,13 @@ Subscription.prototype.getMetadata = function(gaxOpts, callback) { method: 'getSubscription', reqOpts: reqOpts, gaxOpts: gaxOpts - }, callback); + }, function(err, apiResponse) { + if (!err) { + self.metadata = apiResponse; + } + + callback(err, apiResponse); + }); }; /** diff --git a/packages/pubsub/src/topic.js b/packages/pubsub/src/topic.js index 0dcaafe4eb1..6fe37cb300b 100644 --- a/packages/pubsub/src/topic.js +++ b/packages/pubsub/src/topic.js @@ -234,6 +234,87 @@ Topic.prototype.delete = function(gaxOpts, callback) { }, callback); }; +/** + * Check if a topic exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.exists - Whether the topic exists or not. + * + * @example + * topic.exists(function(err, exists) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.exists().then(function(data) { + * var exists = data[0]; + * }); + */ +Topic.prototype.exists = function(callback) { + this.getMetadata(function(err) { + if (!err) { + callback(null, true); + return; + } + + if (err.code === 5) { + callback(null, false); + return; + } + + callback(err); + }); +}; + +/** + * Get a topic if it exists. + * + * @param {object=} gaxOpts - Request configuration options, outlined + * here: https://googleapis.github.io/gax-nodejs/CallSettings.html. + * @param {boolean} gaxOpts.autoCreate - Automatically create the topic does not + * already exist. Default: false. + * + * @example + * topic.get(function(err, topic, apiResponse) { + * // The `topic` data has been populated. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * topic.get().then(function(data) { + * var topic = data[0]; + * var apiResponse = data[1]; + * }); + */ +Topic.prototype.get = function(gaxOpts, callback) { + var self = this; + + if (is.fn(gaxOpts)) { + callback = gaxOpts; + gaxOpts = {}; + } + + var autoCreate = !!gaxOpts.autoCreate; + delete gaxOpts.autoCreate; + + this.getMetadata(gaxOpts, function(err, apiResponse) { + if (!err) { + callback(null, self, apiResponse); + return; + } + + if (err.code !== 5 || !autoCreate) { + callback(err, null, apiResponse); + return; + } + + self.create(gaxOpts, callback); + }); +}; + /** * Get the official representation of this topic from the API. * @@ -257,6 +338,8 @@ Topic.prototype.delete = function(gaxOpts, callback) { * }); */ Topic.prototype.getMetadata = function(gaxOpts, callback) { + var self = this; + if (is.fn(gaxOpts)) { callback = gaxOpts; gaxOpts = {}; @@ -271,7 +354,13 @@ Topic.prototype.getMetadata = function(gaxOpts, callback) { method: 'getTopic', reqOpts: reqOpts, gaxOpts: gaxOpts - }, callback); + }, function(err, apiResponse) { + if (!err) { + self.metadata = apiResponse; + } + + callback(err, apiResponse); + }); }; /** diff --git a/packages/pubsub/system-test/pubsub.js b/packages/pubsub/system-test/pubsub.js index 450e1dce683..895eb6d6443 100644 --- a/packages/pubsub/system-test/pubsub.js +++ b/packages/pubsub/system-test/pubsub.js @@ -93,7 +93,14 @@ describe('pubsub', function() { // create all needed topics async.each(TOPICS, function(topic, cb) { topic.create(cb); - }, done); + }, function(err) { + if (err) { + done(err); + return; + } + + setTimeout(done, 5000); + }); }); after(function(done) { @@ -155,6 +162,32 @@ describe('pubsub', function() { }); }); + it('should honor the autoCreate option', function(done) { + var topic = pubsub.topic(generateTopicName()); + + topic.get({ autoCreate: true }, done); + }); + + it('should confirm if a topic exists', function(done) { + var topic = pubsub.topic(TOPIC_NAMES[0]); + + topic.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, true); + done(); + }); + }); + + it('should confirm if a topic does not exist', function(done) { + var topic = pubsub.topic('should-not-exist'); + + topic.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + it('should publish a message', function(done) { var topic = pubsub.topic(TOPIC_NAMES[0]); var publisher = topic.publisher(); @@ -302,6 +335,32 @@ describe('pubsub', function() { }); }); + it('should honor the autoCreate option', function(done) { + var sub = topic.subscription(generateSubName()); + + sub.get({ autoCreate: true }, done); + }); + + it('should confirm if a sub exists', function(done) { + var sub = topic.subscription(SUB_NAMES[0]); + + sub.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, true); + done(); + }); + }); + + it('should confirm if a sub does not exist', function(done) { + var sub = topic.subscription('should-not-exist'); + + sub.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + it('should create a subscription with message retention', function(done) { var subName = generateSubName(); var threeDaysInSeconds = 3 * 24 * 60 * 60; diff --git a/packages/pubsub/test/subscription.js b/packages/pubsub/test/subscription.js index 1d0b2f6c610..63fa029b502 100644 --- a/packages/pubsub/test/subscription.js +++ b/packages/pubsub/test/subscription.js @@ -702,6 +702,46 @@ describe('Subscription', function() { }); }); + describe('exists', function() { + it('should return true if it finds metadata', function(done) { + subscription.getMetadata = function(callback) { + callback(null, {}); + }; + + subscription.exists(function(err, exists) { + assert.ifError(err); + assert(exists); + done(); + }); + }); + + it('should return false if a not found error occurs', function(done) { + subscription.getMetadata = function(callback) { + callback({ code: 5 }); + }; + + subscription.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + + it('should pass back any other type of error', function(done) { + var error = { code: 4 }; + + subscription.getMetadata = function(callback) { + callback(error); + }; + + subscription.exists(function(err, exists) { + assert.strictEqual(err, error); + assert.strictEqual(exists, undefined); + done(); + }); + }); + }); + describe('flushQueues_', function() { beforeEach(function() { subscription.inventory_.ack = ['abc', 'def']; @@ -878,27 +918,180 @@ describe('Subscription', function() { }); }); + describe('get', function() { + beforeEach(function() { + subscription.create = fakeUtil.noop; + }); + + it('should delete the autoCreate option', function(done) { + var options = { + autoCreate: true, + a: 'a' + }; + + subscription.getMetadata = function(gaxOpts) { + assert.strictEqual(gaxOpts, options); + assert.strictEqual(gaxOpts.autoCreate, undefined); + done(); + }; + + subscription.get(options, assert.ifError); + }); + + describe('success', function() { + var fakeMetadata = {}; + + beforeEach(function() { + subscription.getMetadata = function(gaxOpts, callback) { + callback(null, fakeMetadata); + }; + }); + + it('should call through to getMetadata', function(done) { + subscription.get(function(err, sub, resp) { + assert.ifError(err); + assert.strictEqual(sub, subscription); + assert.strictEqual(resp, fakeMetadata); + done(); + }); + }); + + it('should optionally accept options', function(done) { + var options = {}; + + subscription.getMetadata = function(gaxOpts, callback) { + assert.strictEqual(gaxOpts, options); + callback(); // the done fn + }; + + subscription.get(options, done); + }); + }); + + describe('error', function() { + it('should pass back errors when not auto-creating', function(done) { + var error = { code: 4 }; + var apiResponse = {}; + + subscription.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + subscription.get(function(err, sub, resp) { + assert.strictEqual(err, error); + assert.strictEqual(sub, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should pass back 404 errors if autoCreate is false', function(done) { + var error = { code: 5 }; + var apiResponse = {}; + + subscription.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + subscription.get(function(err, sub, resp) { + assert.strictEqual(err, error); + assert.strictEqual(sub, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should pass back 404 errors if create doesnt exist', function(done) { + var error = { code: 5 }; + var apiResponse = {}; + + subscription.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + delete subscription.create; + + subscription.get(function(err, sub, resp) { + assert.strictEqual(err, error); + assert.strictEqual(sub, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should create the sub if 404 + autoCreate is true', function(done) { + var error = { code: 5 }; + var apiResponse = {}; + + var fakeOptions = { + autoCreate: true + }; + + subscription.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + subscription.create = function(options, callback) { + assert.strictEqual(options, fakeOptions); + callback(); // the done fn + }; + + subscription.get(fakeOptions, done); + }); + }); + }); + describe('getMetadata', function() { it('should make the correct request', function(done) { - subscription.request = function(config, callback) { + subscription.request = function(config) { assert.strictEqual(config.client, 'subscriberClient'); assert.strictEqual(config.method, 'getSubscription'); assert.deepEqual(config.reqOpts, { subscription: subscription.name }); - callback(); // the done fn + done(); }; - subscription.getMetadata(done); + subscription.getMetadata(assert.ifError); }); it('should optionally accept gax options', function(done) { var gaxOpts = {}; - subscription.request = function(config, callback) { + subscription.request = function(config) { assert.strictEqual(config.gaxOpts, gaxOpts); - callback(); // the done fn + done(); + }; + + subscription.getMetadata(gaxOpts, assert.ifError); + }); + + it('should pass back any errors that occur', function(done) { + var error = new Error('err'); + var apiResponse = {}; + + subscription.request = function(config, callback) { + callback(error, apiResponse); }; - subscription.getMetadata(gaxOpts, done); + subscription.getMetadata(function(err, metadata) { + assert.strictEqual(err, error); + assert.strictEqual(metadata, apiResponse); + done(); + }); + }); + + it('should set the metadata if no error occurs', function(done) { + var apiResponse = {}; + + subscription.request = function(config, callback) { + callback(null, apiResponse); + }; + + subscription.getMetadata(function(err, metadata) { + assert.ifError(err); + assert.strictEqual(metadata, apiResponse); + assert.strictEqual(subscription.metadata, apiResponse); + done(); + }); }); }); diff --git a/packages/pubsub/test/topic.js b/packages/pubsub/test/topic.js index c23ca5f97b4..ef73a6464ff 100644 --- a/packages/pubsub/test/topic.js +++ b/packages/pubsub/test/topic.js @@ -213,27 +213,198 @@ describe('Topic', function() { }); }); + describe('get', function() { + it('should delete the autoCreate option', function(done) { + var options = { + autoCreate: true, + a: 'a' + }; + + topic.getMetadata = function(gaxOpts) { + assert.strictEqual(gaxOpts, options); + assert.strictEqual(gaxOpts.autoCreate, undefined); + done(); + }; + + topic.get(options, assert.ifError); + }); + + describe('success', function() { + var fakeMetadata = {}; + + beforeEach(function() { + topic.getMetadata = function(gaxOpts, callback) { + callback(null, fakeMetadata); + }; + }); + + it('should call through to getMetadata', function(done) { + topic.get(function(err, _topic, resp) { + assert.ifError(err); + assert.strictEqual(_topic, topic); + assert.strictEqual(resp, fakeMetadata); + done(); + }); + }); + + it('should optionally accept options', function(done) { + var options = {}; + + topic.getMetadata = function(gaxOpts, callback) { + assert.strictEqual(gaxOpts, options); + callback(); // the done fn + }; + + topic.get(options, done); + }); + }); + + describe('error', function() { + it('should pass back errors when not auto-creating', function(done) { + var error = { code: 4 }; + var apiResponse = {}; + + topic.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + topic.get(function(err, _topic, resp) { + assert.strictEqual(err, error); + assert.strictEqual(_topic, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should pass back 404 errors if autoCreate is false', function(done) { + var error = { code: 5 }; + var apiResponse = {}; + + topic.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + topic.get(function(err, _topic, resp) { + assert.strictEqual(err, error); + assert.strictEqual(_topic, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should create the topic if 404 + autoCreate is true', function(done) { + var error = { code: 5 }; + var apiResponse = {}; + + var fakeOptions = { + autoCreate: true + }; + + topic.getMetadata = function(gaxOpts, callback) { + callback(error, apiResponse); + }; + + topic.create = function(options, callback) { + assert.strictEqual(options, fakeOptions); + callback(); // the done fn + }; + + topic.get(fakeOptions, done); + }); + }); + }); + + describe('exists', function() { + it('should return true if it finds metadata', function(done) { + topic.getMetadata = function(callback) { + callback(null, {}); + }; + + topic.exists(function(err, exists) { + assert.ifError(err); + assert(exists); + done(); + }); + }); + + it('should return false if a not found error occurs', function(done) { + topic.getMetadata = function(callback) { + callback({ code: 5 }); + }; + + topic.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + + it('should pass back any other type of error', function(done) { + var error = { code: 4 }; + + topic.getMetadata = function(callback) { + callback(error); + }; + + topic.exists(function(err, exists) { + assert.strictEqual(err, error); + assert.strictEqual(exists, undefined); + done(); + }); + }); + }); + describe('getMetadata', function() { it('should make the proper request', function(done) { - topic.request = function(config, callback) { + topic.request = function(config) { assert.strictEqual(config.client, 'publisherClient'); assert.strictEqual(config.method, 'getTopic'); assert.deepEqual(config.reqOpts, { topic: topic.name }); - callback(); // the done fn + done(); }; - topic.getMetadata(done); + topic.getMetadata(assert.ifError); }); it('should optionally accept gax options', function(done) { var options = {}; - topic.request = function(config, callback) { + topic.request = function(config) { assert.strictEqual(config.gaxOpts, options); - callback(); + done(); }; - topic.getMetadata(options, done); + topic.getMetadata(options, assert.ifError); + }); + + it('should pass back any errors that occur', function(done) { + var error = new Error('err'); + var apiResponse = {}; + + topic.request = function(config, callback) { + callback(error, apiResponse); + }; + + topic.getMetadata(function(err, metadata) { + assert.strictEqual(err, error); + assert.strictEqual(metadata, apiResponse); + done(); + }); + }); + + it('should set the metadata if no error occurs', function(done) { + var apiResponse = {}; + + topic.request = function(config, callback) { + callback(null, apiResponse); + }; + + topic.getMetadata(function(err, metadata) { + assert.ifError(err); + assert.strictEqual(metadata, apiResponse); + assert.strictEqual(topic.metadata, apiResponse); + done(); + }); }); }); From 31ab74869ca1f6a938560e6b4a95a69b5165894c Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 24 Aug 2017 10:09:31 -0700 Subject: [PATCH 65/67] tweak test --- packages/pubsub/test/connection-pool.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/pubsub/test/connection-pool.js b/packages/pubsub/test/connection-pool.js index e2101e96c81..e610035c22b 100644 --- a/packages/pubsub/test/connection-pool.js +++ b/packages/pubsub/test/connection-pool.js @@ -475,6 +475,13 @@ describe('ConnectionPool', function() { }); it('should capture the date when no connections are found', function() { + var dateNow = global.Date.now; + + var fakeDate = Date.now(); + global.Date.now = function() { + return fakeDate; + }; + pool.noConnectionsTime = 0; pool.isConnected = function() { return false; @@ -483,7 +490,8 @@ describe('ConnectionPool', function() { pool.createConnection(); fakeConnection.emit('status', {}); - assert.strictEqual(pool.noConnectionsTime, Date.now()); + assert.strictEqual(pool.noConnectionsTime, fakeDate); + global.Date.now = dateNow; }); it('should not capture the date when already set', function() { From ca6e982ae4f94eb3ef1086f93a72f80e4ccf6643 Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 24 Aug 2017 10:09:55 -0700 Subject: [PATCH 66/67] update readmes to list pubsub as beta --- README.md | 2 +- packages/pubsub/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6fb83e07c77..1a1c09e64e1 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ This client supports the following Google Cloud Platform services at a [General This client supports the following Google Cloud Platform services at a [Beta](#versioning) quality level: * [Cloud Natural Language](#cloud-natural-language-beta) (Beta) +* [Cloud Pub/Sub](#cloud-pubsub-alpha) (Beta) * [Cloud Spanner](#cloud-spanner-beta) (Beta) * [Cloud Vision](#cloud-vision-beta) (Beta) * [Google BigQuery](#google-bigquery-beta) (Beta) @@ -28,7 +29,6 @@ This client supports the following Google Cloud Platform services at an [Alpha]( * [Cloud Bigtable](#cloud-bigtable-alpha) (Alpha) * [Cloud DNS](#cloud-dns-alpha) (Alpha) -* [Cloud Pub/Sub](#cloud-pubsub-alpha) (Alpha) * [Cloud Resource Manager](#cloud-resource-manager-alpha) (Alpha) * [Cloud Speech](#cloud-speech-alpha) (Alpha) * [Google Compute Engine](#google-compute-engine-alpha) (Alpha) diff --git a/packages/pubsub/README.md b/packages/pubsub/README.md index 4244d2bb77c..d225bdf5f92 100644 --- a/packages/pubsub/README.md +++ b/packages/pubsub/README.md @@ -1,4 +1,4 @@ -# @google-cloud/pubsub ([Alpha][versioning]) +# @google-cloud/pubsub ([Beta][versioning]) > Cloud Pub/Sub Client Library for Node.js *Looking for more Google APIs than just Pub/Sub? You might want to check out [`google-cloud`][google-cloud].* From f758d0740d9bfef1c59d3b96617d92e7cd8fb0fa Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Thu, 24 Aug 2017 10:19:28 -0700 Subject: [PATCH 67/67] move pubsub sample placement --- README.md | 102 +++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 1a1c09e64e1..9900d7f5ddf 100644 --- a/README.md +++ b/README.md @@ -450,6 +450,57 @@ languageClient.analyzeSentiment({document: document}).then(function(responses) { ``` +## Cloud Pub/Sub (Beta) + +- [API Documentation][gcloud-pubsub-docs] +- [Official Documentation][cloud-pubsub-docs] + +#### Using the Cloud Pub/Sub API module + +``` +$ npm install --save @google-cloud/pubsub +``` + +```js +var pubsub = require('@google-cloud/pubsub'); +``` + +#### Authentication + +See [Authentication](#authentication). + +#### Preview + +```js +var pubsubClient = pubsub({ + projectId: 'grape-spaceship-123', + keyFilename: '/path/to/keyfile.json' +}); + +// Reference a topic that has been previously created. +var topic = pubsubClient.topic('my-topic'); + +// Publish a message to the topic. +var publisher = topic.publisher(); +var message = new Buffer('New message!'); + +publisher.publish(message, function(err, messageId) {}); + +// Subscribe to the topic. +topic.createSubscription('subscription-name', function(err, subscription) { + // Register listeners to start pulling for messages. + function onError(err) {} + function onMessage(message) {} + subscription.on('error', onError); + subscription.on('message', onMessage); + + // Remove listeners to stop pulling for messages. + subscription.removeListener('message', onMessage); + subscription.removeListener('error', onError); +}); +``` + + ## Cloud Spanner (Beta) - [API Documentation][gcloud-spanner-docs] @@ -816,57 +867,6 @@ zone.export('/zonefile.zone', function(err) {}); ``` -## Cloud Pub/Sub (Alpha) - -- [API Documentation][gcloud-pubsub-docs] -- [Official Documentation][cloud-pubsub-docs] - -#### Using the Cloud Pub/Sub API module - -``` -$ npm install --save @google-cloud/pubsub -``` - -```js -var pubsub = require('@google-cloud/pubsub'); -``` - -#### Authentication - -See [Authentication](#authentication). - -#### Preview - -```js -var pubsubClient = pubsub({ - projectId: 'grape-spaceship-123', - keyFilename: '/path/to/keyfile.json' -}); - -// Reference a topic that has been previously created. -var topic = pubsubClient.topic('my-topic'); - -// Publish a message to the topic. -var publisher = topic.publisher(); -var message = new Buffer('New message!'); - -publisher.publish(message, function(err, messageId) {}); - -// Subscribe to the topic. -topic.createSubscription('subscription-name', function(err, subscription) { - // Register listeners to start pulling for messages. - function onError(err) {} - function onMessage(message) {} - subscription.on('error', onError); - subscription.on('message', onMessage); - - // Remove listeners to stop pulling for messages. - subscription.removeListener('message', onMessage); - subscription.removeListener('error', onError); -}); -``` - - ## Cloud Resource Manager (Alpha) - [API Documentation][gcloud-resource-docs]