[Main][Task]32698211: Add a Config to Allow Users to Change Max Number of Events Per Batch

channels/1ds-post-js/src/DataModels.ts

@@ -68,6 +68,13 @@ export interface IChannelConfiguration {
      */
     requestLimit?: IRequestSizeLimit;
 
+    /**
+     * [Optional] Sets the limit number of events per batch.
+     * Default is 500
+     * @since 3.3.7
+     */
+    maxEvtPerBatch?: number
+
     /**
      * [Optional] The HTTP override that should be used to send requests, as an IXHROverride object.
      * By default during the unload of a page or if the event specifies that it wants to use sendBeacon() or sync fetch (with keep-alive),

channels/1ds-post-js/src/HttpManager.ts

@@ -179,6 +179,7 @@ export class HttpManager {
         let _excludeCsMetaData: boolean;
         let _sendPostMgr: SenderPostManager;
         let _fetchCredentials: RequestCredentials;
+        let _maxEvtPerBatch: number = maxEventsPerBatch; // Sets default value in case the value is null
 
         dynamicProto(HttpManager, this, (_self) => {
             _initDefaults();
@@ -214,6 +215,8 @@ export class HttpManager {
                         _urlString = endpointUrl + UrlQueryString;
                         _useHeaders = !isUndefined(channelConfig.avoidOptions) ? !channelConfig.avoidOptions : true;
                         _enableEventTimings = !channelConfig.disableEventTimings;
+                        let maxEvtCfg = channelConfig.maxEvtPerBatch;
+                        _maxEvtPerBatch = maxEvtCfg && maxEvtCfg <= maxEventsPerBatch? maxEvtCfg : maxEventsPerBatch;
 
                         let valueSanitizer = channelConfig.valueSanitizer;
                         let stringifyObjects = channelConfig.stringifyObjects;
@@ -377,7 +380,7 @@ export class HttpManager {
                         let theBatch = theBatches.shift();
                         if (theBatch && theBatch.count() > 0) {
                             thePayload = thePayload || _serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
-                            _serializer.appendPayload(thePayload, theBatch, maxEventsPerBatch)
+                            _serializer.appendPayload(thePayload, theBatch, _maxEvtPerBatch)
                         }
                     }
 
@@ -426,7 +429,7 @@ export class HttpManager {
             }
 
             _self["_getDbgPlgTargets"] = () => {
-                return [_sendInterfaces[EventSendType.Batched], _killSwitch, _serializer, _sendInterfaces, _getSendPostMgrConfig(), _urlString];
+                return [_sendInterfaces[EventSendType.Batched], _killSwitch, _serializer, _sendInterfaces, _getSendPostMgrConfig(), _urlString, _maxEvtPerBatch];
             };
 
             function _getSendPostMgrConfig(): _ISendPostMgrConfig {
@@ -500,6 +503,7 @@ export class HttpManager {
                 _timeoutWrapper = createTimeoutWrapper();
                 _excludeCsMetaData = false;
                 _sendPostMgr = null;
+                _maxEvtPerBatch = null;
             }
 
             function _fetchOnComplete(response: Response, onComplete: OnCompleteCallback, resValue?: string, payload?: IPayloadData) {
@@ -788,7 +792,7 @@ export class HttpManager {
                                     thePayload = thePayload || _serializer.createPayload(retryCount, isTeardown, isSynchronous, isReducedPayload, sendReason, sendType);
 
                                     // Add the batch to the current payload
-                                    if (!_serializer.appendPayload(thePayload, theBatch, maxEventsPerBatch)) {
+                                    if (!_serializer.appendPayload(thePayload, theBatch, _maxEvtPerBatch)) {
                                         // Entire batch was not added so send the payload and retry adding this batch
                                         _doPayloadSend(thePayload, serializationStart, getTime(), sendReason);
                                         serializationStart = getTime();

channels/1ds-post-js/src/PostChannel.ts

@@ -89,6 +89,7 @@ const defaultPostChannelConfig: IConfigDefaults<IChannelConfiguration> = objDeep
     maxEventRetryAttempts: { isVal: isNumber, v: MaxSendAttempts },
     maxUnloadEventRetryAttempts: { isVal: isNumber, v: MaxSyncUnloadSendAttempts},
     addNoResponse: undefValue,
+    maxEvtPerBatch: {isVal: isNumber, v: MaxNumberEventPerBatch},
     excludeCsMetaData: undefValue,
     requestLimit: {} as IRequestSizeLimit
 });
@@ -146,6 +147,7 @@ export class PostChannel extends BaseTelemetryPlugin implements IChannelControls
         let _unloadHandlersAdded: boolean;
         let _overrideInstrumentationKey: string;
         let _disableTelemetry: boolean;
+        let _maxEvtPerBatch: number;
 
         dynamicProto(PostChannel, this, (_self, _base) => {
             _initDefaults();
@@ -181,6 +183,7 @@ export class PostChannel extends BaseTelemetryPlugin implements IChannelControls
                             _maxEventSendAttempts = _postConfig.maxEventRetryAttempts;
                             _maxUnloadEventSendAttempts = _postConfig.maxUnloadEventRetryAttempts;
                             _disableAutoBatchFlushLimit = _postConfig.disableAutoBatchFlushLimit;
+                            _maxEvtPerBatch = _postConfig.maxEvtPerBatch;
 
                             if (isPromiseLike(coreConfig.endpointUrl)) {
                                 _self.pause();
@@ -701,8 +704,10 @@ export class PostChannel extends BaseTelemetryPlugin implements IChannelControls
                 _maxUnloadEventSendAttempts = MaxSyncUnloadSendAttempts;
                 _evtNamespace = null;
                 _overrideInstrumentationKey = null;
+                _maxEvtPerBatch = null;
                 _disableTelemetry = false;
                 _timeoutWrapper = createTimeoutWrapper();
+                // httpManager init should use the default value, because _maxEvtPerBatch is null currently
                 _httpManager = new HttpManager(MaxNumberEventPerBatch, MaxConnections, MaxRequestRetriesBeforeBackoff, {
                     requeue: _requeueEvents,
                     send: _sendingEvent,
@@ -1140,7 +1145,7 @@ export class PostChannel extends BaseTelemetryPlugin implements IChannelControls
 
             function _setAutoLimits() {
                 if (!_disableAutoBatchFlushLimit) {
-                    _autoFlushBatchLimit = mathMax(MaxNumberEventPerBatch * (MaxConnections + 1), _queueSizeLimit / 6);
+                    _autoFlushBatchLimit = mathMax(_maxEvtPerBatch * (MaxConnections + 1), _queueSizeLimit / 6);
                 } else {
                     _autoFlushBatchLimit = 0;
                 }

channels/1ds-post-js/test/Unit/src/HttpManagerTest.ts

@@ -251,6 +251,85 @@ export class HttpManagerTest extends AITestClass {
             }
         });
 
+        this.testCase({
+            name: "Http Manager with maxNumberEvtPerBatch config set",
+            useFakeTimers: true,
+            test: () => {
+                var xhrOverride: IXHROverride = {
+                    sendPOST: (payload: IPayloadData,
+                        oncomplete: (status: number, headers: { [headerName: string]: string }) => void, sync?: boolean) => {
+                        oncomplete(200, null);
+                    }
+                };
+
+                let testBatch = EventBatch.create("testToken", [this._createEvent()]);
+
+                var manager: HttpManager = new HttpManager(500, 2, 1, {
+                    requeue: _requeueNotification,
+                    send: _sendNotification,
+                    sent: _sentNotification,
+                    drop: _dropNotification
+                });
+                const hookSpy = this.sandbox.spy(_sendHook);
+                this.core.config.extensionConfig = this.core.config.extensionConfig || {};
+                this.core.config.extensionConfig[this.postManager.identifier].maxEvtPerBatch = 3;
+                this.core.config.extensionConfig[this.postManager.identifier].payloadPreprocessor = hookSpy;
+                this.core.config.extensionConfig[this.postManager.identifier].httpXHROverride = xhrOverride;
+
+                manager.initialize(this.core.config, this.core, this.postManager);
+                let maxNumberEvtPerBatch = manager["_getDbgPlgTargets"]()[6];
+                QUnit.assert.equal(maxNumberEvtPerBatch, 3, "max number of events per batch should be 3");
+                let serializer = manager["_getDbgPlgTargets"]()[2];
+                let spy = this.sandbox.spy(serializer, "appendPayload");
+                QUnit.assert.equal(manager["_getDbgPlgTargets"]()[0]._transport, undefined, "Make sure that no transport value is defined");
+
+                QUnit.assert.ok(hookSpy.notCalled); // precondition
+                QUnit.assert.equal(this._sendEvents.length, 0, "No batches sent yet");
+                QUnit.assert.equal(this._sentEvents.length, 0, "No batches Completed yet");
+                QUnit.assert.ok(spy.notCalled, "appendPayload should not be called yet");
+                manager.sendSynchronousBatch(testBatch);
+                QUnit.assert.ok(spy.calledOnce, "appendPayload should be called when the manager makes an HTTP request");
+                let appendPayloadArgs = spy.args[0][2];
+                QUnit.assert.equal(appendPayloadArgs, 3, "appendPayload should be called with max number of events per batch");
+                QUnit.assert.ok(hookSpy.calledOnce, "preprocessor should be called when the manager makes an HTTP request");
+                QUnit.assert.ok(hookSpy.args[0][2], "preprocessor should have been told its a sync request");
+                QUnit.assert.equal(this._sendEvents.length, 1, "batches sent");
+                QUnit.assert.equal(this._sentEvents.length, 1, "batches Completed");
+
+            }
+        });
+
+        this.testCase({
+            name: "Http Manager with maxNumberEvtPerBatch config set to 0 or large than the default value",
+            useFakeTimers: true,
+            test: () => {
+
+                var manager: HttpManager = new HttpManager(500, 2, 1, {
+                    requeue: _requeueNotification,
+                    send: _sendNotification,
+                    sent: _sentNotification,
+                    drop: _dropNotification
+                });
+                this.core.config.extensionConfig = this.core.config.extensionConfig || {};
+                this.core.config.extensionConfig[this.postManager.identifier].maxEvtPerBatch = 0;
+                manager.initialize(this.core.config, this.core, this.postManager);
+                let maxNumberEvtPerBatch = manager["_getDbgPlgTargets"]()[6];
+                QUnit.assert.equal(maxNumberEvtPerBatch, 500, "max number of events per batch should be 500");
+
+                var manager1: HttpManager = new HttpManager(500, 2, 1, {
+                    requeue: _requeueNotification,
+                    send: _sendNotification,
+                    sent: _sentNotification,
+                    drop: _dropNotification
+                });
+                this.core.config.extensionConfig[this.postManager.identifier].maxEvtPerBatch = 1000;
+                manager1.initialize(this.core.config, this.core, this.postManager);
+                maxNumberEvtPerBatch = manager1["_getDbgPlgTargets"]()[6];
+                QUnit.assert.equal(maxNumberEvtPerBatch, 500, "max number of events per batch should be 500 test1");
+            }
+        });
+
+
         this.testCase({
             name: "payloadPreprocessor with override",
             useFakeTimers: true,

channels/1ds-post-js/test/Unit/src/PostChannelTest.ts

@@ -177,6 +177,7 @@ export class PostChannelTest extends AITestClass {
                     fetchCredentials: undefValue,
                     maxEventRetryAttempts: 6,
                     maxUnloadEventRetryAttempts: 2,
+                    maxEvtPerBatch: 500,
                     addNoResponse: undefValue,
                     excludeCsMetaData: undefValue,
                     requestLimit: {}

channels/1ds-post-js/test/Unit/src/SerializerTest.ts

@@ -37,6 +37,86 @@ export class SerializerTest extends AITestClass {
             }
         });
 
+        this.testCase({
+            name: "Append payload with max number per batch",
+            test: () => {
+                let maxNumberEvtPerBatch = 3;
+                let serializer = new Serializer(null, null, null, false, null, null);
+                let ikey = "1234-5678";
+                let event: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+                let event1: IPostTransmissionTelemetryItem = {
+                    name: "testEvent1",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+
+                let payload = serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
+                let batch = EventBatch.create(ikey, [event, event1]);
+                serializer.appendPayload(payload, batch, maxNumberEvtPerBatch);
+                let evts = payload.payloadBlob;
+                let expectedPayload = "{\"name\":\"testEvent\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}" + "\n" +
+                "{\"name\":\"testEvent1\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}"
+                QUnit.assert.equal(evts, expectedPayload, "should contain both events");
+                let overflow = payload.overflow;
+                QUnit.assert.equal(overflow, null, "should not have overflow batch");
+                let sizeExceed = payload.sizeExceed;
+                QUnit.assert.equal(sizeExceed.length, 0, "should not have size exceed batch");
+            }
+        });
+
+        this.testCase({
+            name: "Append payload with exceed max number per batch",
+            test: () => {
+                let maxNumberEvtPerBatch = 1;
+                let serializer = new Serializer(null, null, null, false, null, null);
+                let ikey = "1234-5678";
+                let event: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+                let event1: IPostTransmissionTelemetryItem = {
+                    name: "testEvent1",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+
+                let payload = serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
+                let batch = EventBatch.create(ikey, [event, event1]);
+                serializer.appendPayload(payload, batch, maxNumberEvtPerBatch);
+                let evts = payload.payloadBlob;
+                let expectedPayload = "{\"name\":\"testEvent\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}";
+                QUnit.assert.equal(evts, expectedPayload, "should contain both events");
+                let overflow = payload.overflow;
+                QUnit.assert.equal(overflow.count(), 1, "should have only one overflow batch");
+                let overflowEvts = overflow.events();
+                QUnit.assert.equal(overflowEvts.length, 1, "should have only one overflow event");
+                QUnit.assert.equal(overflowEvts[0], event1, "overflow should have event1");
+                let sizeExceed = payload.sizeExceed;
+                QUnit.assert.equal(sizeExceed.length, 0, "should not have size exceed batch");
+            }
+        });
+
         this.testCase({
             name: "Append payload with size limit channel config",
             test: () => {