DRIVERS-2884 Avoid connection churn when operations timeout #1675
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,344 @@ | ||
| description: "Connection churn is prevented by reading pending responses during connection checkout" | ||
| schemaVersion: "1.24" | ||
| runOnRequirements: | ||
| - minServerVersion: "4.4" | ||
| # TODO(SERVER-96344): When using failpoints, mongos returns MaxTimeMSExpired | ||
| # after maxTimeMS, whereas mongod returns it after | ||
| # max(blockTimeMS, maxTimeMS). Until this ticket is resolved, these tests | ||
| # will not pass on sharded clusters. | ||
| topologies: ["single", "replicaset"] | ||
| createEntities: | ||
| - client: | ||
| id: &failPointClient failPointClient | ||
| useMultipleMongoses: false | ||
| - client: | ||
| id: &client client | ||
| uriOptions: | ||
| maxPoolSize: 1 | ||
| useMultipleMongoses: false | ||
| observeEvents: | ||
| - commandFailedEvent | ||
| - commandSucceededEvent | ||
| - connectionCheckedOutEvent | ||
| - connectionCheckedInEvent | ||
| - connectionClosedEvent | ||
| - connectionPendingResponseSucceeded | ||
| - connectionPendingResponseStarted | ||
| - connectionPendingResponseFailed | ||
| - database: | ||
| id: &database test | ||
| client: *client | ||
| databaseName: *database | ||
| - collection: | ||
| id: &collection coll | ||
| database: *database | ||
| collectionName: *collection | ||
| initialData: | ||
| - collectionName: *collection | ||
| databaseName: *database | ||
| documents: [] | ||
| tests: | ||
| # Attempting a pending response read on a non-timeout operation that can | ||
| # immediately read from the TCP buffer should complete the pending read and | ||
| # the connection should be checked out. | ||
| - description: "non-timeout op with response and no error" | ||
| operations: | ||
| # Run a ping command to pre-load the pool with a connection. | ||
| - name: runCommand | ||
| object: *database | ||
| arguments: | ||
| command: | ||
| ping: 1 | ||
| commandName: ping | ||
| # Create a failpoint to block the first operation. | ||
| - name: failPoint | ||
| object: testRunner | ||
| arguments: | ||
| client: *failPointClient | ||
| failPoint: | ||
| configureFailPoint: failCommand | ||
| mode: {times: 1} | ||
| data: | ||
| failCommands: ["insert"] | ||
| blockConnection: true | ||
| blockTimeMS: 100 | ||
| # Execute operation with timeout less than block time. | ||
| - name: insertOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 75 | ||
| document: {_id: 3, x: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| # Execute a subsequent operation to complete the read. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| filter: {_id: 1} | ||
| expectEvents: | ||
| - client: *client | ||
| events: | ||
| - commandSucceededEvent: | ||
| commandName: ping # Pre-loading the connection pool. | ||
| - commandFailedEvent: | ||
| commandName: insert | ||
| - commandSucceededEvent: | ||
| commandName: find | ||
| - client: *client | ||
| eventType: cmap | ||
| events: | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Ping finishes. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Insert fails. | ||
| - connectionPendingResponseStarted: {} | ||
| - connectionPendingResponseSucceeded: {} # Find operation drains connection. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Find succeeds. | ||
| # Attempting a pending response read on a non-timeout operation that gets no | ||
| # response from the server after 3s should close the connection. | ||
| - description: "non-timeout op with no response" | ||
| operations: | ||
| # Run a ping command to pre-load the pool with a connection. | ||
| - name: runCommand | ||
| object: *database | ||
| arguments: | ||
| command: | ||
| ping: 1 | ||
| commandName: ping | ||
| # Create a failpoint to block the first operation. | ||
| - name: failPoint | ||
| object: testRunner | ||
| arguments: | ||
| client: *failPointClient | ||
| failPoint: | ||
| configureFailPoint: failCommand | ||
| mode: {times: 1} | ||
| data: | ||
| failCommands: ["insert"] | ||
| blockConnection: true | ||
| blockTimeMS: 3100 | ||
| # Execute operation with timeout less than block time. | ||
| - name: insertOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 50 | ||
| document: {_id: 3, x: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| # Execute a subsequent operation to complete the read. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| filter: {_id: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| expectEvents: | ||
| - client: *client | ||
| events: | ||
| - commandSucceededEvent: | ||
| commandName: ping # Pre-loading the connection pool. | ||
| - commandFailedEvent: | ||
| commandName: insert | ||
| # No second failed event since we timed out attempting to check out | ||
| # the connection for the second operation. | ||
| - client: *client | ||
| eventType: cmap | ||
| events: | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Ping finishes. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Insert fails. | ||
| - connectionPendingResponseStarted: {} | ||
| - connectionPendingResponseFailed: | ||
| reason: timeout | ||
| - connectionClosedEvent: | ||
| reason: error | ||
| # Attempting a pending response read on a realistic timeout operation that can | ||
| # immediately read from the TCP buffer should complete the pending read and | ||
| # the connection should be checked out. | ||
| - description: "timeout op with response and no error" | ||
| operations: | ||
| # Run a ping command to pre-load the pool with a connection. | ||
| - name: runCommand | ||
| object: *database | ||
| arguments: | ||
| command: | ||
| ping: 1 | ||
| commandName: ping | ||
| # Create a failpoint to block the first operation. | ||
| - name: failPoint | ||
| object: testRunner | ||
| arguments: | ||
| client: *failPointClient | ||
| failPoint: | ||
| configureFailPoint: failCommand | ||
| mode: {times: 1} | ||
| data: | ||
| failCommands: ["insert"] | ||
| blockConnection: true | ||
| blockTimeMS: 250 | ||
| # Execute operation with timeout less than block time. | ||
| - name: insertOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 75 | ||
| document: {_id: 3, x: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| # Execute a subsequent operation to complete the read. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 200 | ||
| filter: {_id: 1} | ||
| expectEvents: | ||
| - client: *client | ||
| events: | ||
| - commandSucceededEvent: | ||
| commandName: ping # Pre-loading the connection pool. | ||
| - commandFailedEvent: | ||
| commandName: insert | ||
| - commandSucceededEvent: | ||
| commandName: find | ||
| - client: *client | ||
| eventType: cmap | ||
| events: | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Ping finishes. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Insert fails. | ||
| - connectionPendingResponseStarted: {} | ||
| - connectionPendingResponseSucceeded: {} | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Find succeeds. | ||
| # It may take multiple calls to the pending response handler to drain the | ||
| # inbound buffer. | ||
| - description: "multiple calls to drain buffer" | ||
| operations: | ||
| # Run a ping command to pre-load the pool with a connection. | ||
| - name: runCommand | ||
| object: *database | ||
| arguments: | ||
| command: | ||
| ping: 1 | ||
| commandName: ping | ||
| # Create a failpoint to block the first and second operation. | ||
| - name: failPoint | ||
| object: testRunner | ||
| arguments: | ||
| client: *failPointClient | ||
| failPoint: | ||
| configureFailPoint: failCommand | ||
| mode: {times: 1} | ||
| data: | ||
| failCommands: ["insert"] | ||
| blockConnection: true | ||
| blockTimeMS: 500 | ||
| # Execute operation with timeout less than block time. | ||
| - name: insertOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 50 | ||
| document: {_id: 3, x: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| # Execute a subsequent operation with a timeout less than the block time. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 50 | ||
| filter: {_id: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| # Execute a final operation to drain the buffer. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| filter: {_id: 1} | ||
| expectEvents: | ||
| - client: *client | ||
| events: | ||
| - commandSucceededEvent: | ||
| commandName: ping # Pre-loading the connection pool. | ||
| - commandFailedEvent: | ||
| commandName: insert | ||
| - commandSucceededEvent: | ||
| commandName: find | ||
| - client: *client | ||
| eventType: cmap | ||
| events: | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Ping finishes. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Insert fails. | ||
| - connectionPendingResponseStarted: {} # First find fails | ||
| - connectionPendingResponseFailed: | ||
| reason: timeout | ||
| - connectionPendingResponseStarted: {} # Second find drains the buffer. | ||
| - connectionPendingResponseSucceeded: {} | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Second find succeeds. | ||
| # If the connection is closed server-side while draining the response, the | ||
| # driver must close the connection. | ||
| - description: "connection closed server-side while draining response" | ||
|
There was a problem hiding this comment. This seems to be a sufficient check that if the |
||
| operations: | ||
| # Run a ping command to pre-load the pool with a connection. | ||
| - name: runCommand | ||
| object: *database | ||
| arguments: | ||
| command: | ||
| ping: 1 | ||
| commandName: ping | ||
| # Create a failpoint to block the first and second operation. | ||
| - name: failPoint | ||
| object: testRunner | ||
| arguments: | ||
| client: *failPointClient | ||
| failPoint: | ||
| configureFailPoint: failCommand | ||
| mode: {times: 1} | ||
| data: | ||
| failCommands: ["insert"] | ||
| blockConnection: true | ||
| blockTimeMS: 500 | ||
| closeConnection: true | ||
| # Execute operation with timeout less than block time. | ||
| - name: insertOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 50 | ||
| document: {_id: 3, x: 1} | ||
| expectError: | ||
| isTimeoutError: true | ||
| - name: wait | ||
| object: testRunner | ||
| arguments: | ||
| ms: 500 | ||
| # Execute a subsequent operation with a timeout less than the block time. | ||
| - name: findOne | ||
| object: *collection | ||
| arguments: | ||
| timeoutMS: 50 | ||
| filter: {_id: 1} | ||
| expectError: | ||
| isTimeoutError: false | ||
| expectEvents: | ||
| - client: *client | ||
| events: | ||
| - commandSucceededEvent: | ||
| commandName: ping # Pre-loading the connection pool. | ||
| - commandFailedEvent: | ||
| commandName: insert | ||
| - client: *client | ||
| eventType: cmap | ||
| events: | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Ping finishes. | ||
| - connectionCheckedOutEvent: {} | ||
| - connectionCheckedInEvent: {} # Insert fails. | ||
| - connectionPendingResponseStarted: {} # First find fails | ||
| - connectionPendingResponseFailed: | ||
| reason: error | ||
| - connectionClosedEvent: | ||
| reason: error | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The UTR runCommand operation requires the
commandName: "ping"field:https://github.com/mongodb/specifications/blob/30d15e0600a798834fb40d36eb468ccf9c49bc04/source/unified-test-format/unified-test-format.md#runcommand