grpc-swift/Tests/GRPCTests/GRPCIdleHandlerStateMachineTests.swift

/*
 * Copyright 2020, gRPC Authors All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import NIOCore
import NIOEmbedded
import NIOHTTP2
import XCTest

@testable import GRPC

class GRPCIdleHandlerStateMachineTests: GRPCTestCase {
  private func makeClientStateMachine() -> GRPCIdleHandlerStateMachine {
    return GRPCIdleHandlerStateMachine(role: .client, logger: self.clientLogger)
  }

  private func makeServerStateMachine() -> GRPCIdleHandlerStateMachine {
    return GRPCIdleHandlerStateMachine(role: .server, logger: self.serverLogger)
  }

  private func makeNoOpScheduled() -> Scheduled<Void> {
    let loop = EmbeddedEventLoop()
    return loop.scheduleTask(deadline: .distantFuture) { return () }
  }

  func testInactiveBeforeSettings() {
    var stateMachine = self.makeClientStateMachine()
    let op1 = stateMachine.channelInactive()
    op1.assertConnectionManager(.inactive)
  }

  func testInactiveAfterSettings() {
    var stateMachine = self.makeClientStateMachine()
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)

    let readyStateMachine = stateMachine

    // Inactive with a stream open.
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()
    let op3 = stateMachine.channelInactive()
    op3.assertConnectionManager(.inactive)

    // Inactive with no open streams.
    stateMachine = readyStateMachine
    let op4 = stateMachine.channelInactive()
    op4.assertConnectionManager(.idle)
  }

  func testInactiveWhenWaitingToIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the timeout.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Become inactive unexpectedly.
    let op3 = stateMachine.channelInactive()
    op3.assertConnectionManager(.idle)
  }

  func testInactiveWhenQuiescing() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)

    // Try a few combinations: initiator of shutdown, and whether streams are open or not when
    // shutdown is initiated.
    let readyStateMachine = stateMachine

    // (1) Peer initiates shutdown, no streams are open.
    do {
      let op2 = stateMachine.receiveGoAway()
      op2.assertGoAway(streamID: .rootStream)
      op2.assertShouldClose()

      // We become idle.
      let op3 = stateMachine.channelInactive()
      op3.assertConnectionManager(.idle)
    }

    // (2) We initiate shutdown, no streams are open.
    stateMachine = readyStateMachine
    do {
      let op2 = stateMachine.initiateGracefulShutdown()
      op2.assertGoAway(streamID: .rootStream)
      op2.assertShouldClose()

      // We become idle.
      let op3 = stateMachine.channelInactive()
      op3.assertConnectionManager(.idle)
    }

    stateMachine = readyStateMachine
    _ = stateMachine.streamCreated(withID: 1)
    let streamOpenStateMachine = stateMachine

    // (3) Peer initiates shutdown, streams are open.
    do {
      let op2 = stateMachine.receiveGoAway()
      op2.assertNoGoAway()
      op2.assertShouldNotClose()

      // We become inactive.
      let op3 = stateMachine.channelInactive()
      op3.assertConnectionManager(.inactive)
    }

    // (4) We initiate shutdown, streams are open.
    stateMachine = streamOpenStateMachine
    do {
      let op2 = stateMachine.initiateGracefulShutdown()
      op2.assertShouldNotClose()

      // We become inactive.
      let op3 = stateMachine.channelInactive()
      op3.assertConnectionManager(.inactive)
    }
  }

  func testReceiveSettings() {
    var stateMachine = self.makeClientStateMachine()

    // No open streams.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Open streams.
    stateMachine = self.makeClientStateMachine()
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()
    let op3 = stateMachine.receiveSettings([])
    // No idle timeout to cancel.
    op3.assertConnectionManager(.ready)
    op3.assertNoIdleTimeoutTask()
  }

  func testReceiveSettingsWhenWaitingToIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Receive more settings.
    let op2 = stateMachine.receiveSettings([])
    op2.assertDoNothing()

    // Schedule the timeout.
    let op3 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op3.assertDoNothing()

    // More settings.
    let op4 = stateMachine.receiveSettings([])
    op4.assertDoNothing()
  }

  func testReceiveGoAwayWhenWaitingToIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the timeout.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Receive a GOAWAY frame.
    let op3 = stateMachine.receiveGoAway()
    op3.assertGoAway(streamID: .rootStream)
    op3.assertShouldClose()
    op3.assertCancelIdleTimeout()
    op3.assertConnectionManager(.quiescing)

    // Close; we were going to go idle anyway.
    let op4 = stateMachine.channelInactive()
    op4.assertConnectionManager(.idle)
  }

  func testInitiateGracefulShutdownWithNoOpenStreams() {
    var stateMachine = self.makeClientStateMachine()

    // No open streams: so GOAWAY and close.
    let op1 = stateMachine.initiateGracefulShutdown()
    op1.assertGoAway(streamID: .rootStream)
    op1.assertShouldClose()
    op1.assertConnectionManager(.quiescing)

    // Closed.
    let op2 = stateMachine.channelInactive()
    op2.assertConnectionManager(.inactive)
  }

  func testInitiateGracefulShutdownWithOpenStreams() {
    var stateMachine = self.makeClientStateMachine()

    // Open a stream.
    let op1 = stateMachine.streamCreated(withID: 1)
    op1.assertDoNothing()

    // Initiate shutdown.
    let op2 = stateMachine.initiateGracefulShutdown()
    op2.assertShouldNotClose()
    op2.assertConnectionManager(.quiescing)

    // Receive a GOAWAY; no change.
    let op3 = stateMachine.receiveGoAway()
    op3.assertDoNothing()

    // Close the remaining open stream, connection should close as a result.
    let op4 = stateMachine.streamClosed(withID: 1)
    op4.assertShouldClose()

    // Connection closed.
    let op5 = stateMachine.channelInactive()
    op5.assertConnectionManager(.inactive)
  }

  func testInitiateGracefulShutdownWhenWaitingToIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become 'ready'
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the task.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Initiate shutdown: cancel the timeout, send a GOAWAY and close.
    let op3 = stateMachine.initiateGracefulShutdown()
    op3.assertCancelIdleTimeout()
    op3.assertGoAway(streamID: .rootStream)
    op3.assertShouldClose()

    // Closed: become inactive.
    let op4 = stateMachine.channelInactive()
    op4.assertConnectionManager(.inactive)
  }

  func testInitiateGracefulShutdownWhenQuiescing() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Open a few streams.
    for streamID in stride(from: HTTP2StreamID(1), to: HTTP2StreamID(6), by: 2) {
      let op = stateMachine.streamCreated(withID: streamID)
      op.assertDoNothing()
    }

    // Receive a GOAWAY.
    let op2 = stateMachine.receiveGoAway()
    op2.assertNoGoAway()

    // Initiate shutdown from our side: we've already sent GOAWAY and have a stream open, we don't
    // need to do anything.
    let op3 = stateMachine.initiateGracefulShutdown()
    op3.assertDoNothing()

    // Close the first couple of streams; should be a no-op.
    for streamID in [HTTP2StreamID(1), HTTP2StreamID(3)] {
      let op = stateMachine.streamClosed(withID: streamID)
      op.assertDoNothing()
    }
    // Close the final stream.
    let op4 = stateMachine.streamClosed(withID: 5)
    op4.assertShouldClose()

    // Initiate shutdown again: we're closing so this should be a no-op.
    let op5 = stateMachine.initiateGracefulShutdown()
    op5.assertDoNothing()

    // Closed.
    let op6 = stateMachine.channelInactive()
    op6.assertConnectionManager(.inactive)
  }

  func testScheduleIdleTaskWhenStreamsAreOpen() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Open a stream before scheduling the task.
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()

    // Schedule an idle timeout task: there are open streams so this should be cancelled.
    let op3 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op3.assertCancelIdleTimeout()
  }

  func testScheduleIdleTaskWhenQuiescing() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Save the state machine so we can test a few branches.
    let readyStateMachine = stateMachine

    // (1) Scheduled when quiescing.
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()
    // Start shutting down.
    _ = stateMachine.initiateGracefulShutdown()
    // Schedule an idle timeout task: we're quiescing, so cancel the task.
    let op4 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op4.assertCancelIdleTimeout()

    // (2) Scheduled when closing.
    stateMachine = readyStateMachine
    let op5 = stateMachine.initiateGracefulShutdown()
    op5.assertGoAway(streamID: .rootStream)
    op5.assertShouldClose()
    // Schedule an idle timeout task: we're already closing, so cancel the task.
    let op6 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op6.assertCancelIdleTimeout()
  }

  func testIdleTimeoutTaskFiresWhenIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the task.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Fire the task.
    let op3 = stateMachine.idleTimeoutTaskFired()
    op3.assertGoAway(streamID: .rootStream)
    op3.assertShouldClose()

    // Close.
    let op4 = stateMachine.channelInactive()
    op4.assertConnectionManager(.idle)
  }

  func testIdleTimeoutTaskFiresWhenClosed() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the task.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Close.
    let op3 = stateMachine.channelInactive()
    op3.assertCancelIdleTimeout()

    // Fire the idle timeout task.
    let op4 = stateMachine.idleTimeoutTaskFired()
    op4.assertDoNothing()
  }

  func testShutdownNow() {
    var stateMachine = self.makeClientStateMachine()

    let op1 = stateMachine.shutdownNow()
    op1.assertGoAway(streamID: .rootStream)
    op1.assertShouldClose()

    let op2 = stateMachine.channelInactive()
    op2.assertConnectionManager(.inactive)
  }

  func testShutdownNowWhenWaitingToIdle() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the task.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    let op3 = stateMachine.shutdownNow()
    op3.assertGoAway(streamID: .rootStream)
    op3.assertShouldClose()

    let op4 = stateMachine.channelInactive()
    op4.assertConnectionManager(.inactive)
  }

  func testShutdownNowWhenQuiescing() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Open a stream.
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()

    // Initiate shutdown.
    let op3 = stateMachine.initiateGracefulShutdown()
    op3.assertNoGoAway()

    // Shutdown now.
    let op4 = stateMachine.shutdownNow()
    op4.assertShouldClose()
  }

  func testNormalFlow() {
    var stateMachine = self.makeClientStateMachine()

    // Become ready.
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the task.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Create a stream to cancel the task.
    let op3 = stateMachine.streamCreated(withID: 1)
    op3.assertCancelIdleTimeout()

    // Close the stream.
    let op4 = stateMachine.streamClosed(withID: 1)
    op4.assertScheduleIdleTimeout()

    // Receive a GOAWAY frame.
    let op5 = stateMachine.receiveGoAway()
    // We're the client, there are no server initiated streams, so GOAWAY with root stream.
    op5.assertGoAway(streamID: 0)
    // No open streams, so we can close now. Also assert the connection manager got a quiescing event.
    op5.assertShouldClose()
    op5.assertConnectionManager(.quiescing)

    // Closed.
    let op6 = stateMachine.channelInactive()
    // The peer initiated shutdown by sending GOAWAY, we'll idle.
    op6.assertConnectionManager(.idle)
  }

  func testClientSendsGoAwayAndOpensStream() {
    var stateMachine = self.makeServerStateMachine()

    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)
    op1.assertScheduleIdleTimeout()

    // Schedule the idle timeout.
    let op2 = stateMachine.scheduledIdleTimeoutTask(self.makeNoOpScheduled())
    op2.assertDoNothing()

    // Create a stream to cancel the task.
    let op3 = stateMachine.streamCreated(withID: 1)
    op3.assertCancelIdleTimeout()

    // Receive a GOAWAY frame from the client.
    let op4 = stateMachine.receiveGoAway()
    op4.assertGoAway(streamID: .maxID)
    op4.assertShouldPingAfterGoAway()
    op4.assertConnectionManager(.quiescing)

    // Create another stream. This is fine, the client hasn't ack'd the ping yet.
    let op5 = stateMachine.streamCreated(withID: 7)
    op5.assertDoNothing()

    // Receiving the ping is handled by a different state machine which will tell us to ratchet
    // down the go away stream ID.
    let op6 = stateMachine.ratchetDownGoAwayStreamID()
    op6.assertGoAway(streamID: 7)
    op6.assertShouldNotPingAfterGoAway()

    let op7 = stateMachine.streamClosed(withID: 7)
    op7.assertDoNothing()

    let op8 = stateMachine.streamClosed(withID: 1)
    op8.assertShouldClose()
  }

  func testRatchetDownStreamIDWhenNotQuiescing() {
    var stateMachine = self.makeServerStateMachine()
    _ = stateMachine.receiveSettings([])

    // from the 'operating' state.
    stateMachine.ratchetDownGoAwayStreamID().assertDoNothing()

    // move to the 'waiting to idle' state.
    let promise = EmbeddedEventLoop().makePromise(of: Void.self)
    let task = Scheduled(promise: promise, cancellationTask: {})
    stateMachine.scheduledIdleTimeoutTask(task).assertDoNothing()
    promise.succeed(())
    stateMachine.ratchetDownGoAwayStreamID().assertDoNothing()

    // move to 'closing'
    _ = stateMachine.idleTimeoutTaskFired()
    stateMachine.ratchetDownGoAwayStreamID().assertDoNothing()

    // move to 'closed'
    _ = stateMachine.channelInactive()
    stateMachine.ratchetDownGoAwayStreamID().assertDoNothing()
  }

  func testStreamIDWhenQuiescing() {
    var stateMachine = self.makeClientStateMachine()
    let op1 = stateMachine.receiveSettings([])
    op1.assertConnectionManager(.ready)

    // Open a stream so we enter quiescing when receiving the GOAWAY.
    let op2 = stateMachine.streamCreated(withID: 1)
    op2.assertDoNothing()

    let op3 = stateMachine.receiveGoAway()
    op3.assertConnectionManager(.quiescing)

    // Create a new stream. This can happen if the GOAWAY races with opening the stream; HTTP2 will
    // open and then close the stream with an error.
    let op4 = stateMachine.streamCreated(withID: 3)
    op4.assertDoNothing()

    // Close the newly opened stream.
    let op5 = stateMachine.streamClosed(withID: 3)
    op5.assertDoNothing()

    // Close the original stream.
    let op6 = stateMachine.streamClosed(withID: 1)
    // Now we can send a GOAWAY with stream ID zero (we're the client and the server didn't open
    // any streams).
    XCTAssertEqual(op6.sendGoAwayWithLastPeerInitiatedStreamID, 0)
  }
}

extension GRPCIdleHandlerStateMachine.Operations {
  func assertDoNothing() {
    XCTAssertNil(self.connectionManagerEvent)
    XCTAssertNil(self.idleTask)
    XCTAssertNil(self.sendGoAwayWithLastPeerInitiatedStreamID)
    XCTAssertFalse(self.shouldCloseChannel)
    XCTAssertFalse(self.shouldPingAfterGoAway)
  }

  func assertGoAway(streamID: HTTP2StreamID) {
    XCTAssertEqual(self.sendGoAwayWithLastPeerInitiatedStreamID, streamID)
  }

  func assertNoGoAway() {
    XCTAssertNil(self.sendGoAwayWithLastPeerInitiatedStreamID)
  }

  func assertScheduleIdleTimeout() {
    switch self.idleTask {
    case .some(.schedule):
      ()
    case .some(.cancel), .none:
      XCTFail("Expected 'schedule' but was '\(String(describing: self.idleTask))'")
    }
  }

  func assertCancelIdleTimeout() {
    switch self.idleTask {
    case .some(.cancel):
      ()
    case .some(.schedule), .none:
      XCTFail("Expected 'cancel' but was '\(String(describing: self.idleTask))'")
    }
  }

  func assertNoIdleTimeoutTask() {
    XCTAssertNil(self.idleTask)
  }

  func assertConnectionManager(_ event: GRPCIdleHandlerStateMachine.ConnectionManagerEvent) {
    XCTAssertEqual(self.connectionManagerEvent, event)
  }

  func assertNoConnectionManagerEvent() {
    XCTAssertNil(self.connectionManagerEvent)
  }

  func assertShouldClose() {
    XCTAssertTrue(self.shouldCloseChannel)
  }

  func assertShouldNotClose() {
    XCTAssertFalse(self.shouldCloseChannel)
  }

  func assertShouldPingAfterGoAway() {
    XCTAssert(self.shouldPingAfterGoAway)
  }

  func assertShouldNotPingAfterGoAway() {
    XCTAssertFalse(self.shouldPingAfterGoAway)
  }
}