Add more flexibility to perf tests (#1354)

Motivation:

It's useful to run the perf tests in different configurations, e.g. to
compare NIOTS with NIOPosix or to measure the affect of using TLS.

Modifications:

Add two options to the perf test suite:
- `--use-nio-transport-services`: uses a NIOTS event loop group if
  available on the host platform and,
- `--use-tls`: uses TLS if the test runs both a client and server

Also exposes a `--repeats` option.

Result:

Tests are more flexible.

Package.swift

@@ -232,6 +232,7 @@ extension Target {
     name: "GRPCPerformanceTests",
     dependencies: [
       .grpc,
+      .grpcSampleData,
       .nioCore,
       .nioEmbedded,
       .nioPosix,

Sources/GRPCPerformanceTests/Benchmarks/ServerProvidingBenchmark.swift

@@ -14,26 +14,56 @@
  * limitations under the License.
  */
 import GRPC
+import GRPCSampleData
 import NIOCore
 import NIOPosix
 
 class ServerProvidingBenchmark: Benchmark {
   private let providers: [CallHandlerProvider]
   private let threadCount: Int
+  private let useNIOTSIfAvailable: Bool
+  private let useTLS: Bool
   private var group: EventLoopGroup!
   private(set) var server: Server!
 
-  init(providers: [CallHandlerProvider], threadCount: Int = 1) {
+  init(
+    providers: [CallHandlerProvider],
+    useNIOTSIfAvailable: Bool,
+    useTLS: Bool,
+    threadCount: Int = 1
+  ) {
     self.providers = providers
+    self.useNIOTSIfAvailable = useNIOTSIfAvailable
+    self.useTLS = useTLS
     self.threadCount = threadCount
   }
 
   func setUp() throws {
-    self.group = MultiThreadedEventLoopGroup(numberOfThreads: self.threadCount)
-    self.server = try Server.insecure(group: self.group)
-      .withServiceProviders(self.providers)
-      .bind(host: "127.0.0.1", port: 0)
-      .wait()
+    if self.useNIOTSIfAvailable {
+      self.group = PlatformSupport.makeEventLoopGroup(loopCount: self.threadCount)
+    } else {
+      self.group = MultiThreadedEventLoopGroup(numberOfThreads: self.threadCount)
+    }
+
+    if self.useTLS {
+      #if canImport(NIOSSL)
+      self.server = try Server.usingTLSBackedByNIOSSL(
+        on: self.group,
+        certificateChain: [SampleCertificate.server.certificate],
+        privateKey: SamplePrivateKey.server
+      ).withTLS(trustRoots: .certificates([SampleCertificate.ca.certificate]))
+        .withServiceProviders(self.providers)
+        .bind(host: "127.0.0.1", port: 0)
+        .wait()
+      #else
+      fatalError("NIOSSL must be imported to use TLS")
+      #endif
+    } else {
+      self.server = try Server.insecure(group: self.group)
+        .withServiceProviders(self.providers)
+        .bind(host: "127.0.0.1", port: 0)
+        .wait()
+    }
   }
 
   func tearDown() throws {

Sources/GRPCPerformanceTests/Benchmarks/UnaryThroughput.swift

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import GRPC
+import GRPCSampleData
 import NIOCore
 import NIOPosix
 
@@ -22,23 +23,50 @@ import NIOPosix
 /// Requests are sent in batches of (up-to) 100 requests. This is due to
 /// https://github.com/apple/swift-nio-http2/issues/87#issuecomment-483542401.
 class Unary: ServerProvidingBenchmark {
+  private let useNIOTSIfAvailable: Bool
+  private let useTLS: Bool
   private var group: EventLoopGroup!
   private(set) var client: Echo_EchoClient!
 
   let requestCount: Int
   let requestText: String
 
-  init(requests: Int, text: String) {
+  init(requests: Int, text: String, useNIOTSIfAvailable: Bool, useTLS: Bool) {
+    self.useNIOTSIfAvailable = useNIOTSIfAvailable
+    self.useTLS = useTLS
     self.requestCount = requests
     self.requestText = text
-    super.init(providers: [MinimalEchoProvider()])
+    super.init(
+      providers: [MinimalEchoProvider()],
+      useNIOTSIfAvailable: useNIOTSIfAvailable,
+      useTLS: useTLS
+    )
   }
 
   override func setUp() throws {
     try super.setUp()
-    self.group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-    let channel = ClientConnection.insecure(group: self.group)
-      .connect(host: "127.0.0.1", port: self.server.channel.localAddress!.port!)
+
+    if self.useNIOTSIfAvailable {
+      self.group = PlatformSupport.makeEventLoopGroup(loopCount: 1)
+    } else {
+      self.group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
+    }
+
+    let channel: ClientConnection
+
+    if self.useTLS {
+      #if canImport(NIOSSL)
+      channel = ClientConnection.usingTLSBackedByNIOSSL(on: self.group)
+        .withTLS(trustRoots: .certificates([SampleCertificate.ca.certificate]))
+        .withTLS(serverHostnameOverride: "localhost")
+        .connect(host: "127.0.0.1", port: self.server.channel.localAddress!.port!)
+      #else
+      fatalError("NIOSSL must be imported to use TLS")
+      #endif
+    } else {
+      channel = ClientConnection.insecure(group: self.group)
+        .connect(host: "127.0.0.1", port: self.server.channel.localAddress!.port!)
+    }
 
     self.client = .init(channel: channel)
   }
@@ -72,9 +100,14 @@ class Unary: ServerProvidingBenchmark {
 class Bidi: Unary {
   let batchSize: Int
 
-  init(requests: Int, text: String, batchSize: Int) {
+  init(requests: Int, text: String, batchSize: Int, useNIOTSIfAvailable: Bool, useTLS: Bool) {
     self.batchSize = batchSize
-    super.init(requests: requests, text: text)
+    super.init(
+      requests: requests,
+      text: text,
+      useNIOTSIfAvailable: useNIOTSIfAvailable,
+      useTLS: useTLS
+    )
   }
 
   override func run() throws -> Int {

Sources/GRPCPerformanceTests/main.swift

@@ -24,31 +24,59 @@ let largeRequest = String(repeating: "x", count: 1 << 16) // 65k
 func runBenchmarks(spec: TestSpec) {
   measureAndPrint(
     description: "unary_10k_small_requests",
-    benchmark: Unary(requests: 10000, text: smallRequest),
+    benchmark: Unary(
+      requests: 10000,
+      text: smallRequest,
+      useNIOTSIfAvailable: spec.useNIOTransportServices,
+      useTLS: spec.useTLS
+    ),
     spec: spec
   )
 
   measureAndPrint(
     description: "unary_10k_long_requests",
-    benchmark: Unary(requests: 10000, text: largeRequest),
+    benchmark: Unary(
+      requests: 10000,
+      text: largeRequest,
+      useNIOTSIfAvailable: spec.useNIOTransportServices,
+      useTLS: spec.useTLS
+    ),
     spec: spec
   )
 
   measureAndPrint(
     description: "bidi_10k_small_requests_in_batches_of_1",
-    benchmark: Bidi(requests: 10000, text: smallRequest, batchSize: 1),
+    benchmark: Bidi(
+      requests: 10000,
+      text: smallRequest,
+      batchSize: 1,
+      useNIOTSIfAvailable: spec.useNIOTransportServices,
+      useTLS: spec.useTLS
+    ),
     spec: spec
   )
 
   measureAndPrint(
     description: "bidi_10k_small_requests_in_batches_of_5",
-    benchmark: Bidi(requests: 10000, text: smallRequest, batchSize: 5),
+    benchmark: Bidi(
+      requests: 10000,
+      text: smallRequest,
+      batchSize: 5,
+      useNIOTSIfAvailable: spec.useNIOTransportServices,
+      useTLS: spec.useTLS
+    ),
     spec: spec
   )
 
   measureAndPrint(
     description: "bidi_1k_large_requests_in_batches_of_5",
-    benchmark: Bidi(requests: 1000, text: largeRequest, batchSize: 1),
+    benchmark: Bidi(
+      requests: 1000,
+      text: largeRequest,
+      batchSize: 1,
+      useNIOTSIfAvailable: spec.useNIOTransportServices,
+      useTLS: spec.useTLS
+    ),
     spec: spec
   )
 
@@ -153,10 +181,14 @@ func runBenchmarks(spec: TestSpec) {
 struct TestSpec {
   var action: Action
   var repeats: Int
+  var useNIOTransportServices: Bool
+  var useTLS: Bool
 
-  init(action: Action, repeats: Int = 10) {
+  init(action: Action, repeats: Int, useNIOTransportServices: Bool, useTLS: Bool) {
     self.action = action
     self.repeats = repeats
+    self.useNIOTransportServices = useNIOTransportServices
+    self.useTLS = useTLS
   }
 
   enum Action {
@@ -190,6 +222,15 @@ struct PerformanceTests: ParsableCommand {
   @Flag(name: .shortAndLong, help: "Run all tests")
   var all: Bool = false
 
+  @Flag(help: "Use NIO Transport Services (if available)")
+  var useNIOTransportServices: Bool = false
+
+  @Flag(help: "Use TLS for tests which support it")
+  var useTLS: Bool = false
+
+  @Option(help: "The number of times to run each test")
+  var repeats: Int = 10
+
   @Argument(help: "The tests to run")
   var tests: [String] = []
 
@@ -197,11 +238,26 @@ struct PerformanceTests: ParsableCommand {
     let spec: TestSpec
 
     if self.list {
-      spec = TestSpec(action: .list)
+      spec = TestSpec(
+        action: .list,
+        repeats: self.repeats,
+        useNIOTransportServices: self.useNIOTransportServices,
+        useTLS: self.useTLS
+      )
     } else if self.all {
-      spec = TestSpec(action: .run(.all))
+      spec = TestSpec(
+        action: .run(.all),
+        repeats: self.repeats,
+        useNIOTransportServices: self.useNIOTransportServices,
+        useTLS: self.useTLS
+      )
     } else {
-      spec = TestSpec(action: .run(.some(self.tests)))
+      spec = TestSpec(
+        action: .run(.some(self.tests)),
+        repeats: self.repeats,
+        useNIOTransportServices: self.useNIOTransportServices,
+        useTLS: self.useTLS
+      )
     }
 
     runBenchmarks(spec: spec)