/* * Copyright 2019, gRPC Authors All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import Foundation import GRPC import NIO import NIOSSL import EchoImplementation import EchoModel import Logging struct ConnectionFactory { var configuration: ClientConnection.Configuration func makeConnection() -> ClientConnection { return ClientConnection(configuration: self.configuration) } func makeEchoClient() -> Echo_EchoServiceClient { return Echo_EchoServiceClient(connection: self.makeConnection()) } } protocol Benchmark: class { func setUp() throws func tearDown() throws func run() throws } /// Tests unary throughput by sending requests on a single connection. /// /// Requests are sent in batches of (up-to) 100 requests. This is due to /// https://github.com/apple/swift-nio-http2/issues/87#issuecomment-483542401. class UnaryThroughput: Benchmark { let factory: ConnectionFactory let requests: Int let requestLength: Int var client: Echo_EchoServiceClient! var request: String! init(factory: ConnectionFactory, requests: Int, requestLength: Int) { self.factory = factory self.requests = requests self.requestLength = requestLength } func setUp() throws { self.client = self.factory.makeEchoClient() self.request = String(repeating: "0", count: self.requestLength) } func run() throws { let batchSize = 100 for lowerBound in stride(from: 0, to: self.requests, by: batchSize) { let upperBound = min(lowerBound + batchSize, self.requests) let requests = (lowerBound.. var ready: EventLoopFuture { return promise.futureResult } init(promise: EventLoopPromise) { self.promise = promise } func connectivityStateDidChange(from oldState: ConnectivityState, to newState: ConnectivityState) { switch newState { case .ready: promise.succeed(()) case .shutdown: promise.fail(GRPCStatus(code: .unavailable, message: nil)) default: break } } } init(factory: ConnectionFactory, connections: Int) { self.factory = factory self.connections = connections } func setUp() throws { } func run() throws { let connectionsAndDelegates: [(ClientConnection, ConnectionReadinessDelegate)] = (0.., [, ] var asCSV: String { let items = [self.benchmarkDescription, String(self.durations.count)] + self.durations.map { String($0) } return items.joined(separator: ", ") } } /// Runs the given benchmark multiple times, recording the wall time for each iteration. /// /// - Parameter description: A description of the benchmark. /// - Parameter benchmark: The benchmark to run. /// - Parameter repeats: The number of times to run the benchmark. func measure(description: String, benchmark: Benchmark, repeats: Int) -> BenchmarkResults { var durations: [TimeInterval] = [] for _ in 0.. Server.Configuration.TLS? { // Commander doesn't have Optional options; we use empty strings to indicate no value. guard certificatePath.isEmpty == privateKeyPath.isEmpty && privateKeyPath.isEmpty == caCertificatePath.isEmpty else { print("Paths for CA certificate, certificate and private key must be provided") exit(1) } // No need to check them all because of the guard statement above. if caCertificatePath.isEmpty { return nil } return .init( certificateChain: try NIOSSLCertificate.fromPEMFile(certificatePath).map { .certificate($0) }, privateKey: .file(privateKeyPath), trustRoots: .file(caCertificatePath) ) } private func makeClientTLSConfiguration( caCertificatePath: String, certificatePath: String, privateKeyPath: String ) throws -> ClientConnection.Configuration.TLS? { // Commander doesn't have Optional options; we use empty strings to indicate no value. guard certificatePath.isEmpty == privateKeyPath.isEmpty && privateKeyPath.isEmpty == caCertificatePath.isEmpty else { print("Paths for CA certificate, certificate and private key must be provided") exit(1) } // No need to check them all because of the guard statement above. if caCertificatePath.isEmpty { return nil } return .init( certificateChain: try NIOSSLCertificate.fromPEMFile(certificatePath).map { .certificate($0) }, privateKey: .file(privateKeyPath), trustRoots: .file(caCertificatePath) ) } enum Benchmarks: String, CaseIterable { case unaryThroughputSmallRequests = "unary_throughput_small" case unaryThroughputLargeRequests = "unary_throughput_large" case bidirectionalThroughputSmallRequests = "bidi_throughput_small" case bidirectionalThroughputLargeRequests = "bidi_throughput_large" case connectionThroughput = "connection_throughput" static let smallRequest = 8 static let largeRequest = 1 << 16 var description: String { switch self { case .unaryThroughputSmallRequests: return "10k unary requests of size \(Benchmarks.smallRequest)" case .unaryThroughputLargeRequests: return "10k unary requests of size \(Benchmarks.largeRequest)" case .bidirectionalThroughputSmallRequests: return "20k bidirectional messages of size \(Benchmarks.smallRequest)" case .bidirectionalThroughputLargeRequests: return "10k bidirectional messages of size \(Benchmarks.largeRequest)" case .connectionThroughput: return "100 connections created" } } func makeBenchmark(factory: ConnectionFactory) -> Benchmark { switch self { case .unaryThroughputSmallRequests: return UnaryThroughput(factory: factory, requests: 10_000, requestLength: Benchmarks.smallRequest) case .unaryThroughputLargeRequests: return UnaryThroughput(factory: factory, requests: 10_000, requestLength: Benchmarks.largeRequest) case .bidirectionalThroughputSmallRequests: return BidirectionalThroughput(factory: factory, requests: 20_000, requestLength: Benchmarks.smallRequest) case .bidirectionalThroughputLargeRequests: return BidirectionalThroughput(factory: factory, requests: 10_000, requestLength: Benchmarks.largeRequest) case .connectionThroughput: return ConnectionCreationThroughput(factory: factory, connections: 100) } } func run(using factory: ConnectionFactory, repeats: Int = 10) -> BenchmarkResults { let benchmark = self.makeBenchmark(factory: factory) return measure(description: self.description, benchmark: benchmark, repeats: repeats) } } enum Command { case listBenchmarks case benchmark(name: String, host: String, port: Int, tls: (ca: String, cert: String)?) case server(port: Int, tls: (ca: String, cert: String, key: String)?) init?(from args: [String]) { guard !args.isEmpty else { return nil } var args = args let command = args.removeFirst() switch command { case "server": guard let port = args.popLast().flatMap(Int.init) else { return nil } let caPath = args.suffixOfFirst(prefixedWith: "--caPath=") let certPath = args.suffixOfFirst(prefixedWith: "--certPath=") let keyPath = args.suffixOfFirst(prefixedWith: "--keyPath=") // We need all or nothing here: switch (caPath, certPath, keyPath) { case let (.some(ca), .some(cert), .some(key)): self = .server(port: port, tls: (ca: ca, cert: cert, key: key)) case (.none, .none, .none): self = .server(port: port, tls: nil) default: return nil } case "benchmark": guard let name = args.popLast(), let port = args.popLast().flatMap(Int.init), let host = args.popLast() else { return nil } let caPath = args.suffixOfFirst(prefixedWith: "--caPath=") let certPath = args.suffixOfFirst(prefixedWith: "--certPath=") // We need all or nothing here: switch (caPath, certPath) { case let (.some(ca), .some(cert)): self = .benchmark(name: name, host: host, port: port, tls: (ca: ca, cert: cert)) case (.none, .none): self = .benchmark(name: name, host: host, port: port, tls: nil) default: return nil } case "list_benchmarks": self = .listBenchmarks default: return nil } } } func printUsageAndExit(program: String) -> Never { print(""" Usage: \(program) COMMAND [OPTIONS...] benchmark: Run the given benchmark (see 'list_benchmarks' for possible options) against a server on the specified host and port. TLS may be used by spefifying the path to the PEM formatted certificate and CA certificate. benchmark [--ca=CA --cert=CERT] HOST PORT BENCHMARK_NAME Note: eiether all or none of CA and CERT must be provided. list_benchmarks: List the available benchmarks to run. server: Start the server on the given PORT. TLS may be used by specifying the paths to the PEM formatted certificate, private key and CA certificate. server [--ca=CA --cert=CERT --key=KEY] PORT Note: eiether all or none of CA, CERT and KEY must be provided. """) exit(1) } fileprivate extension Array where Element == String { func suffixOfFirst(prefixedWith prefix: String) -> String? { return self.first { $0.hasPrefix(prefix) }.map { String($0.dropFirst(prefix.count)) } } } func main(args: [String]) { var args = args let program = args.removeFirst() guard let command = Command(from: args) else { printUsageAndExit(program: program) } switch command { case let .server(port: port, tls: tls): let group = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) defer { try! group.syncShutdownGracefully() } // Quieten the logs. LoggingSystem.bootstrap { var handler = StreamLogHandler.standardOutput(label: $0) handler.logLevel = .warning return handler } do { let configuration = try Server.Configuration( target: .hostAndPort("localhost", port), eventLoopGroup: group, serviceProviders: [EchoProvider()], tls: tls.map { tlsArgs in return .init( certificateChain: try NIOSSLCertificate.fromPEMFile(tlsArgs.cert).map { .certificate($0) }, privateKey: .file(tlsArgs.key), trustRoots: .file(tlsArgs.ca) ) } ) let server = try Server.start(configuration: configuration).wait() print("server started on port: \(server.channel.localAddress?.port ?? port)") // Stop the program from exiting. try? server.onClose.wait() } catch { print("unable to start server: \(error)") exit(1) } case let .benchmark(name: name, host: host, port: port, tls: tls): guard let benchmark = Benchmarks(rawValue: name) else { printUsageAndExit(program: program) } // Quieten the logs. LoggingSystem.bootstrap { var handler = StreamLogHandler.standardOutput(label: $0) handler.logLevel = .critical return handler } let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) defer { try! group.syncShutdownGracefully() } do { let configuration = try ClientConnection.Configuration( target: .hostAndPort(host, port), eventLoopGroup: group, tls: tls.map { tlsArgs in return .init( certificateChain: try NIOSSLCertificate.fromPEMFile(tlsArgs.cert).map { .certificate($0) }, trustRoots: .file(tlsArgs.ca) ) } ) let factory = ConnectionFactory(configuration: configuration) let results = benchmark.run(using: factory) print(results.asCSV) } catch { print("unable to run benchmark: \(error)") exit(1) } case .listBenchmarks: Benchmarks.allCases.forEach { print($0.rawValue) } } } main(args: CommandLine.arguments)