Capture packets using man-in-the-middle (MITM) proxy

I am trying to implement an iOS proxy running inside the network extension to log the packets.
I am referring the connect-proxy example which relays the data properly.
From what I have read, to capture packet data, I'll have to install and trust a root certificate on iOS device and use that but am kinda stuck at this.

I generated my self-signed certificate with:

openssl genrsa -passout pass:my_pass -out key.pem 8192
openssl req -new -x509 -key key.pem -out root-ca.pem

And installed & trusted it on my iOS device.

My local proxy configuration in the extension looks something like:

 private func startLocalProxy() {
        logger.logLevel = .debug

        let bundle = Bundle(for: type(of: self))
        
        guard let certificatePath = bundle.path(forResource: "root-ca", ofType: "pem"),
              let keyPath = bundle.path(forResource: "key", ofType: "pem"),
              let certChain = (try? NIOSSLCertificate.fromPEMFile(certificatePath).map{ NIOSSLCertificateSource.certificate($0) }) else {
                  logger.error("Failed to setup certificate chain")
                  self.pendingCompletion?(NEVPNError(.connectionFailed))
                  return
              }
        
        
        let sslPrivateKey = try! NIOSSLPrivateKeySource.privateKey(NIOSSLPrivateKey(file: keyPath, format: .pem) { providePassword in
            providePassword("my_pass".utf8)
        })
                
        var serverConfiguration = TLSConfiguration.makeServerConfiguration( certificateChain: certChain, privateKey: sslPrivateKey)
        serverConfiguration.applicationProtocols = NIOHTTP2SupportedALPNProtocols
        let serverContext = try! NIOSSLContext(configuration: serverConfiguration)
        
        var clientConfiguration = TLSConfiguration.makeClientConfiguration()
        clientConfiguration.certificateVerification = .none
        let clientContext = try! NIOSSLContext(configuration: clientConfiguration)

        self.startBasicProxy(serverContext: serverContext, clientContext: clientContext)
    }
    
    
    private func startBasicProxy(serverContext: NIOSSLContext, clientContext: NIOSSLContext) {
        let bootstrap = ServerBootstrap(group: localProxyGroup)
            .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1)
           .childChannelInitializer { channel in
              channel.pipeline.addHandler(ByteToMessageHandler(HTTPRequestDecoder(leftOverBytesStrategy: .forwardBytes)), name: "http-request-decoder")
                  .flatMap { channel.pipeline.addHandler(HTTPResponseEncoder(), name: "http-response-encoder") }
                  .flatMap { channel.pipeline.addHandler(ConnectHandler(logger: Logger(label: "com.apple.nio-connect-proxy.ConnectHandler"), serverContext: serverContext, clientContext: clientContext), name: "connect-handler") }
                  .flatMap { channel.pipeline.addHandler(ErrorHandler(), name: "error-handler") }
           }

        bootstrap.bind(host: configuration.hostname, port: Int(configuration.port)!).whenComplete { result in
            // Need to create this here for thread-safety purposes
            switch result {
            case .success(let channel):
                self.logger.info("Listening on \(String(describing: channel.localAddress))")
                self.startTunnel()
            case .failure(let error):
                self.logger.error("Failed to bind 127.0.0.1:8028, \(error)")
                self.pendingCompletion?(NEVPNError(.connectionFailed))
                self.pendingCompletion = nil
            }
        }
    }

And inside ConnectHandler, I also add the TLS handler after CONNECT like:

 private func glue(_ peerChannel: Channel, context: ChannelHandlerContext) {
        self.logger.debug("Gluing together \(ObjectIdentifier(context.channel)) and \(ObjectIdentifier(peerChannel))")

        // Ok, upgrade has completed! We now need to begin the upgrade process.
        // First, send the 200 message.
        // This content-length header is MUST NOT, but we need to workaround NIO's insistence that we set one.
        let headers = HTTPHeaders([("Content-Length", "0")])
        let head = HTTPResponseHead(version: .init(major: 1, minor: 1), status: .ok, headers: headers)
        context.write(self.wrapOutboundOut(.head(head)), promise: nil)
        context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil)

        // Now remove the HTTP encoder.
        self.removeEncoder(context: context)
        context.pipeline.removeHandler(self, promise: nil)

        // Upgrade to tls
        let tlsHandler = NIOSSLServerHandler(context: serverContext)
        context.channel.pipeline.addHandler(tlsHandler, name: "tls-handler", position: .first).whenComplete { result1 in
            // Now we need to glue our channel and the peer channel together.
            let (localGlue, peerGlue) = GlueHandler.matchedPair()
            context.channel.pipeline.addHandler(localGlue).and(peerChannel.pipeline.addHandler(peerGlue)).whenComplete { result in
                switch result {
                case .success(_):
                    print("Added handler")
                case .failure(_):
                    // Close connected peer channel before closing our channel.
                    peerChannel.close(mode: .all, promise: nil)
                    context.close(promise: nil)
                }
            }
        }
    }

I also add a SSL handler for the client:

 private func connectTo(host: String, port: Int, context: ChannelHandlerContext) {
        self.logger.info("Connecting to \(host):\(port)")

        let channelFuture = ClientBootstrap(group: context.eventLoop)
            .channelInitializer { channel in
                channel.pipeline.addHandlers(try! NIOSSLClientHandler(context: self.clientContext, serverHostname: host),
                                             CloseOnErrorHandler(logger: self.logger))
            }
            .connect(host: String(host), port: port)

        channelFuture.whenSuccess { channel in
            self.connectSucceeded(channel: channel, context: context)
        }
        channelFuture.whenFailure { error in
            self.connectFailed(error: error, context: context)
        }
    }

But I keep getting handshakeFailed whenever I try to load a page in Safari.
Can someone point me in the right direction? Been banging my head against the wall :slight_smile:
Would appreciate any help!

2 Likes

Your root certificate is highly unlikely to be trusted by iOS. Does Safari provide any error message?

Concretely, the problems you have with your cert are many, but the biggest one is that it probably isn't valid for the hostname in question. You'll need to generate certificates on the fly that are valid for the appropriate hostnames.

I created a root certificate with:

openssl genrsa -aes256 -out root-key.pem 4096
openssl req -x509 -new -nodes -key root-key.pem -sha256 -days 1825 -out root-ca.pem

And added that to iOS VPN & Device Management by serving it from local server spawned inside the host app.
I also enabled full trust from iOS Settings > General > About > Certificate Trust settings

For now instead of generating the certificate on fly, I created server and client certificates with:

openssl genrsa -out server-key.pem 4096
openssl req -new -key server-key.pem -config ../root.conf -out server.csr
openssl x509 -req -in server.csr -CA root-ca.pem -CAkey root-key.pem -CAcreateserial -days 1825 -sha256 -out server.crt 

openssl genrsa -out client-key.pem 4096
openssl req -new -key client-key.pem -config ../client.conf -out client.csr
openssl x509 -req -in client.csr -CA root-ca.pem -CAkey root-key.pem -CAcreateserial -days 1825 -sha256 -out client.crt 

And added forums.swift.org as Common Name, along with:

DNS.1   = forums.swift.org
DNS.2   = swift.org
DNS.3   = www.swift.org

Now if I run the proxy and open https://forums.swift.org in safari it says "This Connection is not Private"

And if I view the certificate it says "Not trusted":

However I have trusted the certificate in VPN & Device Management and the client/server certificate are signed with the root CA


A few questions to make sure I am doing it right:

  • Should the ConnectHandler (from swift-nio-examples) add a NIOSSLServerHandler when it sets up the Gluehandler?
  • Should I add the NIOSSLClientHandler to the ClientBootstrap In ConnectHandler?

Thoughts?

And I see handShakeFailed error in GlueHandler:

▿ NIOSSLError
  ▿ handshakeFailed : BoringSSLError
    - sslError : 0 elements
Printing description of error:
▿ NIOSSLError
  ▿ handshakeFailed : BoringSSLError
    ▿ sslError : 1 element
      ▿ 0 : Error: 268435613 error:1000009d:SSL routines:OPENSSL_internal:INAPPROPRIATE_FALLBACK
        - errorCode : 268435613

This is a handshake fail, I think you should use a ApplicationProtocolNegotiationHandler to complete the negotiation. @lukasa Hi, lukasa, What is your suggestion?

ALPN is not at fault here: we aren't even successfully completing the TLS handshake.

Why are we creating client certs? Additionally, can you please show me the root cert PEM file (the one that was installed in your iOS device trust settings) as well as the server cert?

Yeah you're right we don't need the client certs.
Here is the root and server cert, Dropbox - mitm-cert.zip - Simplify your life
The server cert was signed with root and seems to verify correctly with:

openssl verify -CAfile certs/root-ca.crt certs/server.crt

@lukasa Also here's a sample project: Dropbox - vpn-mitm-proxy.zip - Simplify your life

If I had to guess, Security.framework probably doesn't like either or both of your CA and root at the very least. Certificate authorities need appropriate BasicConstraints, KeyUsage, and ExtendedKeyUsage extension values, as do the server certificates.

Thanks @lukasa I was missing that from the certificate and also the validity needs to be 825 days or fewer as mentioned here: Requirements for trusted certificates in iOS 13 and macOS 10.15 - Apple Support

Next part is to write the incoming decrypted data from iPhone via my local proxy server to the remote server via the GlueHandler where I get this error:

NIO-ELT-0-#3 (7): Fatal error: tried to decode as type HTTPPart<HTTPRequestHead, IOData> but found HTTPPart<HTTPRequestHead, ByteBuffer> with contents other(NIOHTTP1.HTTPPart<NIOHTTP1.HTTPRequestHead, NIOCore.ByteBuffer>.head(HTTPRequestHead { method: GET, uri: "/", version: HTTP/1.1, headers: [("Host", "forums.swift.org"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Language", "en-IN,en-GB;q=0.9,en;q=0.8"), ("Connection", "keep-alive"), ("Accept-Encoding", "gzip, deflate, br"), 

My local server pipeline is:

                                                                                                          [I] ↓↑ [O]
                                                                                          NIOSSLServerHandler ↓↑ NIOSSLServerHandler            [ssl-handler]
                                                                                                              ↓↑ HTTPResponseEncoder            [handler0]
 ByteToMessageHandler<HTTPDecoder<HTTPPart<HTTPRequestHead, ByteBuffer>, HTTPPart<HTTPResponseHead, IOData>>> ↓↑                                [handler1]
                                                                                    HTTPServerPipelineHandler ↓↑ HTTPServerPipelineHandler      [handler2]
                                                                               HTTPServerProtocolErrorHandler ↓↑ HTTPServerProtocolErrorHandler [handler3]
                                                                                                  GlueHandler ↓↑ GlueHandler                    [local-glue]

And the pipeline to connect to remote server is:

                                                                                                          [I] ↓↑ [O]
                                                                                          NIOSSLClientHandler ↓↑ NIOSSLClientHandler                                                                                          [handler0]
                                                                                          CloseOnErrorHandler ↓↑                                                                                                              [handler1]
                                                                                                              ↓↑ HTTPRequestEncoder                                                                                           [handler2]
 ByteToMessageHandler<HTTPDecoder<HTTPPart<HTTPResponseHead, ByteBuffer>, HTTPPart<HTTPRequestHead, IOData>>> ↓↑ ByteToMessageHandler<HTTPDecoder<HTTPPart<HTTPResponseHead, ByteBuffer>, HTTPPart<HTTPRequestHead, IOData>>> [handler3]
                                                                                                  GlueHandler ↓↑ GlueHandler                                                                                                  [peer-glue]

Looks like it's trying to write data as HTTPPart<HTTPRequestHead, IOData> but the Glue is reading the request from local proxy as <HTTPRequestHead, ByteBuffer>

My client is configured as:

ClientBootstrap(group: context.eventLoop)
            .channelInitializer { channel in
                channel.pipeline.addHandlers(try! NIOSSLClientHandler(context: sslContext, serverHostname: nil), CloseOnErrorHandler(logger: self.logger))
                    .flatMap{ channel.pipeline.addHTTPClientHandlers() }
            }
            .connect(host: String(host), port: port)

Okay @lukasa that IOData/ByteBuffer issue is resolved, I just modified the GlueHandler to wrap inbound/outbound data based on if it is server (local proxy) or client:

static func matchedPair() -> (GlueHandler, GlueHandler) {
        let server = GlueHandler(server: true)
        let client = GlueHandler(server: false)

        server.partner = client
        client.partner = server

        return (server, client)
    }

  func channelRead(context: ChannelHandlerContext, data: NIOAny) {
        if isServer == true {
            switch self.unwrapInboundIn(data) {
            case .head(let head):
                self.partner?.partnerWrite(self.wrapInboundOut(.head(head)))
            case .body(let body):
                self.partner?.partnerWrite(self.wrapInboundOut(.body(.byteBuffer(body))))
            case .end(let trailers):
                self.partner?.partnerWrite(self.wrapInboundOut(.end(trailers)))
            }
        } else {
            switch self.unwrapOutboundIn(data) {
            case .head(let head):
                self.partner?.partnerWrite(self.wrapOutboundOut(.head(head)))
            case .body(let body):
                self.partner?.partnerWrite(self.wrapOutboundOut(.body(.byteBuffer(body))))
            case .end(let trailers):
                self.partner?.partnerWrite(self.wrapOutboundOut(.end(trailers)))
            }
        }
    }

Now I see this error :frowning:
▿ 0 : Error: 268435648 error:100000c0:SSL routines:OPENSSL_internal:PEER_DID_NOT_RETURN_A_CERTIFICATE

PEER_DID_NOT_RETURN_A_CERTIFICATE strongly suggests you're connecting to something very weird. If I had to guess, I'd have said that either your server TLSConfiguration is not set up correctly (requiring client certs) or, more likely, you're not connecting to a TLS service on the remote side. Where are you connecting to, including port?

@lukasa Maybe I have not configured my network extension using NEPacketTunnelProvider properly.

Here's what my setup looks like, can you please check if I am missing something:

  1. I start a local proxy on 127.0.0.1:8028 with ServerBootstrap:
 let bootstrap = ServerBootstrap(group: localProxyGroup)
            .serverChannelOption(ChannelOptions.backlog, value: 256)
            .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1)
            .childChannelInitializer { channel in
                channel.pipeline.addHandler(ByteToMessageHandler(HTTPRequestDecoder(leftOverBytesStrategy: .forwardBytes)), name: "http-request-decoder")
                    .flatMap { channel.pipeline.addHandler(HTTPResponseEncoder(), name: "http-response-encoder") }
                    .flatMap { channel.pipeline.addHandler(ConnectHandler(logger: self.logger), name: "connect-handler") }
            }

        bootstrap.bind(host: configuration.hostname, port: Int(configuration.port)!).whenComplete { result in
            switch result {
            case .success(let channel):
                self.logger.info("NWEXTLOG:  Listening on \(String(describing: channel.localAddress))")
                self.startTunnel()
            case .failure(let error):
                self.logger.error("NWEXTLOG: Failed to bind 127.0.0.1:8028, \(error)")
                self.pendingCompletion?(NEVPNError(.connectionFailed))
                self.pendingCompletion = nil
            }
        }
  1. Start the tunnel when bootstrap binds to a given host/port (127.0.0.1/8028)
        let tunnelSettings = NEPacketTunnelNetworkSettings(tunnelRemoteAddress: "127.0.0.1")

        let proxySettings = NEProxySettings()
        proxySettings.httpServer = NEProxyServer(
            address: configuration.hostname,
            port: Int(configuration.port) ?? 8028
        )
        proxySettings.httpsServer = NEProxyServer(
            address: configuration.hostname,
            port: Int(configuration.port) ?? 8028
        )
        proxySettings.autoProxyConfigurationEnabled = false
        proxySettings.httpEnabled = true
        proxySettings.httpsEnabled = true
        proxySettings.matchDomains = [""]
        tunnelSettings.proxySettings = proxySettings

        let ipv4Settings = NEIPv4Settings(
            addresses: [tunnelSettings.tunnelRemoteAddress],
            subnetMasks: ["255.255.255.0"]
        )
        tunnelSettings.ipv4Settings = ipv4Settings

        tunnelSettings.mtu = 1500

        let dnsSettings = NEDNSSettings(servers: ["8.8.8.8", "1.1.1.1"])
        dnsSettings.matchDomains = [""]
        dnsSettings.matchDomainsNoSearch = false
        tunnelSettings.dnsSettings = dnsSettings

      // Set our tunnel settings
       setTunnelNetworkSettings(tunnelSettings.dnsSettings)

After this I can see the incoming packets from iPhone's safari
Note that I am not calling the createTCPConnection after this since without doing that I can still get the packets in my ServerBootstrap.

How are you configuring TLS for both the server and client sides? There's nothing obviously wrong with your setup that I can see, but I'm not an expert in NEPacketTunnelProvider.

On the server side I am configuring the TLS before I setup the glue:

 private func glue(_ peerChannel: Channel, context: ChannelHandlerContext) {
        self.logger.debug("NWEXTLOG: Gluing together \(ObjectIdentifier(context.channel)) and \(ObjectIdentifier(peerChannel))")

        // Ok, upgrade has completed! We now need to begin the upgrade process.
        // First, send the 200 message.
        // This content-length header is MUST NOT, but we need to workaround NIO's insistence that we set one.
        let headers = HTTPHeaders([("Content-Length", "0")])
        let head = HTTPResponseHead(version: .init(major: 1, minor: 1), status: .ok, headers: headers)
        context.write(self.wrapOutboundOut(.head(head)), promise: nil)
        context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil)

        // Now remove the HTTP encoder.
        self.removeEncoder(context: context)
        let _ = context.pipeline.removeHandler(self, promise: nil)
        
        var serverConfiguration = TLSConfiguration.makeServerConfiguration( certificateChain: self.serverCertificate.map{NIOSSLCertificateSource.certificate($0)}, privateKey: self.rootKey)
        serverConfiguration.certificateVerification = .none
        let serverContext = try! NIOSSLContext(configuration: serverConfiguration)
        
        let tlsHandler = NIOSSLServerHandler(context: serverContext)
        context.channel.pipeline.addHandler(tlsHandler, name: "ssl-handler").whenComplete {result1 in
            context.channel.pipeline.addHandler(self.protocolNegotiationHandler).whenComplete { _ in
                context.channel.pipeline.configureHTTPServerPipeline(withErrorHandling: true).whenComplete { _ in
                    let (localGlue, peerGlue) = GlueHandler.matchedPair()
                    context.channel.pipeline.addHandler(localGlue, name: "local-glue")
                        .and(peerChannel.pipeline.addHandler(peerGlue, name: "peer-glue")).whenComplete { result in
                            let _ = context.channel.pipeline.addHandler(ErrorHandler())
                    }
                }

            }
        }
    }

And for the client I do it here:

 private func connectTo(host: String, port: Int, context: ChannelHandlerContext) {
        self.logger.info("NWEXTLOG: Connecting to \(host):\(port)")

        var tlsConfig = TLSConfiguration.makeClientConfiguration()
        tlsConfig.certificateVerification = .none
        tlsConfig.trustRoots = .certificates(self.rootCertificate)
        let sslContext = try! NIOSSLContext(configuration: tlsConfig)

        let channelFuture = ClientBootstrap(group: context.eventLoop)
            .channelInitializer { channel in
                channel.pipeline.addHandler(try! NIOSSLClientHandler(context: sslContext, serverHostname: host))
                    .flatMap{ channel.pipeline.addHandler(self.protocolNegotiationHandler)}
                    .flatMap{ channel.pipeline.addHTTPClientHandlers() }
                    .flatMap{ channel.pipeline.addHandler(ErrorHandler()) }
            }
            .connect(host: host, port: port)

        channelFuture.whenSuccess { channel in
            self.connectSucceeded(channel: channel, context: context)
        }
        channelFuture.whenFailure { error in
            self.connectFailed(error: error, context: context)
        }
    }

I added the ApplicationProtocolNegotiationHandler to my pipeline for both server and client and the handshake is completed but the negotiatedProtocol value is nil, which is weird.

Okay this is interesting, looks like this is happening only when using private browsing on iOS Safari, otherwise in normal mode it seems to work fine.

@lukasa Any idea how I can decode the responses body? I am maintaining a buffer in my Glue where I keep on appending the read data:

body.getData(at: body.readerIndex, length: body.readableBytes, byteTransferStrategy: .copy) {
                        self.readData.append(stringData)
                    }

And then in channelReadComplete I try to convert my data to string but it prints a bunch if encoded data. Any idea how I can convert this to human-readable data?

What format is that data supposed to be?

The data is supposed to be in JSON, but if I read the Bytebuffer as utf8 or utf16 string it gives something like:

"��\u{16}\0�Y���z\u{03}ӭ�.n���B��О�\u{19}\u{10}�$;��i��Ŵ