Decodable with child of variable type

:wave:

I'm trying to implement a websocket client with ability to await replies and codable support.

The problem I'm having is I can't figure out how to persist type information to be able to use it when decoding the "raw" packet coming over the websocket. Receiving the type information and the receving of the info to decode are temporally separated.

Simplifying my current implementation I have:

enum Packet<T: Decodable> {
  case push(Push<T>)
}

extension Packet: Decodable {
  init(from decoder: Decoder) throws {
    var container = try decoder.unkeyedContainer()

    if let event = try? container.decode(String.self) {
      let payload = try container.decode(T.self)
      self = .push(.init(event: event, payload: payload))
      return
    }
    
    fatalError("failed to decode packet: \(dump(container))")
  }
}

struct Push<T: Decodable> {
  let event: String
  let payload: T
}

class Socket {
  private let decoder = JSONDecoder()
  private var subscriptions = [String: (Any) -> ()]() // how can I store the "typed" closure?

  // I get the type information on how to decode the payload from the user here
  func on<T: Decodable>(_ event: String, subscription: @escaping (T) -> ()) {
    // should I store something in decoder.userInfo?
    subscriptions[event] = subscription
  }

  // and I lose it by the time data arrives
  func receive(_ data: Data) throws {
    let packet = try decoder.decode(Packet<T>.self, from: data) // where do I get T?

    switch packet {
    case let .push(push):
      if let subscription = subscriptions[push.event] {
        subscription(push.payload)
      }
    }
  }
}

What I want to accomplish:

import Foundation

let socket = Socket()

struct Event: Decodable {
  let id: UUID
}

socket.on<Event>("event") { event in print(event.id) }

let json = """
["event",{"id":"7568CB72-2E8D-44DE-9CC8-45BA435A70FE"}]
""".data(using: .utf8)!

try socket.receive(json) // prints 7568CB72-2E8D-44DE-9CC8-45BA435A70FE

It seems like I almost got it. Decoding works, but subscription fails:

import Foundation

enum Packet {
  case push(Push)
}

extension Packet: Decodable {
  init(from decoder: Decoder) throws {
    var container = try decoder.unkeyedContainer()

    if let event = try? container.decode(String.self) {
      let payloadDecoder = decoder.userInfo[.init(rawValue: event)!] as! (UnkeyedDecodingContainer) throws -> Any
      let payload = try payloadDecoder(container)
      let push = Push(event: event, payload: payload)
      self = .push(push)
      return
    }
    
    fatalError("failed to decode packet: \(dump(container))")
  }
}

struct Push {
  let event: String
  let payload: Any
}

final class Subscription {
  private var callback: (Any) -> ()
  
  init(callback: @escaping (Any) -> ()) {
    self.callback = callback
  }
  
  func exec(_ payload: Any) {
    callback(payload)
  }
}

final class Socket {
  private let decoder = JSONDecoder()
  private var subscriptions = [String: Subscription]()

  func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
    decoder.userInfo[.init(rawValue: event)!] = { (container: UnkeyedDecodingContainer) -> Any in
      var container = container
      return try container.decode(T.self)
    }
    
    // results in error: Cannot convert value of type '(T) -> ()' to expected argument type '(Any) -> ()'
    subscriptions[event] = Subscription(callback: callback)
  }

  func receive(_ data: Data) throws {
    let packet = try decoder.decode(Packet.self, from: data)

    switch packet {
    case let .push(push):
      if let subscription = subscriptions[push.event] {
        subscription.exec(push.payload)
      }
    }
  }
}

let socket = Socket()

struct Event: Decodable {
  let id: UUID
}

socket.on("event") { (event: Event) in print(event.id) }

let json = """
["event",{"id":"7568CB72-2E8D-44DE-9CC8-45BA435A70FE"}]
""".data(using: .utf8)!

try socket.receive(json)

but I'm getting

Cannot convert value of type '(T) -> ()' to expected argument type '(Any) -> ()'

at

subscriptions[event] = Subscription(callback: callback)

Shouldn't Any type subsume T: Decodable?

This works, but it's very ugly for my taste. I wonder if there is a way to improve on it and not pass a closure in decoder's userInfo?

import Foundation

enum Packet {
  case push(Push)
}

extension Packet: Decodable {
  init(from decoder: Decoder) throws {
    var container = try decoder.unkeyedContainer()

    if let event = try? container.decode(String.self) {
      guard let payloadDecoder = decoder.userInfo[.init(rawValue: event)!] as? (UnkeyedDecodingContainer) throws -> Any else {
        fatalError("missing decoder for \(event)")
      }

      let payload = try payloadDecoder(container)
      let push = Push(event: event, payload: payload)
      self = .push(push)
      return
    }
    
    fatalError("failed to decode packet: \(dump(container))")
  }
}

struct Push {
  let event: String
  let payload: Any
}

protocol SubscriptionType {
  func exec(_ payload: Any)
}

final class Subscription<T: Decodable>: SubscriptionType {
  private var callback: (T) -> ()
  
  init(callback: @escaping (T) -> ()) {
    self.callback = callback
  }
  
  func exec(_ payload: Any) {
    callback(payload as! T)
  }
}

final class Socket {
  private let decoder = JSONDecoder()
  private var subscriptions = [String: SubscriptionType]()

  func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
    decoder.userInfo[.init(rawValue: event)!] = { (container: UnkeyedDecodingContainer) -> Any in
      var container = container
      return try container.decode(T.self)
    }
    
    subscriptions[event] = Subscription(callback: callback)
  }

  func receive(_ data: Data) throws {
    let packet = try decoder.decode(Packet.self, from: data)

    switch packet {
    case let .push(push):
      if let subscription = subscriptions[push.event] {
        subscription.exec(push.payload)
      }
    }
  }
}

let socket = Socket()

struct Event: Decodable {
  let id: UUID
}

socket.on("event") { (event: Event) in print(event.id) }

let json = """
["event",{"id":"7568CB72-2E8D-44DE-9CC8-45BA435A70FE"}]
""".data(using: .utf8)!

try socket.receive(json)

I think I'll finish with:

import Foundation

fileprivate enum Packet {
  case push(Push)
}

enum PacketError: Error {
  case missingSubscription
  case decode(UnkeyedDecodingContainer)
}

extension Packet: Decodable {
  init(from decoder: Decoder) throws {
    var container = try decoder.unkeyedContainer()

    if let event = try? container.decode(String.self) {
      guard let subscription = decoder.userInfo[.init(rawValue: event)!] as? AnySubscription else {
        throw PacketError.missingSubscription
      }

      let payload = try subscription.decode(container)
      let push = Push(event: event, payload: payload)
      self = .push(push)
      return
    }
    
    throw PacketError.decode(container)
  }
}

fileprivate struct Push {
  let event: String
  let payload: Any
}

fileprivate protocol AnySubscription {
  func exec(_ payload: Any)
  func decode(_ container: UnkeyedDecodingContainer) throws -> Any
}

fileprivate final class Subscription<T: Decodable>: AnySubscription {
  private var callback: (T) -> ()
  
  init(callback: @escaping (T) -> ()) {
    self.callback = callback
  }
  
  func exec(_ payload: Any) {
    callback(payload as! T)
  }
  
  func decode(_ container: UnkeyedDecodingContainer) throws -> Any {
    var container = container
    return try container.decode(T.self)
  }
}

final class Socket {
  private let decoder = JSONDecoder()
  private var subscriptions = [String: AnySubscription]()

  func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
    let subscription = Subscription(callback: callback)
    decoder.userInfo[.init(rawValue: event)!] = subscription
    subscriptions[event] = subscription
  }
  
  func off(_ event: String) {
    decoder.userInfo.removeValue(forKey: .init(rawValue: event)!)
    subscriptions.removeValue(forKey: event)
  }

  fileprivate func receive(_ data: Data) throws {
    let packet = try decoder.decode(Packet.self, from: data)

    switch packet {
    case let .push(push):
      if let subscription = subscriptions[push.event] {
        subscription.exec(push.payload)
      }
    }
  }
}

let socket = Socket()

struct Event: Decodable {
  let id: UUID
  let name: String
}

socket.on("event") { (event: Event) in print(event.id, event.name) }

let json = """
["event",{"id":"7568CB72-2E8D-44DE-9CC8-45BA435A70FE","name":"John"}]
""".data(using: .utf8)!

try socket.receive(json)

If you're already importing Foundation, you may want to take a look at JSONSerialization instead. Decoder protocol is more suited to when you know the data type statically (even as generics).

How would JSONSerialization help me?

It would've been nice if JSONSerialization could leave parts of JSON as Data, then it'd been perfect, but I don't think it can do that. All I get with it is [String: Any] or Any similar for the "payload" which the user then needs to convert to custom types manually.

Ok, hang on. I might misinterpret your question. So do you know that the message will be Event a priori, and want to decode it at some later time? Will the message always be Event or would it be mixed with some other types, and you need to extract Event messages? Is there a list of supported types at compile-time, or would the user be able to swap the list of types dynamically at runtime?

So do you know that the message will be Event a priori, and want to decode it at some later time?

Event is known at compile-time, and it's decoded when the socket receives "event" from backend:


// known at compile-time
struct Event: Decodable {
  let id: UUID
  let name: String
}

// "event" arrives at run-time
socket.on("event") { (event: Event) in print(event.id, event.name) }

Will the message always be Event or would it be mixed with some other types, and you need to extract Event messages?

The message can be any custom type, as specified at compile-time. I don't understand the second part of the question.

Is there a list of supported types at compile-time, or would the user be able to swap the list of types dynamically at runtime?

The types are known at compile-time.

A few examples of how I want the socket to be used:

socket.on("called") { (call: CallPush) in
  print("incoming call \(call.id) from user \(call.caller)")
}

socket.on("invited") { (invite: InvitePush) in 
  print("received invite \(invite.id) from user \(invite.user)")
}

Also, not shown above due to simplification, there is an ability to imitate request / response communication with the server:

socket.push("call", payload: ["id": calledUserId]) { (result: Result<CallResponse, SocketError>) in
  switch result {
  case let .success(call):
    print("called user \(calledUserId) with call \(call.id)")
  case let .failure(error):
    print("failued to call user \(calledUserId) with error code \(error.code)")
  }
}

All of CallPush, InvitePush, and CallResponse are known at compile-time of the application. They are not known at compile-type of the Socket module.

Your approach is mainly correct; you'd want to pass in how to decode the data through userInfo. However, the tight coupling between the decoding data and subscription can be limiting. If you may (in the future) support multiple subscriptions to the same event, this might be quite a mess.

Another thing is that it's typically better to use userInfo as a global namespace. So it might help in the long run if you limit the available keys, especially preventing ones from the userland.

I'd suggest:

fileprivate struct Push {
    let event: String
    let payload: Any
}

enum PacketError: Error {
    case unsupportedEvent
}
class PushInfo {
    var subscriptions: [String: (inout UnkeyedDecodingContainer) throws -> Any] = [:] 
}
extension Push: Decodable {
    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        
        let info = decoder.userInfo[.init(rawValue: "push")!]! as! PushInfo
        
        event = try container.decode(String.self)
        guard let subscription = info.subscriptions[event] else {
            throw PacketError.unsupportedEvent
        }
        payload = try subscription(&container)
    }
}

You can then do

final class Socket {
    private let decoder = JSONDecoder()
    private let pushInfo = PushInfo()
    private var subscriptions: [String: (Any) -> ()] = [:]
    
    init() {
        decoder.userInfo[.init(rawValue: "push")!] = pushInfo
    }
    
    func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
        let subscription: (Any) -> () = { callback($0 as! T) }
        pushInfo.subscriptions[event] = { try $0.decode(T.self) }
        subscriptions[event] = subscription
    }
    
    func off(_ event: String) {
        pushInfo.subscriptions[event] = nil
        subscriptions[event] = nil
    }
    
    fileprivate func receive(_ data: Data) throws {
        let push = try decoder.decode(Push.self, from: data)

        // Use multiple subscriptions here if needed
        subscriptions[push.event]!(push.payload)
    }
}

This would allow you to separate the decoding scheme (pushInfo) and the subscriptions (subscriptions) should it be needed later.

I'm not sure why you need to wrap Push inside Packet, but if needed, I guess you can just thinly wrap it inside.


Another interesting approach for such declaration would be to use Result Builder to declare all available types at compile time. It might be a bit tricky to pull off, but it might work out pretty well.


If you need to wrap the decoding error, I'd suggest that you do it outside of the init(from:), i.e., in the receive function. I normally find it easier to work with that way.

1 Like

Thank you very much, this is cleaner than my approach.

I do it to distinguish pushes from replies. Replies are pushes that come back in return to a client push, they look like [ref,status,payload] where ref is an auto-incremented UInt that the client gives to the push as a reference, status is ok or error, and payload is the same variable-typed value as in the pushes.

That is, on the wire, pushes are:

// ["called",{"id":"123","caller":"John"}]
Packet.push(Push(event: "called", payload: Call(id: "123", caller: "John")))

// ["invited",{"id":"456","user":"Elton"}]
Packet.push(Push(event: "invited", payload: Invite(id: "456", user: "Elton")))

And replies are:

// [10,"ok",{"id":"123"}]
Packet.reply(Reply(ref: 10, result: Result.success(Call(id: "123"))))

// [21,"error",[1001,"user is busy"]]
Packet.reply(Reply(ref: 21, result: Result.failure(SocketError(code: 1001, reason: "user is busy"))))

Hmm, since the Push and Reply are distinguished (shortly) before decoding, I suppose you could decode either Push or Reply, then put the message inside Packet afterward as needed. Otherwise, you can also use the userInfo trick to inject information, though I think it's overkill.

1 Like

Just to make sure I understand you correctly, do you mean something like

fileprivate func receive(_ data: Data) {
  if let push = try? decoder.decode(Push.self, from: data) {
    subscriptions[push.event]!(push.payload)
    return
  }

  if let reply = try? decoder.decode(Reply.self, from: data) {
    callbacks[reply.ref]!(reply.result)
    return
  }
}

?

Oh, so you don't know whether it's a push or a reply until you decode the data. If you can control the message, adding a discriminator between Push and Reply would make things a lot easier. In any case, maybe something like this?

enum Packet {
  case push(Push), reply(Reply)
}

extension Packet: Decodable {
  init(from decoder: Decoder) throws {
    switch try decoder.unkeyedContainer().count {
    case 2: self = try .push(Push(from: decoder))
    case 3: self = try .reply(Reply(from: decoder))
    default: fatalError() // or just throw
    }
  }
}

I'm not a big fan of trying to decode each case until it hits. It muddies between the errors of the packet is of unknown type vs it is a push, but seems ill-formed. That said, you are free to do so if necessary. It's not technically incorrect.

1 Like

Oh, this typecasting stuff is hard. I thought I was almost done and now I'm stuck again.

I want to wrap the reply response in a result type, so I copy-pasted the code from push subscriptions and got:

fileprivate struct Reply {
  let ref: UInt
  let result: Any // I've tried Result<Any, Error> as well, same error as below during `as!` casting in callback
}

extension Reply: Decodable {
  // ...
}

class Socket {
  private var ref: UInt = 0
  private var callbacks: [UInt: (Any) -> ()] = [:]

  // ...

  func push<T: Decodable>(_ event: String, payload: Encodable, callback: @escaping (Result<T, Error>) -> ()) {
    ref += 1
    let callback: (Any) -> () = { callback($0 as! Result<T, Error>) }
    pushInfo.replies[ref] = { try $0.decode(T.self) }
    callbacks[ref] = callback
  }
}

and it works except for $0 as! Result<T, Error> cast which fails. I've been able to reduce the problem to these few lines which produce the same error:

Could not cast value of type 'Swift.Result<Any, Swift.Error>' (0x1d6ad6d78) to 'Swift.Result<__lldb_expr_66.Event, Swift.Error>' (0x1d6acc1d8).

struct Event: Decodable {
  let id: UUID
  let name: String
}

let result: Any = Result<Any, Error>.success(Event(id: UUID(), name: "John"))
result as! Result<Event, Error>

Swift's generics (and their generic parameters) are invariant, try

struct Reply {
  let result: Result<Any, Error>
}

...
let callback: (Result<Any, Error>) -> () = { $0.map { $0 as! T } }
1 Like
let callback: (Result<Any, Error>) -> () = { callback($0.map { $0 as! T }) }

worked.

Now it finally seems I've accomplished what I wanted. I wouldn't have done it without your help @Lantua. Thank you!

1 Like

Part of your problems arise from the fact that type of the payload and type of the callback logically are coupled. You can avoid force cast if you keep them coupled in code.

This can be done in at least two ways:

  1. Call subscription callback directly inside payloadDecoder and return Void
class PushInfo {
    var subscriptions: [String: (inout UnkeyedDecodingContainer) throws -> Void] = [:] 
}

fileprivate struct Push: Decodable {
    // No properties!

    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        
        let info = decoder.userInfo[.init(rawValue: "push")!]! as! PushInfo
        
        event = try container.decode(String.self)
        guard let subscription = info.subscriptions[event] else {
            throw PacketError.unsupportedEvent
        }
        try subscription(&container)
    }
}

final class Socket {
    private let decoder = JSONDecoder()
    private let pushInfo = PushInfo()
    
    init() {
        decoder.userInfo[.init(rawValue: "push")!] = pushInfo
    }
    
    func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
        pushInfo.subscriptions[event] = { container in
            let payload = try container.decode(T.self)
            callback(payload)
        }
    }

    func receive(_ data: Data) throws {
        _ = try decoder.decode(Packet.self, from: data)
    }
  }
  1. Return from payloadDecoder a closure that captures decoded value and subscription callback and will pass decoded value to the subscription callback.
class PushInfo {
    var subscriptions: [String: (inout UnkeyedDecodingContainer) throws -> () -> Void] = [:] 
}

fileprivate struct Push: Decodable {
    var callback: () -> Void

    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        
        let info = decoder.userInfo[.init(rawValue: "push")!]! as! PushInfo
        
        event = try container.decode(String.self)
        guard let subscription = info.subscriptions[event] else {
            throw PacketError.unsupportedEvent
        }
        callback = try subscription(&container)
    }
}

final class Socket {
    private let decoder = JSONDecoder()
    private let pushInfo = PushInfo()
    
    init() {
        decoder.userInfo[.init(rawValue: "push")!] = pushInfo
    }
    
    func on<T: Decodable>(_ event: String, callback: @escaping (T) -> ()) {
        pushInfo.subscriptions[event] = { container in
            let payload = try container.decode(T.self)
            return { callback(payload) }
        }
    }

    func receive(_ data: Data) throws {
        let packet = try decoder.decode(Packet.self, from: data)

        switch packet {
        case let .push(push):
          push.callback()
        }
    }
1 Like

Thank you! My dogmatism that a decoder should only decode prevented me from even considering the first approach ...