I ended up using a set/get:
@AppStorage("storedStartTime") var storedStartTime = Date.now.timeIntervalSinceReferenceDate
var startTime: Date {
set {storedStartTime = newValue.timeIntervalSinceReferenceDate}
get {return Date(timeIntervalSinceReferenceDate: storedStartTime)}
}