SwiftUI - Bug or misunderstanding of views/gestures?

Hi, Im trying to display a dynamic subview as a result of the gestures occurred in the parent view. At the moment I have something like this:

protocol IconDrawer {
    associatedtype Pressed: View
    associatedtype Regular: View
    associatedtype Selected: View

    var pressed: Pressed { get }
    var regular: Regular { get }
    var selected: Selected { get }

private struct AnyIconDrawer {
    var pressed: AnyView
    var regular: AnyView
    var selected: AnyView
    init<D: IconDrawer>(drawer: D) {
        self.pressed = AnyView(drawer.pressed)
        self.regular = AnyView(drawer.regular)
        self.selected = AnyView(drawer.selected)

private struct IconWrapper: View {
    @State private var pressed: Bool = false

    let selected: Bool
    let drawer: AnyIconDrawer

    let action: () -> ()
    var body: some View {
        return GeometryReader { geometry in
            SwiftUI.Group { () -> AnyView in
                if self.pressed {
                    return self.drawer.pressed
                } else if self.selected {
                    return self.drawer.selected
                } else {
                    return self.drawer.regular
            .frame(width: 41, height: 28, alignment: .center)
                DragGesture(minimumDistance: 0)
                    .onChanged({ value in
                        print(geometry.frame(in: .local).contains(value.location))
                        self.pressed = geometry.frame(in: .local).contains(value.location)
                    .onEnded({ value in
                        self.pressed = false
                        if (geometry.frame(in: .local).contains(value.location)) {

I expected that DragGesture kept triggering onChange while mouse is down, but as soon as pressed value change, the content of GeometryReader changes and the DragGesture stops triggering events. If I change the body and keep the subviews the same, the onChange is still called until mouse up.

Am I doing something wrong here?