I'm not really convinced we need to coordinate with the closed source docs to make progress in this area. Most of what's been described should be possible using swift symbolgraph-extract, which is available in toolchains from 5.3 onwards and seems designed for this use case. Its output includes doc comments, type relationships, and guidance on how to format headings/subheadings/etc. of documentation pages without needing to make many individual SourceKit requests.

For example, this command can be used to generate a symbol graph for the macOS standard library, and includes all the information needed to generate documentation pages or an index:
swift symbolgraph-extract --module-name=Swift --pretty-print --output-dir /some/output/dir --target=x86_64-apple-macosx --sdk=/Applications/Xcode-beta.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.16.sdk. I don't think there's anything stopping us from integrating this into the build process and installing symbol graphs for the standard library alongside the other documentation in /usr/share/doc/swift for use by toolchain or external tools.

Partial symbolgraph output (it's very long)
{
  "metadata": {
    "formatVersion": {
      "major": 0,
      "minor": 5,
      "patch": 0
    },
    "generator": "Apple Swift version 5.3 (swiftlang-1200.0.16.9 clang-1200.0.22.5)"
  },
  "module": {
    "name": "Swift",
    "platform": {
      "architecture": "x86_64",
      "vendor": "apple",
      "operatingSystem": {
        "name": "macosx",
        "minimumVersion": {
          "major": 10,
          "minor": 15,
          "patch": 0
        }
      }
    }
  },
  "symbols": [
    {
      "kind": {
        "identifier": "swift.method",
        "displayName": "Instance Method"
      },
      "identifier": {
        "precise": "s:STsE10compactMapySayqd__Gqd__Sg7ElementQzKXEKlF::SYNTHESIZED::s:s5Int16V5WordsV",
        "interfaceLanguage": "swift"
      },
      "pathComponents": [
        "Int16",
        "Words",
        "compactMap(_:)"
      ],
      "names": {
        "title": "compactMap(_:)",
        "navigator": [
          {
            "kind": "keyword",
            "spelling": "func"
          },
          {
            "kind": "text",
            "spelling": " "
          },
          {
            "kind": "identifier",
            "spelling": "compactMap"
          },
          {
            "kind": "text",
            "spelling": "<ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": ">(("
          },
          {
            "kind": "typeIdentifier",
            "spelling": "UInt",
            "preciseIdentifier": "s:Su"
          },
          {
            "kind": "text",
            "spelling": ") "
          },
          {
            "kind": "keyword",
            "spelling": "throws"
          },
          {
            "kind": "text",
            "spelling": " -> ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": "?) "
          },
          {
            "kind": "keyword",
            "spelling": "rethrows"
          },
          {
            "kind": "text",
            "spelling": " -> [ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": "]"
          }
        ],
        "subHeading": [
          {
            "kind": "keyword",
            "spelling": "func"
          },
          {
            "kind": "text",
            "spelling": " "
          },
          {
            "kind": "identifier",
            "spelling": "compactMap"
          },
          {
            "kind": "text",
            "spelling": "<ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": ">(("
          },
          {
            "kind": "typeIdentifier",
            "spelling": "UInt",
            "preciseIdentifier": "s:Su"
          },
          {
            "kind": "text",
            "spelling": ") "
          },
          {
            "kind": "keyword",
            "spelling": "throws"
          },
          {
            "kind": "text",
            "spelling": " -> ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": "?) "
          },
          {
            "kind": "keyword",
            "spelling": "rethrows"
          },
          {
            "kind": "text",
            "spelling": " -> [ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": "]"
          }
        ]
      },
      "docComment": {
        "lines": [
          {
            "text": "Returns an array containing the non-`nil` results of calling the given"
          },
          {
            "text": "transformation with each element of this sequence."
          },
          {
            "text": ""
          },
          {
            "text": "Use this method to receive an array of non-optional values when your"
          },
          {
            "text": "transformation produces an optional value."
          },
          {
            "text": ""
          },
          {
            "text": "In this example, note the difference in the result of using `map` and"
          },
          {
            "text": "`compactMap` with a transformation that returns an optional `Int` value."
          },
          {
            "text": ""
          },
          {
            "text": "    let possibleNumbers = [\"1\", \"2\", \"three\", \"///4///\", \"5\"]"
          },
          {
            "text": ""
          },
          {
            "text": "    let mapped: [Int?] = possibleNumbers.map { str in Int(str) }"
          },
          {
            "text": "    // [1, 2, nil, nil, 5]"
          },
          {
            "text": ""
          },
          {
            "text": "    let compactMapped: [Int] = possibleNumbers.compactMap { str in Int(str) }"
          },
          {
            "text": "    // [1, 2, 5]"
          },
          {
            "text": ""
          },
          {
            "text": "- Parameter transform: A closure that accepts an element of this"
          },
          {
            "text": "  sequence as its argument and returns an optional value."
          },
          {
            "text": "- Returns: An array of the non-`nil` results of calling `transform`"
          },
          {
            "text": "  with each element of the sequence."
          },
          {
            "text": ""
          },
          {
            "text": "- Complexity: O(*m* + *n*), where *n* is the length of this sequence"
          },
          {
            "text": "  and *m* is the length of the result."
          }
        ]
      },
      "functionSignature": {
        "parameters": [
          {
            "name": "transform",
            "declarationFragments": [
              {
                "kind": "identifier",
                "spelling": "transform"
              },
              {
                "kind": "text",
                "spelling": ": ("
              },
              {
                "kind": "typeIdentifier",
                "spelling": "Self",
                "preciseIdentifier": "s:STsE4Selfxmfp"
              },
              {
                "kind": "text",
                "spelling": ".Element"
              },
              {
                "kind": "text",
                "spelling": ") "
              },
              {
                "kind": "keyword",
                "spelling": "throws"
              },
              {
                "kind": "text",
                "spelling": " -> ElementOfResult"
              },
              {
                "kind": "text",
                "spelling": "?"
              }
            ]
          }
        ],
        "returns": [
          {
            "kind": "text",
            "spelling": "[ElementOfResult"
          },
          {
            "kind": "text",
            "spelling": "]"
          }
        ]
      },
      "swiftGenerics": {
        "parameters": [
          {
            "name": "ElementOfResult",
            "index": 0,
            "depth": 1
          }
        ]
      },
      "swiftExtension": {
        "extendedModule": "Swift",
        "constraints": [
          {
            "kind": "conformance",
            "lhs": "Self",
            "rhs": "Sequence"
          }
        ]
      },
      "declarationFragments": [
        {
          "kind": "keyword",
          "spelling": "func"
        },
        {
          "kind": "text",
          "spelling": " "
        },
        {
          "kind": "identifier",
          "spelling": "compactMap"
        },
        {
          "kind": "text",
          "spelling": "<ElementOfResult"
        },
        {
          "kind": "text",
          "spelling": ">("
        },
        {
          "kind": "externalParam",
          "spelling": "_"
        },
        {
          "kind": "text",
          "spelling": " "
        },
        {
          "kind": "internalParam",
          "spelling": "transform"
        },
        {
          "kind": "text",
          "spelling": ": ("
        },
        {
          "kind": "typeIdentifier",
          "spelling": "UInt",
          "preciseIdentifier": "s:Su"
        },
        {
          "kind": "text",
          "spelling": ") "
        },
        {
          "kind": "keyword",
          "spelling": "throws"
        },
        {
          "kind": "text",
          "spelling": " -> ElementOfResult"
        },
        {
          "kind": "text",
          "spelling": "?) "
        },
        {
          "kind": "keyword",
          "spelling": "rethrows"
        },
        {
          "kind": "text",
          "spelling": " -> [ElementOfResult"
        },
        {
          "kind": "text",
          "spelling": "]"
        }
      ],
      "accessLevel": "public"
    },
8 Likes