Help with some pointer code

I'm trying to write a decompression function. Encountering the struggle.

Error is: Overlapping accesses to 'block', but modification requires exclusive access; consider copying to a local variable

func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
    
    return blocks.withUnsafeBytes { ptr in
        
        var block = FineBlockLinear()
        
        return withUnsafeMutableBytes(of: &block) { blockPtr in
            
            var stream = compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
                                            dst_size: MemoryLayout<FineBlockLinear>.size,
                                            src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
                                            src_size: blocks.count, state: nil)
            
            if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
            {
                return false
            }
            
            defer {
                compression_stream_destroy(&stream)
            }
            
            var done = false
            while !done {
                
                while stream.dst_size > 0 {
                    
                    if stream.src_size == 0 {
                        return false
                    }
                    
                    let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
                    
                    if status == COMPRESSION_STATUS_END {
                        done = true
                    }
                    
                    if status == COMPRESSION_STATUS_ERROR {
                        return false
                    }
                }
                
                callback(block)
                
            }
            
            return true
            
        }
        
    }
}

If I could make the withUnsafeMutableBytes smaller, I could avoid having the callback inside it. However I need to initialize stream somehow. I suppose I could create a dummy FineBlockLinear just to initialize stream and then use yet another level of withUnsafeMutableBytes.

Can I define a default constructor for compression_stream? It seems I need non-null pointers passed in. I suppose I could allocate a compression_stream uninitialized?

Maybe there's a totally better way of doing streaming decompression in Swift?

Is the validity of stream tied to blockPtr? If not, the inner block can pretty much be used to create stream.

var block = FineBlockLinear()
var stream = withUnsafeMutableByte(of: &block) { blockPtr in
    compression_stream(...)
}

guard compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) != COMPRESSION_STATUS_ERROR else {
    return false
}

defer { compression_stream_destroy(&stream) }

...

To answer the original question, the inner unsafe block is writing to block, but the closure used is also reading block hence the exclusivity violation.

PS

I'm not sure what FineBlockLinear looks like, but chances are, you don't want to use bindMemory, or at least, you'd want to bind it back to FineBlockLinear.

Oof this gets hairy. Gotta change the logic around to accommodate the second withUnsafeMutableBytes, since I can't simply return from the whole function:

func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
    
    return blocks.withUnsafeBytes { ptr in
        
        var block = FineBlockLinear()
        
        var stream = withUnsafeMutableBytes(of: &block) { blockPtr in
            return compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
            dst_size: MemoryLayout<FineBlockLinear>.size,
            src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
            src_size: blocks.count, state: nil)
        }
        
        if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
        {
            return false
        }
        
        defer {
            compression_stream_destroy(&stream)
        }
        
        var done = false
        while !done {
            
            withUnsafeMutableBytes(of: &block) { blockPtr in
                
                stream.dst_ptr = blockPtr.bindMemory(to: UInt8.self).baseAddress!
                stream.dst_size = MemoryLayout<FineBlockLinear>.size
            
                /*
                while stream.dst_size > 0 {
                    
                    if stream.src_size == 0 {
                        return false
                    }
                    
                    let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
                    
                    if status == COMPRESSION_STATUS_END {
                        done = true
                    }
                    
                    if status == COMPRESSION_STATUS_ERROR {
                        return false
                    }
                }
                */
                
            }
            
            callback(block)
            
        }
        
        return true
        
    }
}

Also realized I've gotta reset dst_ptr and dst_size at the begging of the outer while.

FWIW here's the original C function I'm translating, because evidently I'm a masochist.

bool StreamBlocks(NSData* data, void (^callback)(const FineBlockLinear* block)) {
        
    compression_stream stream;
    if(compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR) {
        return false;
    }
    
    stream.src_ptr = (uint8_t*) data.bytes;
    stream.src_size = data.length;
    bool done = false;
    
    while(!done) {
        
        FineBlockLinear block;
        
        compression_status status = COMPRESSION_STATUS_END;
        stream.dst_ptr = (uint8_t*) &block;
        stream.dst_size = sizeof(FineBlockLinear);
        
        // We need to fill the buffer.
        while(stream.dst_size > 0) {
            
            // If we've used up all the input, get some more.
            if(stream.src_size == 0) {
                return false;
            }
            
            status = compression_stream_process(&stream, COMPRESSION_STREAM_FINALIZE);
            
            if(status == COMPRESSION_STATUS_END) {
                done = YES;
            }
            
            if(status == COMPRESSION_STATUS_ERROR) {
                return false;
            }
            
        }
        
        // Load the texture block.
        callback(&block);
        
    }
    
    compression_stream_destroy(&stream);
    
    return true;
    
}

Well I'm just messing around until I get the right type to pass to the C function. What should I do instead of blockPtr.bindMemory(to: UInt8.self).baseAddress!?

Before we dive in deeper, could you elaborate a bit:

  • what's the relationship between compression_stream return value and dst_ptr. Are they the same? Or is stream a brand new value that you need to deallocate (and rightfully so at defer). Do they point to one another?
  • does compression_stream_process alter values pointed by dst_ptr, which you want to be reflected in block.

So compression_stream is a C struct. Does that answer your questions?

Yes, compression_stream_process writes data out to dst_ptr. It also advances dst_ptr and reduces dst_size.

Thanks for your help, by the way :slight_smile:

Well, throwing exceptions makes the error handling easier:

enum MyError: Error {
    case compressionError
}

func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) throws {
    
    return try blocks.withUnsafeBytes { ptr in
        
        var block = FineBlockLinear()
        
        var stream = withUnsafeMutableBytes(of: &block) { blockPtr in
            return compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
            dst_size: MemoryLayout<FineBlockLinear>.size,
            src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
            src_size: blocks.count, state: nil)
        }
        
        if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
        {
            throw MyError.compressionError
        }
        
        defer {
            compression_stream_destroy(&stream)
        }
        
        var done = false
        while !done {
            
            try withUnsafeMutableBytes(of: &block) { blockPtr in
                
                stream.dst_ptr = blockPtr.bindMemory(to: UInt8.self).baseAddress!
                stream.dst_size = MemoryLayout<FineBlockLinear>.size
            
                while stream.dst_size > 0 {
                    
                    if stream.src_size == 0 {
                        throw MyError.compressionError
                    }
                    
                    let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
                    
                    if status == COMPRESSION_STATUS_END {
                        done = true
                    }
                    
                    if status == COMPRESSION_STATUS_ERROR {
                        throw MyError.compressionError
                    }
                }
                
            }
            
            callback(block)
            
        }
        
    }
}

It might cause a problem since stream, and by extension, dst_ptr, is outliving the unsafe block. The first blockPtr are not necessarily the same as the second blockPtr. I think the cleanest way is to create block manually, and pass them in

func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
  let block = UnsafeMutableRawPointer.allocate(byteCount: MemoryLayout<FineBlockLinear>.size, alignment: MemoryLayout<FineBlockLinear>.alignment)
  defer { block.deallocate() }

  return blocks.withUnsafeBytes { ptr in
    var stream = compression_stream(dst_ptr: block.assumingMemoryBound(to: UInt8.self),
                                    dst_size: MemoryLayout<FineBlockLinear>.size,
                                    src_ptr: ptr.assumingMemoryBount(to: UInt8.self).baseAddress!,
                                    src_size: blocks.count, state: nil)

    ...
    while ... {
      callback(block.load(as: FineBlockLinear.self))
    }
}

Though I'm not sure if this is the right way to have both UnsafeMutablePointer<UInt8> in a C struct, and to load FineBlockLinear at the same time. I'm also not sure if assumingMemoryBound would work here since UInt8 and FineBlockLinear doesn't seem to be layout compatible, though I believe it's more fitting here than bindMemory. @Andrew_Trick Is this the right usage surrounding binding?

@audulus After inspecting your original code more closely, I've got a question.

It seems your while loop is made so that callback is called only once, and only when the compression succeed. Is that the case? If so, we might be able to make the unsafe block much cleaner.

No, the callback is called many times. Every time dst_size gets down to zero (so the block data is filled), the callback is called.

Ok, I see the reset at the beginning of the first while loop in the original c code.

I think it’d be easier to make block an array of UInt8, then load them as FineBlockLinear as needed

func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) throws {
  let block = UnsafeMutableRawPointer.allocate(byteCount: size, alignment: MemoryLayout<FineBlockLinear>.alignment)
  defer { block.deallocate() }

  let dst_ptr = block.bindMemory(to: UInt8.self)
  let dst_size = MemoryLayout<FineBlockLinear>.size

  let stream = compression_stream(dst_ptr: nil,
                                  dst_size: 0,
                                  src_ptr: nil,
                                  src_size: blocks.count, state: nil)

  guard compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) != COMPRESSION_STATUS_ERROR else {
    throw MyError.compressionError
  }

  defer { compression_stream_destroy(&stream) }

  try blocks.withUnsafeBytes { ptr in
    stream.src_ptr = ptr.assumingMemoryBound(to: UInt8.self).baseAddress

    var done = false
    while !done {
      stream.dst_ptr = dst_ptr
      stream.dst_size = size

      repeat {
        guard stream.src_size > 0 else {
          throw MyError.compressionError
        }

        let status = compression_stream_process(&stream, COMPRESSION_STREAM_FINALIZE)

        if status == COMPRESSION_STATUS_END {
          done = true
        } else if status == COMPRESSION_STATUS_ERROR {
          throw MyError.compressionError
        }
      } while stream.dst_size > 0

      callback(block.load(as: FineBlockLinear.self))
    }
  }
}