audulus
(Taylor Holliday)
1
I'm trying to write a decompression function. Encountering the struggle.
Error is: Overlapping accesses to 'block', but modification requires exclusive access; consider copying to a local variable
func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
return blocks.withUnsafeBytes { ptr in
var block = FineBlockLinear()
return withUnsafeMutableBytes(of: &block) { blockPtr in
var stream = compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
dst_size: MemoryLayout<FineBlockLinear>.size,
src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
src_size: blocks.count, state: nil)
if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
{
return false
}
defer {
compression_stream_destroy(&stream)
}
var done = false
while !done {
while stream.dst_size > 0 {
if stream.src_size == 0 {
return false
}
let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
if status == COMPRESSION_STATUS_END {
done = true
}
if status == COMPRESSION_STATUS_ERROR {
return false
}
}
callback(block)
}
return true
}
}
}
If I could make the withUnsafeMutableBytes smaller, I could avoid having the callback inside it. However I need to initialize stream somehow. I suppose I could create a dummy FineBlockLinear just to initialize stream and then use yet another level of withUnsafeMutableBytes.
Can I define a default constructor for compression_stream? It seems I need non-null pointers passed in. I suppose I could allocate a compression_stream uninitialized?
Maybe there's a totally better way of doing streaming decompression in Swift?
Lantua
2
Is the validity of stream tied to blockPtr? If not, the inner block can pretty much be used to create stream.
var block = FineBlockLinear()
var stream = withUnsafeMutableByte(of: &block) { blockPtr in
compression_stream(...)
}
guard compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) != COMPRESSION_STATUS_ERROR else {
return false
}
defer { compression_stream_destroy(&stream) }
...
To answer the original question, the inner unsafe block is writing to block, but the closure used is also reading block hence the exclusivity violation.
PS
I'm not sure what FineBlockLinear looks like, but chances are, you don't want to use bindMemory, or at least, you'd want to bind it back to FineBlockLinear.
audulus
(Taylor Holliday)
3
Oof this gets hairy. Gotta change the logic around to accommodate the second withUnsafeMutableBytes, since I can't simply return from the whole function:
func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
return blocks.withUnsafeBytes { ptr in
var block = FineBlockLinear()
var stream = withUnsafeMutableBytes(of: &block) { blockPtr in
return compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
dst_size: MemoryLayout<FineBlockLinear>.size,
src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
src_size: blocks.count, state: nil)
}
if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
{
return false
}
defer {
compression_stream_destroy(&stream)
}
var done = false
while !done {
withUnsafeMutableBytes(of: &block) { blockPtr in
stream.dst_ptr = blockPtr.bindMemory(to: UInt8.self).baseAddress!
stream.dst_size = MemoryLayout<FineBlockLinear>.size
/*
while stream.dst_size > 0 {
if stream.src_size == 0 {
return false
}
let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
if status == COMPRESSION_STATUS_END {
done = true
}
if status == COMPRESSION_STATUS_ERROR {
return false
}
}
*/
}
callback(block)
}
return true
}
}
Also realized I've gotta reset dst_ptr and dst_size at the begging of the outer while.
FWIW here's the original C function I'm translating, because evidently I'm a masochist.
bool StreamBlocks(NSData* data, void (^callback)(const FineBlockLinear* block)) {
compression_stream stream;
if(compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR) {
return false;
}
stream.src_ptr = (uint8_t*) data.bytes;
stream.src_size = data.length;
bool done = false;
while(!done) {
FineBlockLinear block;
compression_status status = COMPRESSION_STATUS_END;
stream.dst_ptr = (uint8_t*) █
stream.dst_size = sizeof(FineBlockLinear);
// We need to fill the buffer.
while(stream.dst_size > 0) {
// If we've used up all the input, get some more.
if(stream.src_size == 0) {
return false;
}
status = compression_stream_process(&stream, COMPRESSION_STREAM_FINALIZE);
if(status == COMPRESSION_STATUS_END) {
done = YES;
}
if(status == COMPRESSION_STATUS_ERROR) {
return false;
}
}
// Load the texture block.
callback(&block);
}
compression_stream_destroy(&stream);
return true;
}
audulus
(Taylor Holliday)
4
Well I'm just messing around until I get the right type to pass to the C function. What should I do instead of blockPtr.bindMemory(to: UInt8.self).baseAddress!?
Lantua
5
Before we dive in deeper, could you elaborate a bit:
- what's the relationship between
compression_stream return value and dst_ptr. Are they the same? Or is stream a brand new value that you need to deallocate (and rightfully so at defer). Do they point to one another?
- does
compression_stream_process alter values pointed by dst_ptr, which you want to be reflected in block.
audulus
(Taylor Holliday)
6
So compression_stream is a C struct. Does that answer your questions?
Yes, compression_stream_process writes data out to dst_ptr. It also advances dst_ptr and reduces dst_size.
Thanks for your help, by the way 
audulus
(Taylor Holliday)
7
Well, throwing exceptions makes the error handling easier:
enum MyError: Error {
case compressionError
}
func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) throws {
return try blocks.withUnsafeBytes { ptr in
var block = FineBlockLinear()
var stream = withUnsafeMutableBytes(of: &block) { blockPtr in
return compression_stream(dst_ptr: blockPtr.bindMemory(to: UInt8.self).baseAddress!,
dst_size: MemoryLayout<FineBlockLinear>.size,
src_ptr: ptr.bindMemory(to: UInt8.self).baseAddress!,
src_size: blocks.count, state: nil)
}
if compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) == COMPRESSION_STATUS_ERROR
{
throw MyError.compressionError
}
defer {
compression_stream_destroy(&stream)
}
var done = false
while !done {
try withUnsafeMutableBytes(of: &block) { blockPtr in
stream.dst_ptr = blockPtr.bindMemory(to: UInt8.self).baseAddress!
stream.dst_size = MemoryLayout<FineBlockLinear>.size
while stream.dst_size > 0 {
if stream.src_size == 0 {
throw MyError.compressionError
}
let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
if status == COMPRESSION_STATUS_END {
done = true
}
if status == COMPRESSION_STATUS_ERROR {
throw MyError.compressionError
}
}
}
callback(block)
}
}
}
Lantua
8
It might cause a problem since stream, and by extension, dst_ptr, is outliving the unsafe block. The first blockPtr are not necessarily the same as the second blockPtr. I think the cleanest way is to create block manually, and pass them in
func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) -> Bool {
let block = UnsafeMutableRawPointer.allocate(byteCount: MemoryLayout<FineBlockLinear>.size, alignment: MemoryLayout<FineBlockLinear>.alignment)
defer { block.deallocate() }
return blocks.withUnsafeBytes { ptr in
var stream = compression_stream(dst_ptr: block.assumingMemoryBound(to: UInt8.self),
dst_size: MemoryLayout<FineBlockLinear>.size,
src_ptr: ptr.assumingMemoryBount(to: UInt8.self).baseAddress!,
src_size: blocks.count, state: nil)
...
while ... {
callback(block.load(as: FineBlockLinear.self))
}
}
Though I'm not sure if this is the right way to have both UnsafeMutablePointer<UInt8> in a C struct, and to load FineBlockLinear at the same time. I'm also not sure if assumingMemoryBound would work here since UInt8 and FineBlockLinear doesn't seem to be layout compatible, though I believe it's more fitting here than bindMemory. @Andrew_Trick Is this the right usage surrounding binding?
Lantua
9
@audulus After inspecting your original code more closely, I've got a question.
It seems your while loop is made so that callback is called only once, and only when the compression succeed. Is that the case? If so, we might be able to make the unsafe block much cleaner.
audulus
(Taylor Holliday)
10
No, the callback is called many times. Every time dst_size gets down to zero (so the block data is filled), the callback is called.
Lantua
11
Ok, I see the reset at the beginning of the first while loop in the original c code.
I think it’d be easier to make block an array of UInt8, then load them as FineBlockLinear as needed
func Stream(blocks: Data, callback: (FineBlockLinear) -> Void) throws {
let block = UnsafeMutableRawPointer.allocate(byteCount: size, alignment: MemoryLayout<FineBlockLinear>.alignment)
defer { block.deallocate() }
let dst_ptr = block.bindMemory(to: UInt8.self)
let dst_size = MemoryLayout<FineBlockLinear>.size
let stream = compression_stream(dst_ptr: nil,
dst_size: 0,
src_ptr: nil,
src_size: blocks.count, state: nil)
guard compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_LZ4) != COMPRESSION_STATUS_ERROR else {
throw MyError.compressionError
}
defer { compression_stream_destroy(&stream) }
try blocks.withUnsafeBytes { ptr in
stream.src_ptr = ptr.assumingMemoryBound(to: UInt8.self).baseAddress
var done = false
while !done {
stream.dst_ptr = dst_ptr
stream.dst_size = size
repeat {
guard stream.src_size > 0 else {
throw MyError.compressionError
}
let status = compression_stream_process(&stream, COMPRESSION_STREAM_FINALIZE)
if status == COMPRESSION_STATUS_END {
done = true
} else if status == COMPRESSION_STATUS_ERROR {
throw MyError.compressionError
}
} while stream.dst_size > 0
callback(block.load(as: FineBlockLinear.self))
}
}
}
