beamformer_core.c - ogl_beamforming - Ultrasound Beamforming Implemented with OpenGL

beamformer_core.c (80694B)
      1 /* See LICENSE for license details. */
      2 /* TODO(rnp):
      3  * [ ]: backtrace dumping on SIGSEGV
      4  * [ ]: cooperative shared memory loading in decode shader
      5  * [ ]: upload previously exported data for display. maybe this is a UI thing but doing it
      6  *      programatically would be nice.
      7  * [ ]: Add interface for multi frame upload. RF upload already uses an offset into SM so
      8  *      that part works fine. We just need a way of specify a multi frame upload. (Data must
      9  *      be organized for simple offset access per frame).
     10  * [ ]: refactor: do_compute should build its own "command graph" which tracks
     11  *      dependencies better. It is very important that unnecessary barriers are
     12  *      not placed between compute stages which requires knowledge of the entire
     13  *      graph.
     14  * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable,
     15  *      use below to spin wait in library
     16  * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64)
     17  *      for power efficient low latency waiting
     18  * [ ]: BeamformWorkQueue -> BeamformerWorkQueue
     19  * [ ]: refactor: work queue needs a cleanup, we should only have a single one
     20  *      - that queue isn't really considered hot so a lock is probably fine
     21  * [ ]: bug: reinit cuda on hot-reload
     22  */
     23 
     24 #include "base_platform.h"
     25 
     26 #if defined(BEAMFORMER_DEBUG) && !defined(BEAMFORMER_EXPORT) && OS_WINDOWS
     27   #define BEAMFORMER_EXPORT __declspec(dllexport)
     28 #endif
     29 
     30 #include "beamformer_internal.h"
     31 
     32 typedef struct BeamformerComputeGraphNode BeamformerComputeGraphNode;
     33 struct BeamformerComputeGraphNode {
     34 	// NOTE(rnp): will be BeamformerShaderKind_Count for root node
     35 	BeamformerShaderKind kind;
     36 
     37 	// NOTE(rnp): when any of input or output stride is assigned it is assumed that
     38 	// the shader requires a fixed layout for input, output, or both. When two adjacent
     39 	// nodes require incompatible layouts the second pass over the graph will insert
     40 	// Reshape shaders in between.
     41 	BeamformerDataKind input_data_kind;
     42 	iv3                input_stride;
     43 
     44 	BeamformerDataKind output_data_kind;
     45 	iv3                output_stride;
     46 
     47 	i32                user_pipeline_index;
     48 
     49 	BeamformerComputeGraphNode *prev;
     50 	BeamformerComputeGraphNode *next;
     51 };
     52 
     53 typedef struct {
     54 	BeamformerComputeGraphNode *first;
     55 	BeamformerComputeGraphNode *last;
     56 	u64                         count;
     57 } BeamformerComputeGraph;
     58 
     59 read_only global u32 beamformer_compute_array_parameter_sizes[] = {
     60 	#define X(k, type, elements) sizeof(type) * elements,
     61 	BEAMFORMER_COMPUTE_ARRAY_PARAMETERS_LIST
     62 	#undef X
     63 };
     64 
     65 read_only global u32 beamformer_compute_array_parameter_offsets[] = {
     66 	#define X(k, ...) offsetof(BeamformerComputeArrayParameters, k),
     67 	BEAMFORMER_COMPUTE_ARRAY_PARAMETERS_LIST
     68 	#undef X
     69 };
     70 
     71 read_only global BeamformerFrame       beamformer_nil_frame;
     72 read_only global BeamformerComputePlan beamformer_nil_compute_plan;
     73 
     74 global BeamformerCtx   *beamformer_context;
     75 global BeamformerInput *beamformer_input;
     76 global f32 dt_for_frame;
     77 
     78 #define beamformer_frame_arena() (beamformer_context->frame_arenas[beamformer_context->frame_index % countof(beamformer_context->frame_arenas)])
     79 #define beamformer_registers() (&beamformer_context->registers->v)
     80 #define beamformer_push_registers(...) beamformer_push_registers_(&(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__})
     81 #define BeamformerRegistersScope(...) DeferLoop(beamformer_push_registers(__VA_ARGS__), beamformer_pop_registers())
     82 #define beamformer_command(name, ...) beamformer_push_command(name, &(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__})
     83 
     84 function BeamformerRegisters *
     85 beamformer_pop_registers(void)
     86 {
     87 	BeamformerRegisters *result = &beamformer_context->registers->v;
     88 	SLLStackPop(beamformer_context->registers, next);
     89 	if (beamformer_context->registers == 0)
     90 		beamformer_context->registers = &beamformer_context->base_registers;
     91 	return result;
     92 }
     93 
     94 function BeamformerRegisters *
     95 beamformer_push_registers_(BeamformerRegisters *registers)
     96 {
     97 	BeamformerRegistersNode *node   = push_struct(beamformer_frame_arena(), BeamformerRegistersNode);
     98 	BeamformerRegisters     *result = &node->v;
     99 	memory_copy(result, registers, sizeof(node->v));
    100 	SLLStackPush(beamformer_context->registers, node, next);
    101 	return result;
    102 }
    103 
    104 function void
    105 beamformer_command_list_push_new(Arena *arena, BeamformerCommandList *commands, str8 name, BeamformerRegisters *registers)
    106 {
    107 	BeamformerCommandNode *node = push_struct(arena, BeamformerCommandNode);
    108 	node->command.registers = push_struct_no_zero(arena, BeamformerRegisters);
    109 	node->command.name      = push_str8(arena, name);
    110 	memory_copy(node->command.registers, registers, sizeof(*registers));
    111 	DLLInsertLast(0, commands->first, commands->last, node, next, prev);
    112 	commands->count += 1;
    113 }
    114 
    115 function void
    116 beamformer_push_command(str8 name, BeamformerRegisters *registers)
    117 {
    118 	beamformer_command_list_push_new(beamformer_frame_arena(), beamformer_context->command_queues + 0,
    119 	                                 name, registers);
    120 }
    121 
    122 function BeamformerCommandKind
    123 beamformer_command_kind_from_string(str8 s)
    124 {
    125 	BeamformerCommandKind result = BeamformerCommandKind_Nil;
    126 	for EachElement(beamformer_command_infos, it) {
    127 		if (str8_equal(beamformer_command_infos[it].string, s)) {
    128 			result = (BeamformerCommandKind)it;
    129 			break;
    130 		}
    131 	}
    132 	return result;
    133 }
    134 
    135 function BeamformerPanelKind
    136 beamformer_panel_kind_from_string(str8 s)
    137 {
    138 	BeamformerPanelKind result = BeamformerPanelKind_Nil;
    139 	for EachElement(beamformer_panel_infos, it) {
    140 		if (str8_equal(beamformer_panel_infos[it].string, s)) {
    141 			result = (BeamformerPanelKind)it;
    142 			break;
    143 		}
    144 	}
    145 	return result;
    146 }
    147 
    148 function BeamformerFrame *
    149 beamformer_frame_from_index(u64 index)
    150 {
    151 	BeamformerFrame *result = &beamformer_nil_frame;
    152 	if (index < countof(beamformer_context->compute_context.backlog.frames)) {
    153 		BeamformerFrame *frame = beamformer_context->compute_context.backlog.frames + index;
    154 		if (frame->timeline_valid_value != 0)
    155 			result = frame;
    156 	}
    157 	return result;
    158 }
    159 
    160 function b32
    161 beamformer_frame_valid(u64 index)
    162 {
    163 	b32 result = beamformer_frame_from_index(index) != &beamformer_nil_frame;
    164 	return result;
    165 }
    166 
    167 function void
    168 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block)
    169 {
    170 	assert(block < countof(cc->compute_plans));
    171 	BeamformerComputePlan *cp = cc->compute_plans[block];
    172 	if (cp) {
    173 		vk_buffer_release(&cp->array_parameters);
    174 		for (u32 i = 0; i < countof(cp->filters); i++)
    175 			vk_buffer_release(&cp->filters[i].buffer);
    176 		cc->compute_plans[block] = 0;
    177 		SLLPushFreelist(cp, cc->compute_plan_freelist);
    178 	}
    179 }
    180 
    181 function BeamformerComputePlan *
    182 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena)
    183 {
    184 	assert(block < countof(cc->compute_plans));
    185 	BeamformerComputePlan *result = cc->compute_plans[block];
    186 	if (!result) {
    187 		result = SLLPopFreelist(cc->compute_plan_freelist);
    188 		if (!result) result = push_struct_no_zero(arena, BeamformerComputePlan);
    189 		zero_struct(result);
    190 		cc->compute_plans[block] = result;
    191 
    192 		result->ui_voxel_transform = m4_identity();
    193 
    194 		Stream label = arena_stream(arena);
    195 		stream_append_str8(&label, str8("ComputeParameterArray["));
    196 		stream_append_u64(&label, block);
    197 		stream_append_str8(&label, str8("]"));
    198 		stream_append_byte(&label, 0);
    199 
    200 		GPUBufferAllocateInfo allocate_info = {
    201 			.size  = sizeof(BeamformerComputeArrayParameters),
    202 			.flags = VulkanUsageFlag_HostReadWrite,
    203 			.label = stream_to_str8(&label),
    204 		};
    205 		vk_buffer_allocate(&result->array_parameters, &allocate_info);
    206 		assert((result->array_parameters.gpu_pointer & 63) == 0);
    207 	}
    208 	return result;
    209 }
    210 
    211 function void
    212 beamformer_filter_update(BeamformerFilter *f, BeamformerFilterParameters fp, u32 block, u32 slot, Arena *arena)
    213 {
    214 	Temp scratch = temp_begin(arena);
    215 	Stream sb = arena_stream(arena);
    216 	stream_append_str8s(&sb,
    217 	                    beamformer_filter_kind_strings[fp.kind % countof(beamformer_filter_kind_strings)],
    218 	                    str8("Filter["));
    219 	stream_append_u64(&sb, block);
    220 	stream_append_str8(&sb, str8("]["));
    221 	stream_append_u64(&sb, slot);
    222 	stream_append_byte(&sb, ']');
    223 	str8 label = arena_stream_commit(arena, &sb);
    224 
    225 	void *filter = 0;
    226 	switch (fp.kind) {
    227 	case BeamformerFilterKind_Kaiser:{
    228 		/* TODO(rnp): this should also support complex */
    229 		/* TODO(rnp): implement this as an IFIR filter instead to reduce computation */
    230 		filter = kaiser_low_pass_filter(arena, fp.kaiser.cutoff_frequency, fp.sampling_frequency,
    231 		                                fp.kaiser.beta, (i32)fp.kaiser.length);
    232 		f->length     = (i32)fp.kaiser.length;
    233 		f->time_delay = (f32)f->length / 2.0f / fp.sampling_frequency;
    234 	}break;
    235 	case BeamformerFilterKind_MatchedChirp:{
    236 		typeof(fp.matched_chirp) *mc = &fp.matched_chirp;
    237 		f32 fs    = fp.sampling_frequency;
    238 		f->length = (i32)(mc->duration * fs);
    239 		if (fp.complex) {
    240 			filter = baseband_chirp(arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1, 0.5f);
    241 			f->time_delay = complex_filter_first_moment(filter, f->length, fs);
    242 		} else {
    243 			filter = rf_chirp(arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1);
    244 			f->time_delay = real_filter_first_moment(filter, f->length, fs);
    245 		}
    246 	}break;
    247 	InvalidDefaultCase;
    248 	}
    249 
    250 	f->parameters = fp;
    251 
    252 	u32 byte_size = f->length * (i32)sizeof(f32) * (fp.complex? 2 : 1);
    253 	if (f->buffer.size < byte_size) {
    254 		GPUBufferAllocateInfo allocate_info = {
    255 			.size  = byte_size,
    256 			.flags = VulkanUsageFlag_HostReadWrite,
    257 			.label = label,
    258 		};
    259 		vk_buffer_allocate(&f->buffer, &allocate_info);
    260 	}
    261 	vk_buffer_range_upload(&f->buffer, filter, 0, byte_size, 0);
    262 
    263 	temp_end(scratch);
    264 }
    265 
    266 function iv3
    267 das_valid_points(iv3 points)
    268 {
    269 	iv3 result;
    270 	result.x = Max(points.x, 1);
    271 	result.y = Max(points.y, 1);
    272 	result.z = Max(points.z, 1);
    273 	return result;
    274 }
    275 
    276 function void
    277 beamformer_update_hadamard(BeamformerComputePlan *cp, i32 order, b32 row_major, b32 das_matrix, Arena *arena)
    278 {
    279 	f16 *hadamard = make_hadamard_transpose(arena, order, row_major);
    280 	if (hadamard) {
    281 		u64 offset = das_matrix ? offsetof(BeamformerComputeArrayParameters, DasHadamard)
    282 		                        : offsetof(BeamformerComputeArrayParameters, DecodeHadamard);
    283 		u64 size   = das_matrix ? sizeof(*((BeamformerComputeArrayParameters *)0)->DasHadamard)
    284 		                        : sizeof(*((BeamformerComputeArrayParameters *)0)->DecodeHadamard);
    285 		size *= order * order;
    286 		vk_buffer_range_upload(&cp->array_parameters, hadamard, offset, size, 0);
    287 		if(!das_matrix) cp->hadamard_order = order;
    288 	}
    289 }
    290 
    291 function u64
    292 beamformer_frame_byte_size(iv3 points, BeamformerDataKind kind)
    293 {
    294 	u64 result = points.x * points.y * points.z * beamformer_data_kind_byte_size[kind];
    295 	result = round_up_to(result, 64);
    296 	return result;
    297 }
    298 
    299 function BeamformerFrame *
    300 beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 complex, u64 reserved_size)
    301 {
    302 	BeamformerFrameBacklog *bl = &cc->backlog;
    303 
    304 	BeamformerDataKind kind = complex ? BeamformerDataKind_Float32Complex : BeamformerDataKind_Float32;
    305 	u64 frame_size = beamformer_frame_byte_size(output_points, kind);
    306 
    307 	// TODO(rnp): handle this somewhat gracefully (even it produces garbled output)
    308 	assert(frame_size + reserved_size <= (u64)bl->buffer->size);
    309 
    310 	if (bl->next_offset > (u64)bl->buffer->size - frame_size - reserved_size)
    311 		bl->next_offset = 0;
    312 
    313 	u64 id = bl->counter++;
    314 
    315 	BeamformerFrame *result = bl->frames + (id % countof(bl->frames));
    316 	atomic_store_u64(&result->timeline_valid_value, -1ULL);
    317 	result->id            = id & U32_MAX;
    318 	result->buffer_offset = bl->next_offset;
    319 	result->points        = output_points;
    320 	result->data_kind     = kind;
    321 
    322 	bl->next_offset += frame_size;
    323 
    324 	return result;
    325 }
    326 
    327 function void
    328 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info)
    329 {
    330 	u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer);
    331 	t->buffer[index] = info;
    332 }
    333 
    334 function uv3
    335 layout_for_output(iv3 points)
    336 {
    337 	uv3 result = {{1, 1, 1}};
    338 
    339 	b32 has_x = points.x > 1;
    340 	b32 has_y = points.y > 1;
    341 	b32 has_z = points.z > 1;
    342 
    343 	u32 subgroup_size  = vk_gpu_info()->subgroup_size;
    344 	u32 grid_3d_z_size = Max(1, subgroup_size / (4 * 4));
    345 	u32 grid_2d_y_size = Max(1, subgroup_size / 8);
    346 
    347 	switch (iv3_dimension(points)) {
    348 	case 1:{
    349 		if (has_x) result.x = subgroup_size;
    350 		if (has_y) result.y = subgroup_size;
    351 		if (has_z) result.z = subgroup_size;
    352 	}break;
    353 
    354 	case 2:{
    355 		if (has_x && has_y) {result.x = 8; result.y = grid_2d_y_size;}
    356 		if (has_x && has_z) {result.x = 8; result.z = grid_2d_y_size;}
    357 		if (has_y && has_z) {result.y = 8; result.z = grid_2d_y_size;}
    358 	}break;
    359 
    360 	case 3:{result = (uv3){{4, 4, grid_3d_z_size}};}break;
    361 
    362 	InvalidDefaultCase;
    363 	}
    364 
    365 	return result;
    366 }
    367 
    368 function uv3
    369 dispatch_for_output(uv3 layout, iv3 points)
    370 {
    371 	uv3 result;
    372 	result.x = (u32)ceil_f32((f32)points.x / layout.x);
    373 	result.y = (u32)ceil_f32((f32)points.y / layout.y);
    374 	result.z = (u32)ceil_f32((f32)points.z / layout.z);
    375 	return result;
    376 }
    377 
    378 function b32
    379 compute_plan_push_shader(BeamformerComputePlan *p, BeamformerComputeGraphNode *node, BeamformerShaderParameters *sp)
    380 {
    381 	b32 result = 0;
    382 	if (p->pipeline.shader_count < countof(p->pipeline.shaders)) {
    383 		u32 index = p->pipeline.shader_count++;
    384 		p->pipeline.shaders[index]    = node->kind;
    385 		zero_struct(p->shader_descriptors + index);
    386 		p->pipeline.parameters[index] = sp ? *sp : (BeamformerShaderParameters){0};
    387 
    388 		p->shader_descriptors[index].input_data_kind  = node->input_data_kind;
    389 		p->shader_descriptors[index].output_data_kind = node->output_data_kind;
    390 
    391 		result = 1;
    392 	}
    393 	return result;
    394 }
    395 
    396 function BeamformerComputeGraphNode *
    397 push_compute_graph_node(BeamformerComputeGraph *graph, BeamformerShaderKind kind, Arena *arena)
    398 {
    399 	BeamformerComputeGraphNode *result = push_struct(arena, BeamformerComputeGraphNode);
    400 	if (graph) {
    401 		DLLInsertLast(0, graph->first, graph->last, result, next, prev);
    402 		graph->count++;
    403 	}
    404 	result->kind = kind;
    405 	result->user_pipeline_index = -1;
    406 	// NOTE(rnp): initially don't care data kind
    407 	result->input_data_kind  = BeamformerDataKind_Count;
    408 	result->output_data_kind = BeamformerDataKind_Count;
    409 	return result;
    410 }
    411 
    412 function void
    413 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, Arena *scratch)
    414 {
    415 	b32 run_hilbert = 0;
    416 	b32 demodulate  = 0;
    417 
    418 	for (u32 i = 0; i < pb->pipeline.shader_count; i++) {
    419 		switch (pb->pipeline.shaders[i]) {
    420 		case BeamformerShaderKind_Hilbert:{run_hilbert = 1;}break;
    421 		case BeamformerShaderKind_Demodulate:{demodulate = 1;}break;
    422 		default:{}break;
    423 		}
    424 	}
    425 
    426 	if (demodulate) run_hilbert = 0;
    427 
    428 	f32 sampling_frequency = pb->parameters.sampling_frequency;
    429 	u32 input_sample_count = pb->parameters.sample_count;
    430 	u32 acquisition_count  = pb->parameters.acquisition_count;
    431 	u32 decimation_rate    = Max(pb->parameters.decimation_rate, 1);
    432 
    433 	cp->raw_channel_byte_stride = pb->parameters.sample_count * pb->parameters.acquisition_count
    434 	                              * beamformer_data_kind_byte_size[pb->pipeline.data_kind];
    435 
    436 	BeamformerDataKind input_data_kind = pb->pipeline.data_kind;
    437 	if (demodulate) {
    438 		switch (input_data_kind) {
    439 		case BeamformerDataKind_Int16:{  input_data_kind = BeamformerDataKind_Int16Complex;  }break;
    440 		case BeamformerDataKind_Float16:{input_data_kind = BeamformerDataKind_Float16Complex;}break;
    441 		case BeamformerDataKind_Float32:{input_data_kind = BeamformerDataKind_Float32Complex;}break;
    442 		default:{}break;
    443 		}
    444 		input_sample_count /= (2 * decimation_rate);
    445 		sampling_frequency /= (2 * decimation_rate);
    446 	}
    447 
    448 	cp->iq_pipeline = beamformer_data_kind_complex[input_data_kind] || run_hilbert;
    449 
    450 	BeamformerDataKind das_data_kind = cp->iq_pipeline ? BeamformerDataKind_Float32Complex
    451 	                                                   : BeamformerDataKind_Float32;
    452 
    453 	cp->channel_count = pb->parameters.channel_count;
    454 	u32 chunk_channel_count = Min(cp->channel_count, BeamformerChunkChannelCount);
    455 
    456 	cp->rf_size = input_sample_count * pb->parameters.acquisition_count * chunk_channel_count
    457 	              * beamformer_data_kind_byte_size[das_data_kind];
    458 
    459 	read_only local_persist BeamformerDataKind data_kind_to_element_kind[] = {
    460 		[BeamformerDataKind_Int16]          = BeamformerDataKind_Float16,
    461 		[BeamformerDataKind_Float16]        = BeamformerDataKind_Float16,
    462 		[BeamformerDataKind_Float32]        = BeamformerDataKind_Float32,
    463 		[BeamformerDataKind_Int16Complex]   = BeamformerDataKind_Float16,
    464 		[BeamformerDataKind_Float16Complex] = BeamformerDataKind_Float16,
    465 		[BeamformerDataKind_Float32Complex] = BeamformerDataKind_Float32,
    466 	};
    467 
    468 	//////////////////////////////////////
    469 	// NOTE(rnp): First Pass: build initial graph and insert hard layout constraints
    470 	BeamformerComputeGraph graph = {0};
    471 	BeamformerComputeGraphNode *root_node = push_compute_graph_node(&graph, BeamformerShaderKind_Count, scratch);
    472 	root_node->input_data_kind  = input_data_kind;
    473 	root_node->input_stride.x   = 1;                                               // Sample Stride
    474 	root_node->input_stride.y   = pb->parameters.sample_count * acquisition_count; // Channel Stride
    475 	root_node->input_stride.z   = pb->parameters.sample_count;                     // Receive Event Stride
    476 	root_node->output_data_kind = input_data_kind;
    477 	root_node->output_stride.x  = 1;                                               // Sample Stride
    478 	root_node->output_stride.y  = pb->parameters.sample_count * acquisition_count; // Channel Stride
    479 	root_node->output_stride.z  = pb->parameters.sample_count;                     // Receive Event Stride
    480 
    481 	for EachIndex(pb->pipeline.shader_count, it) {
    482 		// NOTE(rnp): skip unnecessary shaders
    483 		switch (pb->pipeline.shaders[it]) {
    484 		case BeamformerShaderKind_Hilbert:{if (!run_hilbert) continue;}break;
    485 
    486 		case BeamformerShaderKind_Decode:{
    487 			if (pb->parameters.decode_mode == BeamformerDecodeMode_None)
    488 				continue;
    489 		}break;
    490 
    491 		case BeamformerShaderKind_Sum:
    492 		case BeamformerShaderKind_MinMax:
    493 		{
    494 			// NOTE(rnp): currently unsupported
    495 			continue;
    496 		}break;
    497 
    498 		default:{}break;
    499 		}
    500 
    501 		BeamformerComputeGraphNode *node = push_compute_graph_node(&graph, pb->pipeline.shaders[it], scratch);
    502 		node->user_pipeline_index = (i32)it;
    503 		switch (pb->pipeline.shaders[it]) {
    504 		case BeamformerShaderKind_Decode:{
    505 			b32 low_precision   = beamformer_data_kind_element_size[input_data_kind] < 4;
    506 			b32 use_coop_matrix = vk_gpu_info()->cooperative_matrix &&
    507 			                      low_precision &&
    508 			                      (acquisition_count   % 16 == 0) &&
    509 			                      (chunk_channel_count % 16 == 0);
    510 
    511 			// NOTE(rnp): fixed input layout required for reasonable performance
    512 			if (low_precision && beamformer_data_kind_complex[input_data_kind])
    513 				node->input_data_kind = BeamformerDataKind_Float16Complex;
    514 			node->input_stride.x = chunk_channel_count * acquisition_count;
    515 			node->input_stride.y = acquisition_count;
    516 			node->input_stride.z = 1;
    517 
    518 			if (use_coop_matrix) {
    519 				node->input_data_kind  = BeamformerDataKind_Float16;
    520 				node->output_data_kind = data_kind_to_element_kind[das_data_kind];
    521 				node->output_stride    = node->input_stride;
    522 			}
    523 		}break;
    524 
    525 		case BeamformerShaderKind_DAS:{
    526 			node->input_data_kind  = das_data_kind;
    527 			node->input_stride.x   = 1;                                      // Sample Stride
    528 			node->input_stride.y   = input_sample_count * acquisition_count; // Channel Stride
    529 			node->input_stride.z   = input_sample_count;                     // Receive Event Stride
    530 			node->output_stride.x  = 1;
    531 			node->output_stride.y  = cp->output_points.x;
    532 			node->output_stride.z  = cp->output_points.x * cp->output_points.y;
    533 			node->output_data_kind = cp->iq_pipeline ? BeamformerDataKind_Float32Complex
    534 			                                         : BeamformerDataKind_Float32;
    535 
    536 			// NOTE(rnp): insert implicit CoherencyWeighting node
    537 			if (pb->parameters.coherency_weighting)
    538 				node = push_compute_graph_node(&graph, BeamformerShaderKind_CoherencyWeighting, scratch);
    539 		}break;
    540 
    541 		default:{}break;
    542 		}
    543 	}
    544 
    545 	//////////////////////////////////////
    546 	// NOTE(rnp): Second Pass: resolve layout constraints
    547 	for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) {
    548 		b32 needs_reshape = 0;
    549 
    550 		// NOTE(rnp): data strides
    551 		{
    552 			b32 input_dont_care       = bv3_any(iv3_equal(node->input_stride, (iv3){0}));
    553 			b32 prev_output_dont_care = bv3_any(iv3_equal(node->prev->output_stride, (iv3){0}));
    554 
    555 			if (prev_output_dont_care && !input_dont_care)
    556 				node->prev->output_stride = node->input_stride;
    557 
    558 			if (!prev_output_dont_care && input_dont_care)
    559 				node->input_stride = node->prev->output_stride;
    560 
    561 			if (prev_output_dont_care && input_dont_care)
    562 				node->input_stride = node->prev->output_stride = node->prev->input_stride;
    563 
    564 			needs_reshape |= !bv3_all(iv3_equal(node->input_stride, node->prev->output_stride));
    565 		}
    566 
    567 		// NOTE(rnp): data kinds
    568 		{
    569 			b32 input_dont_care       = node->input_data_kind        == BeamformerDataKind_Count;
    570 			b32 prev_output_dont_care = node->prev->output_data_kind == BeamformerDataKind_Count;
    571 
    572 			if (prev_output_dont_care && !input_dont_care)
    573 				node->prev->output_data_kind = node->input_data_kind;
    574 
    575 			if (!prev_output_dont_care && input_dont_care)
    576 				node->input_data_kind = node->prev->output_data_kind;
    577 
    578 			if (prev_output_dont_care && input_dont_care)
    579 				node->input_data_kind = node->prev->output_data_kind = node->prev->input_data_kind;
    580 
    581 			needs_reshape |= node->input_data_kind != node->prev->output_data_kind;
    582 		}
    583 
    584 		// NOTE(rnp): insert reshape if needed
    585 		if (needs_reshape) {
    586 			BeamformerComputeGraphNode *new = push_compute_graph_node(0, BeamformerShaderKind_Reshape, scratch);
    587 			BeamformerComputeGraphNode *last  = node->prev;
    588 			DLLInsertLast(0, node, last, new, next, prev);
    589 			graph.count++;
    590 			new->input_data_kind  = new->prev->output_data_kind;
    591 			new->input_stride     = new->prev->output_stride;
    592 			new->output_data_kind = new->next->input_data_kind;
    593 			new->output_stride    = new->next->input_stride;
    594 		}
    595 	}
    596 
    597 	// NOTE(rnp): ensure last node descriptor gets proper values for output data kind
    598 	if (graph.last->output_data_kind == BeamformerDataKind_Count)
    599 		graph.last->output_data_kind = graph.last->input_data_kind;
    600 
    601 	f32 time_offset   = pb->parameters.time_offset;
    602 	u32 subgroup_size = vk_gpu_info()->subgroup_size;
    603 
    604 	cp->first_image_shader_index = 0;
    605 	cp->pipeline.shader_count = 0;
    606 
    607 	for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) {
    608 		assert(node->prev->output_data_kind == node->input_data_kind);
    609 		assert(bv3_all(iv3_equal(node->prev->output_stride, node->input_stride)));
    610 
    611 		BeamformerShaderParameters *sp = 0;
    612 		if (node->user_pipeline_index >= 0)
    613 			sp = pb->pipeline.parameters + node->user_pipeline_index;
    614 
    615 		if (compute_plan_push_shader(cp, node, sp)) {
    616 			BeamformerShaderDescriptor *sd = cp->shader_descriptors + cp->pipeline.shader_count - 1;
    617 
    618 			switch (node->kind) {
    619 			case BeamformerShaderKind_Decode:{
    620 				BeamformerDecodeBakeParameters *db = &sd->bake.Decode;
    621 
    622 				u32 decode_sample_count = input_sample_count;
    623 				db->DecodeMode    = pb->parameters.decode_mode;
    624 				db->TransmitCount = pb->parameters.acquisition_count;
    625 				db->ChunkChannelCount = chunk_channel_count;
    626 
    627 				// NOTE(rnp): ignored when using coop matrices
    628 				db->OutputSampleStride   = node->output_stride.x;
    629 				db->OutputChannelStride  = node->output_stride.y;
    630 				db->OutputTransmitStride = node->output_stride.z;
    631 
    632 				db->ToProcess = 1;
    633 
    634 				b32 use_coop_matrix = vk_gpu_info()->cooperative_matrix &&
    635 				                      node->input_data_kind == BeamformerDataKind_Float16 &&
    636 				                      (db->TransmitCount % 16 == 0) &&
    637 				                      (chunk_channel_count % 16 == 0);
    638 				if (use_coop_matrix) {
    639 					// TODO(rnp): shared memory for larger sizes
    640 					sd->layout = (uv3){{subgroup_size, 1, 1}};
    641 
    642 					if (demodulate)
    643 						decode_sample_count *= 2;
    644 
    645 					sd->compile_flags |= BeamformerDecodeCompileFlags_CooperativeMatrix;
    646 					db->CooperativeMatrixM = 16;
    647 					db->CooperativeMatrixN = 16;
    648 					db->CooperativeMatrixK = 16;
    649 
    650 					sd->dispatch.x = db->TransmitCount   / db->CooperativeMatrixN;
    651 					sd->dispatch.y = chunk_channel_count / db->CooperativeMatrixM;
    652 					sd->dispatch.z = decode_sample_count;
    653 				} else if (db->TransmitCount > 40) {
    654 					db->UseSharedMemory = 1;
    655 
    656 					if (db->TransmitCount == 48)
    657 						db->ToProcess = db->TransmitCount / 16;
    658 
    659 					b32 use_16x  = db->TransmitCount == 48 || db->TransmitCount == 80 ||
    660 					               db->TransmitCount == 96 || db->TransmitCount == 160;
    661 					sd->layout.x = use_16x ? 16 : 32;
    662 					sd->layout.y = 4;
    663 					sd->layout.z = 1;
    664 
    665 					sd->dispatch.x = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.x / (f32)db->ToProcess);
    666 					sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count              / (f32)sd->layout.y);
    667 					sd->dispatch.z = (u32)ceil_f32((f32)decode_sample_count              / (f32)sd->layout.z);
    668 				} else {
    669 					/* NOTE(rnp): register caching. using more threads will cause the compiler to do
    670 					 * contortions to avoid spilling registers. using less gives higher performance */
    671 					sd->layout = (uv3){{subgroup_size / 2, 1, 1}};
    672 
    673 					sd->dispatch.x = (u32)ceil_f32((f32)decode_sample_count / (f32)sd->layout.x);
    674 					sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count / (f32)sd->layout.y);
    675 					sd->dispatch.z = 1;
    676 				}
    677 			}break;
    678 
    679 			case BeamformerShaderKind_Demodulate:
    680 			case BeamformerShaderKind_Filter:
    681 			{
    682 				b32 demod = node->kind == BeamformerShaderKind_Demodulate;
    683 				BeamformerFilter *f = cp->filters + sp->filter_slot;
    684 
    685 				sd->compile_flags |= BeamformerFilterCompileFlags_Demodulate * demod;
    686 				sd->compile_flags |= BeamformerFilterCompileFlags_ComplexFilter * f->parameters.complex;
    687 
    688 				time_offset += f->time_delay;
    689 
    690 				BeamformerFilterBakeParameters *fb = &sd->bake.Filter;
    691 				fb->FilterLength  = (u32)f->length;
    692 
    693 				fb->SampleCount    = input_sample_count;
    694 				fb->DecimationRate = demod ? decimation_rate : 1;
    695 
    696 				b32 deinterleave =  beamformer_data_kind_complex[node->input_data_kind] &&
    697 				                   !beamformer_data_kind_complex[node->output_data_kind];
    698 				if (deinterleave)
    699 					fb->BatchSampleCount = chunk_channel_count * input_sample_count * pb->parameters.acquisition_count;
    700 
    701 				fb->OutputSampleStride   = node->output_stride.x;
    702 				fb->OutputChannelStride  = node->output_stride.y;
    703 				fb->OutputTransmitStride = node->output_stride.z;
    704 
    705 				fb->InputSampleStride    = node->input_stride.x;
    706 				fb->InputChannelStride   = node->input_stride.y;
    707 				fb->InputTransmitStride  = node->input_stride.z;
    708 
    709 				/* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating
    710 				 * between sampling the I portion and the Q portion of an IQ signal. Therefore there
    711 				 * is an implicit decimation factor of 2 which must always be included. All code here
    712 				 * assumes that the signal was sampled in such a way that supports this operation.
    713 				 * To recover IQ[n] from the sampled data (RF[n]) we do the following:
    714 				 *   I[n]  = RF[n]
    715 				 *   Q[n]  = RF[n + 1]
    716 				 *   IQ[n] = I[n] - j*Q[n]
    717 				 */
    718 				if (demod) {
    719 					fb->DemodulationFrequency = pb->parameters.demodulation_frequency;
    720 					fb->SamplingFrequency     = pb->parameters.sampling_frequency / 2;
    721 				}
    722 
    723 				sd->layout     = (uv3){{subgroup_size, 1, 1}};
    724 				sd->dispatch.x = (u32)ceil_f32((f32)input_sample_count               / (f32)sd->layout.x);
    725 				sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count              / (f32)sd->layout.y);
    726 				sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z);
    727 			}break;
    728 
    729 			case BeamformerShaderKind_DAS:{
    730 				cp->first_image_shader_index = cp->pipeline.shader_count;
    731 
    732 				BeamformerDASBakeParameters *db = &sd->bake.DAS;
    733 				db->SamplingFrequency     = sampling_frequency;
    734 				db->DemodulationFrequency = pb->parameters.demodulation_frequency;
    735 				db->SpeedOfSound          = pb->parameters.speed_of_sound;
    736 				db->TimeOffset            = time_offset;
    737 				db->FNumber               = pb->parameters.f_number;
    738 				db->AcquisitionKind       = pb->parameters.acquisition_kind;
    739 				db->SampleCount           = input_sample_count;
    740 				db->ChannelCount          = pb->parameters.channel_count;
    741 				db->AcquisitionCount      = pb->parameters.acquisition_count;
    742 				db->ChunkChannelCount     = chunk_channel_count;
    743 				db->InterpolationMode     = pb->parameters.interpolation_mode;
    744 				db->TransmitAngle         = pb->parameters.focal_vector.E[0];
    745 				db->FocusDepth            = pb->parameters.focal_vector.E[1];
    746 				db->ReadiGroupCount       = pb->parameters.readi_group_count;
    747 				db->TransmitReceiveOrientation = pb->parameters.transmit_receive_orientation;
    748 
    749 				cp->readi_group = pb->parameters.readi_group;
    750 
    751 				// NOTE(rnp): old gcc will miscompile an assignment
    752 				memory_copy(cp->xdc_transform.E, pb->parameters.xdc_transform.E, sizeof(cp->xdc_transform));
    753 
    754 				cp->voxel_transform   = m4_mul(cp->ui_voxel_transform, pb->parameters.das_voxel_transform);
    755 				cp->xdc_element_pitch = pb->parameters.xdc_element_pitch;
    756 
    757 				memory_copy(cp->das_voxel_transform.E, cp->voxel_transform.E, sizeof(cp->voxel_transform));
    758 
    759 				u32 id = pb->parameters.acquisition_kind;
    760 				if (id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_FORCES)
    761 					cp->das_voxel_transform = m4_mul(cp->xdc_transform, cp->das_voxel_transform);
    762 
    763 				db->Sparse = id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_UHERCULES;
    764 				db->SingleFocus        = pb->parameters.single_focus;
    765 				db->SingleOrientation  = pb->parameters.single_orientation;
    766 
    767 				sd->compile_flags |= BeamformerDASCompileFlags_CoherencyWeighting * pb->parameters.coherency_weighting;
    768 				sd->layout   = layout_for_output(cp->output_points);
    769 				sd->dispatch = dispatch_for_output(sd->layout, cp->output_points);
    770 			}break;
    771 
    772 			case BeamformerShaderKind_CoherencyWeighting:{
    773 				sd->layout   = layout_for_output(cp->output_points);
    774 				sd->dispatch = dispatch_for_output(sd->layout, cp->output_points);
    775 			}break;
    776 
    777 			case BeamformerShaderKind_Reshape:{
    778 				BeamformerReshapeBakeParameters *rb = &sd->bake.Reshape;
    779 				b32 deinterleave =  beamformer_data_kind_complex[node->input_data_kind] &&
    780 				                   !beamformer_data_kind_complex[node->output_data_kind];
    781 				b32 interleave   = !beamformer_data_kind_complex[node->input_data_kind] &&
    782 				                    beamformer_data_kind_complex[node->output_data_kind];
    783 				assert(interleave == 0 || (interleave != deinterleave));
    784 				sd->compile_flags |= BeamformerReshapeCompileFlags_Deinterleave * deinterleave;
    785 				sd->compile_flags |= BeamformerReshapeCompileFlags_Interleave   * interleave;
    786 
    787 				rb->InputStrideX   = node->input_stride.x;
    788 				rb->InputStrideY   = node->input_stride.y;
    789 				rb->InputStrideZ   = node->input_stride.z;
    790 				rb->OutputStrideX  = node->output_stride.x;
    791 				rb->OutputStrideY  = node->output_stride.y;
    792 				rb->OutputStrideZ  = node->output_stride.z;
    793 
    794 				// NOTE(rnp): order doesn't really matter here but it must match the dispatch layout
    795 				rb->SizeX          = input_sample_count;
    796 				rb->SizeY          = chunk_channel_count;
    797 				rb->SizeZ          = acquisition_count;
    798 
    799 				sd->layout.x = 1;
    800 				sd->layout.z = Min(subgroup_size, rb->SizeZ);
    801 				sd->layout.y = subgroup_size / sd->layout.z;
    802 
    803 				sd->dispatch.x = (u32)(ceil_f32((f32)rb->SizeX / sd->layout.x));
    804 				sd->dispatch.y = (u32)(ceil_f32((f32)rb->SizeY / sd->layout.y));
    805 				sd->dispatch.z = (u32)(ceil_f32((f32)rb->SizeZ / sd->layout.z));
    806 			}break;
    807 
    808 			default:{}break;
    809 
    810 			#if 0
    811 			case BeamformerShaderKind_Sum:{
    812 				sd->bake.data_kind = BeamformerDataKind_Float32;
    813 				if (cp->iq_pipeline)
    814 					sd->bake.data_kind = BeamformerDataKind_Float32Complex;
    815 
    816 				sd->layout   = layout_for_output(cp->output_points);
    817 				sd->dispatch = dispatch_for_output(sd->layout, cp->output_points);
    818 
    819 				commit = 1;
    820 			}break;
    821 			#endif
    822 
    823 			}
    824 		}
    825 	}
    826 
    827 	cp->pipeline.data_kind = input_data_kind;
    828 
    829 	if (cp->first_image_shader_index == 0)
    830 		cp->first_image_shader_index = cp->pipeline.shader_count;
    831 }
    832 
    833 function void
    834 stream_append_shader_header(Stream *s, i32 reloadable_index, BeamformerShaderDescriptor *sd, uv3 layout)
    835 {
    836 	stream_append_str8(s, str8("#version 460 core\n\n"
    837 	"#extension GL_EXT_buffer_reference : require\n"
    838 	"#extension GL_EXT_shader_16bit_storage : require\n"
    839 	"#extension GL_EXT_shader_explicit_arithmetic_types : require\n\n"
    840 	"#define f32     float32_t\n"
    841 	"#define f16     float16_t\n"
    842 	"#define s32     int32_t\n"
    843 	"#define u64     uint64_t\n"
    844 	"#define u32     uint32_t\n"
    845 	"#define s16     int16_t\n"
    846 	"#define u16     uint16_t\n"
    847 	"#define s32vec2 i32vec2\n"
    848 	"#define s16vec2 i16vec2\n"
    849 	"\n"));
    850 
    851 	i32  header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index];
    852 	i32 *header_vector        = beamformer_shader_header_vectors[reloadable_index];
    853 	for (i32 index = 0; index < header_vector_length; index++)
    854 		stream_append_str8(s, beamformer_shader_global_header_strings[header_vector[index]]);
    855 
    856 	if (layout.x != 0) {
    857 		stream_append_str8(s, str8("layout(local_size_x = "));
    858 		stream_append_u64(s,  layout.x);
    859 		stream_append_str8(s, str8(", local_size_y = "));
    860 		stream_append_u64(s,  layout.y);
    861 		stream_append_str8(s, str8(", local_size_z = "));
    862 		stream_append_u64(s,  layout.z);
    863 		stream_append_str8(s, str8(") in;\n\n"));
    864 	}
    865 
    866 	{
    867 		u32 max_length = 0;
    868 		for EachElement(beamformer_data_kind_str8, it)
    869 			max_length = Max(max_length, (u32)beamformer_data_kind_str8[it].length);
    870 
    871 		for EachElement(beamformer_data_kind_str8, it) {
    872 			stream_append_str8s(s, str8("#define DataKind_"), beamformer_data_kind_str8[it]);
    873 			stream_pad(s, ' ', max_length - beamformer_data_kind_str8[it].length + 1);
    874 			stream_append_u64(s, it);
    875 			stream_append_byte(s, '\n');
    876 		}
    877 		stream_append_byte(s, '\n');
    878 	}
    879 
    880 	if (sd) {
    881 		BeamformerDataKind data_kinds[] = {sd->input_data_kind, sd->output_data_kind};
    882 		str8 line_prefixes[] = {str8_comp("Input"), str8_comp("Output")};
    883 		for EachElement(data_kinds, it) {
    884 			if (data_kinds[it] != BeamformerDataKind_Count) {
    885 				stream_append_str8s(s, str8("#define "), line_prefixes[it], str8("DataType "),
    886 				                    beamformer_data_kind_glsl_type[data_kinds[it]],
    887 				                    str8("\n#define "), line_prefixes[it], str8("DataKind DataKind_"),
    888 				                    beamformer_data_kind_str8[data_kinds[it]],
    889 				                    str8("\n#define "), line_prefixes[it], str8("DataKindByteSize "));
    890 				stream_append_u64(s, beamformer_data_kind_byte_size[data_kinds[it]]);
    891 				stream_append_byte(s, '\n');
    892 			}
    893 		}
    894 		stream_append_byte(s, '\n');
    895 
    896 		stream_append_str8(s, str8("#define CompileFlags (0x"));
    897 		stream_append_hex_u64_width(s, sd->compile_flags, 8);
    898 		stream_append_str8(s, str8(")\n"));
    899 
    900 		i32 struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index];
    901 		if (struct_id != -1) {
    902 			str8             *names = meta_struct_member_names_by_id[struct_id];
    903 			MetaStructInfo   *si    = meta_struct_info_by_id + struct_id;
    904 			MetaStructMember *sm    = meta_struct_members_by_id[struct_id];
    905 			for (u32 index = 0; index < si->member_count; index++) {
    906 				str8 type = meta_kind_glsl_types[sm[index].type_id];
    907 				stream_append_str8(s, str8("layout(constant_id = "));
    908 				stream_append_u64(s, index);
    909 				stream_append_str8s(s, str8(") const "), type, str8(" "), names[index], str8(" = "), type, str8("(1);\n"));
    910 			}
    911 		}
    912 	}
    913 
    914 	if (!renderdoc_attached())
    915 		stream_append_str8(s, str8("\n\n#line 1\n"));
    916 }
    917 
    918 function void
    919 beamformer_reload_pipeline(VulkanHandle *pipeline, BeamformerShaderReloadInfo *sris, u32 count, Arena *scratch)
    920 {
    921 	assume(count <= 2);
    922 	str8 paths[2];
    923 	VulkanPipelineCreateInfo infos[2];
    924 
    925 	if (!BakeShaders) {
    926 		for (u32 i = 0; i < count; i++)
    927 			paths[i] = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), sris[i].filename_or_data);
    928 	}
    929 
    930 	u32 push_constants_size = 0;
    931 	for (u32 i = 0; i < count; i++) {
    932 		Stream shader_stream = arena_stream(scratch);
    933 		i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[sris[i].shader];
    934 		if (i == 0) push_constants_size = beamformer_shader_push_constant_sizes[reloadable_index];
    935 		else        assert(push_constants_size == beamformer_shader_push_constant_sizes[reloadable_index]);
    936 
    937 		stream_append_shader_header(&shader_stream, reloadable_index, sris[i].shader_descriptor, sris[i].layout);
    938 
    939 		str8 shader_text;
    940 		if (BakeShaders) {
    941 			stream_append_str8(&shader_stream, sris[i].filename_or_data);
    942 			shader_text = arena_stream_commit_zero(scratch, &shader_stream);
    943 		} else {
    944 			str8 stream_data = arena_stream_commit(scratch, &shader_stream);
    945 			str8 shader_data = os_read_entire_file(scratch, (c8 *)paths[i].data);
    946 			// NOTE(rnp): kinda sucky but need to make sure these are a contiguous string
    947 			shader_text = push_str8_from_parts(scratch, str8(""), stream_data, shader_data);
    948 		}
    949 
    950 		infos[i].kind = sris[i].shader_kind;
    951 		infos[i].text = shader_text;
    952 		infos[i].name = beamformer_shader_names[sris[i].shader];
    953 		infos[i].specialization_data      = sris[i].shader_descriptor ? &sris[i].shader_descriptor->bake : 0;
    954 		infos[i].specialization_struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index];
    955 
    956 		//str8 line = str8("---------------\n");
    957 		//str8 nl   = str8("\n");
    958 		//os_console_log(line.data, line.length);
    959 		//os_console_log(infos[i].name.data, infos[i].name.length);
    960 		//os_console_log(nl.data, nl.length);
    961 		//os_console_log(line.data, line.length);
    962 		//os_console_log(infos[i].text.data, infos[i].text.length);
    963 		//os_console_log(line.data, line.length);
    964 	}
    965 
    966 	vk_pipeline_release(*pipeline);
    967 	*pipeline = vk_pipeline(infos, count, push_constants_size);
    968 }
    969 
    970 function void
    971 beamformer_reload_render_pipeline(VulkanHandle *pipeline, BeamformerShaderKind shader, Arena *scratch)
    972 {
    973 	i32 index = beamformer_shader_reloadable_index_by_shader[shader];
    974 	BeamformerShaderReloadInfo infos[2] = {
    975 		{
    976 			.shader      = shader,
    977 			.shader_kind = beamformer_shader_primitive_is_vertex[index] ? VulkanShaderKind_Vertex : VulkanShaderKind_Mesh,
    978 			.filename_or_data = BakeShaders ? beamformer_shader_data[index][0]
    979 			                                : beamformer_reloadable_shader_files[index][0],
    980 		},
    981 		{
    982 			.shader           = shader,
    983 			.shader_kind      = VulkanShaderKind_Fragment,
    984 			.filename_or_data = BakeShaders ? beamformer_shader_data[index][1]
    985 			                                : beamformer_reloadable_shader_files[index][1],
    986 		},
    987 	};
    988 	beamformer_reload_pipeline(pipeline, infos, countof(infos), scratch);
    989 }
    990 
    991 function void
    992 beamformer_reload_compute_pipeline(VulkanHandle *pipeline, BeamformerShaderKind shader,
    993                                    BeamformerShaderDescriptor *shader_descriptor, Arena *scratch)
    994 {
    995 	i32 index  = beamformer_shader_reloadable_index_by_shader[shader];
    996 	uv3 layout = shader_descriptor ? shader_descriptor->layout : (uv3){{vk_gpu_info()->subgroup_size, 1, 1}};
    997 	BeamformerShaderReloadInfo info = {
    998 		.shader            = shader,
    999 		.shader_kind       = VulkanShaderKind_Compute,
   1000 		.shader_descriptor = shader_descriptor,
   1001 		.filename_or_data  = BakeShaders ? beamformer_shader_data[index][0]
   1002 		                                 : beamformer_reloadable_shader_files[index][0],
   1003 		.layout            = layout,
   1004 	};
   1005 	beamformer_reload_pipeline(pipeline, &info, 1, scratch);
   1006 }
   1007 
   1008 function void
   1009 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena *scratch)
   1010 {
   1011 	BeamformerParameterBlock *pb;
   1012 	DeferLoop(pb = beamformer_parameter_block_lock(ctx->shared_memory, block, -1),
   1013 	          beamformer_parameter_block_unlock(ctx->shared_memory, block))
   1014 	for EachBit(pb->region_update_flags, region)
   1015 	{
   1016 		pb->region_update_flags &= ~(1ul << region);
   1017 		switch (region) {
   1018 		case BeamformerParameterRegionFlag_NotifyUI:{
   1019 			atomic_store_u32(&ctx->ui_dirty_parameter_blocks, 1u << block);
   1020 		}break;
   1021 
   1022 		case BeamformerParameterRegionFlag_ComputePipeline:
   1023 		case BeamformerParameterRegionFlag_Parameters:
   1024 		{
   1025 			cp->output_points  = das_valid_points(pb->parameters.output_points.xyz);
   1026 			cp->average_frames = pb->parameters.output_points.E[3];
   1027 
   1028 			plan_compute_pipeline(cp, pb, scratch);
   1029 
   1030 			/* NOTE(rnp): these are both handled by plan_compute_pipeline() */
   1031 			u32 mask = 1 << BeamformerParameterBlockRegion_ComputePipeline |
   1032 			           1 << BeamformerParameterBlockRegion_Parameters;
   1033 			pb->region_update_flags &= ~mask;
   1034 
   1035 			for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) {
   1036 				u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor));
   1037 				if (!u128_equal(hash, cp->shader_hashes[shader_slot]))
   1038 					cp->dirty_programs |= 1 << shader_slot;
   1039 				cp->shader_hashes[shader_slot] = hash;
   1040 			}
   1041 
   1042 			cp->acquisition_count = pb->parameters.acquisition_count;
   1043 			cp->acquisition_kind  = pb->parameters.acquisition_kind;
   1044 			cp->contrast_mode     = pb->parameters.contrast_mode;
   1045 
   1046 			i64 buffer_size = PING_PONG_BUFFER_SLOTS * round_up_to(cp->rf_size, 64);
   1047 			if (ctx->compute_context.ping_pong_buffer.size < buffer_size) {
   1048 				b32 cuda = cuda_supported();
   1049 				GPUBufferAllocateInfo allocate_info = {
   1050 					.size   = buffer_size,
   1051 					.export = cuda ? &ctx->compute_context.ping_pong_export_handle : 0,
   1052 					.label  = str8("PingPongBuffer"),
   1053 				};
   1054 				vk_buffer_allocate(&ctx->compute_context.ping_pong_buffer, &allocate_info);
   1055 
   1056 				BeamformerShaderResourceInfo shader_resource_infos[] = {
   1057 					{
   1058 						.kind   = BeamformerShaderResourceKind_Buffer,
   1059 						.handle = ctx->compute_context.ping_pong_buffer.handle,
   1060 						.slot   = BeamformerShaderBufferSlot_PingPong,
   1061 					},
   1062 				};
   1063 				vk_bind_shader_resources(shader_resource_infos, countof(shader_resource_infos));
   1064 
   1065 				// TODO(rnp): figure out how to share with CUDA
   1066 				// IMPORTANT: on linux the handle is returned to os and should be cleared after import
   1067 				// see usage of glImportMemoryFdEXT and surrounding code in ui.c for examples
   1068 				if (cuda) {
   1069 				}
   1070 			}
   1071 
   1072 			if (pb->parameters.decode_mode != BeamformerDecodeMode_None &&
   1073 			    cp->hadamard_order != (i32)cp->acquisition_count)
   1074 			{
   1075 				beamformer_update_hadamard(cp, (i32)cp->acquisition_count, vk_gpu_info()->cooperative_matrix, 0, scratch);
   1076 				if (pb->parameters.readi_group_count > 1)
   1077 					beamformer_update_hadamard(cp, (i32)pb->parameters.readi_group_count, 0, 1, scratch);
   1078 			}
   1079 		}break;
   1080 
   1081 		case BeamformerParameterBlockRegion_ChannelMapping:{
   1082 			cuda_set_channel_mapping(pb->channel_mapping);
   1083 		}break;
   1084 		case BeamformerParameterRegionFlag_TransmitReceiveOrientations:{
   1085 			GPUBuffer *b = &cp->array_parameters;
   1086 			u32 kind   = BeamformerComputeArrayParameterKind_TransmitReceiveOrientations;
   1087 			u64 offset = beamformer_compute_array_parameter_offsets[kind];
   1088 			u64 size   = beamformer_compute_array_parameter_sizes[kind];
   1089 			{
   1090 				u16 *u16s = push_array(scratch, u16, countof(pb->transmit_receive_orientations));
   1091 				for (u32 i = 0; i < countof(pb->transmit_receive_orientations); i++)
   1092 					u16s[i] = pb->transmit_receive_orientations[i];
   1093 
   1094 				vk_buffer_range_upload(b, u16s, offset, size, 0);
   1095 			}
   1096 		}break;
   1097 		case BeamformerParameterRegionFlag_FocalVectors:
   1098 		case BeamformerParameterRegionFlag_SparseElements:
   1099 		{
   1100 			u32 kind = BeamformerComputeArrayParameterKind_Count;
   1101 			switch (region) {
   1102 			case BeamformerParameterBlockRegion_FocalVectors:{
   1103 				kind = BeamformerComputeArrayParameterKind_FocalVectors;
   1104 			}break;
   1105 			case BeamformerParameterBlockRegion_SparseElements:{
   1106 				kind = BeamformerComputeArrayParameterKind_SparseElements;
   1107 			}break;
   1108 			InvalidDefaultCase;
   1109 			}
   1110 
   1111 			if (kind != BeamformerComputeArrayParameterKind_Count) {
   1112 				GPUBuffer *b = &cp->array_parameters;
   1113 				u64 offset = beamformer_compute_array_parameter_offsets[kind];
   1114 				u64 size   = beamformer_compute_array_parameter_sizes[kind];
   1115 				vk_buffer_range_upload(b, (u8 *)pb + BeamformerParameterBlockRegionOffsets[region], offset, size, 0);
   1116 			}
   1117 		}break;
   1118 		}
   1119 	}
   1120 }
   1121 
   1122 function void
   1123 do_compute_shader(BeamformerCtx *ctx, VulkanHandle cmd, BeamformerComputePlan *cp, BeamformerFrame *frame,
   1124                   u32 shader_slot, u32 channel_offset, u64 rf_pointer)
   1125 {
   1126 	BeamformerComputeContext *cc = &ctx->compute_context;
   1127 
   1128 	u32 output_index     = !cc->ping_pong_input_index;
   1129 	u32 input_index      =  cc->ping_pong_input_index;
   1130 	u32 das_output_index =  PING_PONG_BUFFER_SLOTS - 1;
   1131 
   1132 	u64 pp_size           = cc->ping_pong_buffer.size / PING_PONG_BUFFER_SLOTS;
   1133 	u64 pp_input_pointer  = cc->ping_pong_buffer.gpu_pointer + input_index      * pp_size;
   1134 	u64 pp_output_pointer = cc->ping_pong_buffer.gpu_pointer + output_index     * pp_size;
   1135 	u64 pp_das_pointer    = cc->ping_pong_buffer.gpu_pointer + das_output_index * pp_size;
   1136 
   1137 	u32 das_index = cp->first_image_shader_index - 1;
   1138 
   1139 	uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch;
   1140 
   1141 	vk_command_bind_pipeline(cmd, cp->vulkan_pipelines[shader_slot]);
   1142 
   1143 	switch (cp->pipeline.shaders[shader_slot]) {
   1144 
   1145 	case BeamformerShaderKind_Decode:{
   1146 		BeamformerDecodePushConstants pc = {
   1147 			.hadamard_buffer = cp->array_parameters.gpu_pointer + offsetof(BeamformerComputeArrayParameters, DecodeHadamard),
   1148 			.rf_buffer       = pp_input_pointer,
   1149 		};
   1150 
   1151 		if ((shader_slot + 1) == das_index) pc.output_buffer = pp_das_pointer;
   1152 		else                                pc.output_buffer = pp_output_pointer;
   1153 
   1154 		GPUMemoryBarrierInfo memory_barriers[]= {
   1155 			// NOTE(rnp): first pass or last stage output
   1156 			{
   1157 				.gpu_buffer = &cc->ping_pong_buffer,
   1158 				.offset     = pp_input_pointer - cc->ping_pong_buffer.gpu_pointer,
   1159 				.size       = pp_size,
   1160 			},
   1161 			// NOTE(rnp): output for DAS
   1162 			{
   1163 				.gpu_buffer = &cc->ping_pong_buffer,
   1164 				.offset     = pp_das_pointer - cc->ping_pong_buffer.gpu_pointer,
   1165 				.size       = pp_size,
   1166 			},
   1167 		};
   1168 
   1169 		u32 barrier_count = 1;
   1170 		if (shader_slot + 1 == das_index)
   1171 			barrier_count++;
   1172 
   1173 		vk_command_buffer_memory_barriers(cmd, memory_barriers, barrier_count);
   1174 		vk_command_push_constants(cmd, 0, sizeof(pc), &pc);
   1175 		vk_command_dispatch_compute(cmd, dispatch);
   1176 
   1177 		cc->ping_pong_input_index = !cc->ping_pong_input_index;
   1178 	}break;
   1179 
   1180 	case BeamformerShaderKind_Hilbert:{
   1181 		cuda_hilbert(input_index, output_index);
   1182 		cc->ping_pong_input_index = !cc->ping_pong_input_index;
   1183 	}break;
   1184 
   1185 	case BeamformerShaderKind_Filter:
   1186 	case BeamformerShaderKind_Demodulate:
   1187 	{
   1188 		BeamformerDataKind output_data_kind = cp->shader_descriptors[shader_slot].output_data_kind;
   1189 
   1190 		u64 element_size = beamformer_data_kind_byte_size[output_data_kind];
   1191 		u32 filter_slot  = cp->pipeline.parameters[shader_slot].filter_slot;
   1192 		BeamformerFilterPushConstants pc = {
   1193 			.filter_coefficients   = cp->filters[filter_slot].buffer.gpu_pointer,
   1194 			.input_data            = shader_slot == 0 ? rf_pointer : pp_input_pointer,
   1195 			.output_element_offset = output_index * pp_size / element_size,
   1196 		};
   1197 
   1198 		if ((shader_slot + 1) == das_index)
   1199 			pc.output_element_offset = das_output_index * pp_size / element_size;
   1200 
   1201 		GPUMemoryBarrierInfo memory_barriers[] = {
   1202 			// NOTE(rnp): last stage output
   1203 			{
   1204 				.gpu_buffer = &cc->ping_pong_buffer,
   1205 				.offset     = pp_input_pointer - cc->ping_pong_buffer.gpu_pointer,
   1206 				.size       = pp_size,
   1207 			},
   1208 			// NOTE(rnp): output for DAS
   1209 			{
   1210 				.gpu_buffer = &cc->ping_pong_buffer,
   1211 				.offset     = pp_das_pointer - cc->ping_pong_buffer.gpu_pointer,
   1212 				.size       = pp_size,
   1213 			},
   1214 		};
   1215 		GPUMemoryBarrierInfo *barriers = memory_barriers;
   1216 
   1217 		u32 barrier_count = 2;
   1218 		if (shader_slot == 0) {
   1219 			barriers++;
   1220 			barrier_count--;
   1221 		}
   1222 
   1223 		if ((shader_slot + 1) != das_index)
   1224 			barrier_count--;
   1225 
   1226 		if (barrier_count)
   1227 			vk_command_buffer_memory_barriers(cmd, barriers, barrier_count);
   1228 
   1229 		vk_command_push_constants(cmd, 0, sizeof(pc), &pc);
   1230 		vk_command_dispatch_compute(cmd, dispatch);
   1231 
   1232 		cc->ping_pong_input_index = !cc->ping_pong_input_index;
   1233 	}break;
   1234 
   1235 	case BeamformerShaderKind_DAS:{
   1236 		local_persist u32 das_cycle_t = 0;
   1237 
   1238 		GPUBuffer *b = cc->backlog.buffer;
   1239 
   1240 		u64 frame_size   = beamformer_frame_byte_size(frame->points, frame->data_kind);
   1241 		u64 iframe_size  = frame_size / beamformer_data_kind_element_count[frame->data_kind];
   1242 		u64 element_size = beamformer_data_kind_byte_size[cp->shader_descriptors[shader_slot].input_data_kind];
   1243 
   1244 		BeamformerDASPushConstants pc = {
   1245 			.xdc_element_pitch = cp->xdc_element_pitch,
   1246 			.rf_element_offset = das_output_index * pp_size / element_size,
   1247 			.output_frame      = b->gpu_pointer + frame->buffer_offset,
   1248 			.incoherent_frame  = b->gpu_pointer + b->size - iframe_size,
   1249 			.output_size_x     = cp->output_points.x,
   1250 			.output_size_y     = cp->output_points.y,
   1251 			.output_size_z     = cp->output_points.z,
   1252 			.cycle_t           = das_cycle_t++,
   1253 			.channel_offset    = channel_offset,
   1254 			.readi_group       = cp->readi_group,
   1255 			.array_parameters  = cp->array_parameters.gpu_pointer + offsetof(BeamformerComputeArrayParameters, FocalVectors),
   1256 		};
   1257 		memory_copy(pc.voxel_transform.E, cp->das_voxel_transform.E, sizeof(pc.voxel_transform));
   1258 		memory_copy(pc.xdc_transform.E,   cp->xdc_transform.E,       sizeof(pc.xdc_transform));
   1259 
   1260 		b32 coherent = (cp->shader_descriptors[shader_slot].compile_flags & BeamformerDASCompileFlags_CoherencyWeighting) != 0;
   1261 
   1262 		GPUMemoryBarrierInfo memory_barriers[] = {
   1263 			// NOTE(rnp): last stage data output barrier
   1264 			{
   1265 				.gpu_buffer = &cc->ping_pong_buffer,
   1266 				.offset     = pp_das_pointer - cc->ping_pong_buffer.gpu_pointer,
   1267 				.size       = pp_size,
   1268 			},
   1269 			// NOTE(rnp): output clearing pipeline barriers or last DAS pipeline write barriers
   1270 			{
   1271 				.gpu_buffer = b,
   1272 				.offset     = frame->buffer_offset,
   1273 				.size       = frame_size,
   1274 			},
   1275 			{
   1276 				.gpu_buffer = b,
   1277 				.offset     = pc.incoherent_frame - b->gpu_pointer,
   1278 				.size       = iframe_size,
   1279 			},
   1280 		};
   1281 
   1282 		u32 barrier_count = countof(memory_barriers);
   1283 		if (!coherent) barrier_count--;
   1284 
   1285 		vk_command_buffer_memory_barriers(cmd, memory_barriers, barrier_count);
   1286 		vk_command_push_constants(cmd, 0, sizeof(pc), &pc);
   1287 		vk_command_dispatch_compute(cmd, dispatch);
   1288 	}break;
   1289 
   1290 	case BeamformerShaderKind_CoherencyWeighting:{
   1291 		GPUBuffer *b = cc->backlog.buffer;
   1292 
   1293 		u64 frame_size  = beamformer_frame_byte_size(frame->points, frame->data_kind);
   1294 		u64 iframe_size = frame_size / beamformer_data_kind_element_count[frame->data_kind];
   1295 
   1296 		BeamformerCoherencyWeightingPushConstants pc = {
   1297 			.left_side_buffer  = b->gpu_pointer + frame->buffer_offset,
   1298 			.right_side_buffer = b->gpu_pointer + b->size - iframe_size,
   1299 			.scale             = 1.0f,
   1300 			.output_size_x     = cp->output_points.x,
   1301 			.output_size_y     = cp->output_points.y,
   1302 			.output_size_z     = cp->output_points.z,
   1303 		};
   1304 
   1305 		GPUMemoryBarrierInfo memory_barriers[] = {
   1306 			{
   1307 				.gpu_buffer = b,
   1308 				.offset     = frame->buffer_offset,
   1309 				.size       = frame_size,
   1310 			},
   1311 			{
   1312 				.gpu_buffer = b,
   1313 				.offset     = pc.right_side_buffer - b->gpu_pointer,
   1314 				.size       = iframe_size,
   1315 			},
   1316 		};
   1317 
   1318 		vk_command_buffer_memory_barriers(cmd, memory_barriers, countof(memory_barriers));
   1319 		vk_command_push_constants(cmd, 0, sizeof(pc), &pc);
   1320 		vk_command_dispatch_compute(cmd, dispatch);
   1321 	}break;
   1322 
   1323 	case BeamformerShaderKind_Reshape:{
   1324 		BeamformerDataKind input_data_kind = cp->shader_descriptors[shader_slot].input_data_kind;
   1325 		BeamformerReshapeBakeParameters *rb = &cp->shader_descriptors[shader_slot].bake.Reshape;
   1326 		u64 input_pointer = shader_slot == 0 ? rf_pointer : pp_input_pointer;
   1327 		BeamformerReshapePushConstants pc = {
   1328 			.left_input_buffer  = input_pointer,
   1329 			.right_input_buffer = input_pointer + rb->SizeX * rb->SizeY * rb->SizeZ
   1330 			                                      * beamformer_data_kind_byte_size[input_data_kind],
   1331 		};
   1332 
   1333 		if ((shader_slot + 1) == das_index) pc.output_buffer = pp_das_pointer;
   1334 		else                                pc.output_buffer = pp_output_pointer;
   1335 
   1336 		GPUMemoryBarrierInfo memory_barriers[]= {
   1337 			// NOTE(rnp): first pass or last stage output
   1338 			{
   1339 				.gpu_buffer = &cc->ping_pong_buffer,
   1340 				.offset     = pp_input_pointer - cc->ping_pong_buffer.gpu_pointer,
   1341 				.size       = pp_size,
   1342 			},
   1343 			// NOTE(rnp): output for DAS
   1344 			{
   1345 				.gpu_buffer = &cc->ping_pong_buffer,
   1346 				.offset     = pp_das_pointer - cc->ping_pong_buffer.gpu_pointer,
   1347 				.size       = pp_size,
   1348 			},
   1349 		};
   1350 
   1351 		u32 barrier_count = 1;
   1352 		if (shader_slot + 1 == das_index)
   1353 			barrier_count++;
   1354 
   1355 		vk_command_buffer_memory_barriers(cmd, memory_barriers, barrier_count);
   1356 		vk_command_push_constants(cmd, 0, sizeof(pc), &pc);
   1357 		vk_command_dispatch_compute(cmd, dispatch);
   1358 
   1359 		cc->ping_pong_input_index = !cc->ping_pong_input_index;
   1360 	}break;
   1361 
   1362 	// NOTE(rnp): invalid stages should be filtered in planning phase
   1363 	InvalidDefaultCase;
   1364 	}
   1365 
   1366 	#if 0
   1367 	switch (shader) {
   1368 	case BeamformerShaderKind_MinMax:{
   1369 		for (u32 i = 1; i < frame->image.mip_map_levels; i++) {
   1370 			glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY,  GL_RG32F);
   1371 			glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F);
   1372 			glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i);
   1373 
   1374 			u32 width  = (u32)frame->dim.x >> i;
   1375 			u32 height = (u32)frame->dim.y >> i;
   1376 			u32 depth  = (u32)frame->dim.z >> i;
   1377 			glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32));
   1378 			glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
   1379 		}
   1380 	}break;
   1381 	case BeamformerShaderKind_Sum:{
   1382 		u32 aframe_index = ctx->averaged_frame_index % countof(ctx->averaged_frames);
   1383 		BeamformerFrame *aframe = ctx->averaged_frames + aframe_index;
   1384 		aframe->id              = ctx->averaged_frame_index;
   1385 		atomic_store_u32(&aframe->ready_to_present, 0);
   1386 		/* TODO(rnp): hack we need a better way of specifying which frames to sum;
   1387 		 * this is fine for rolling averaging but what if we want to do something else */
   1388 		assert(frame >= ctx->beamform_frames);
   1389 		assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames));
   1390 		u32 base_index   = (u32)(frame - ctx->beamform_frames);
   1391 		u32 to_average   = (u32)cp->average_frames;
   1392 		u32 frame_count  = 0;
   1393 		u32 *in_textures = push_array(&arena, u32, BeamformerMaxBacklogFrames);
   1394 		ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average);
   1395 		for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi))
   1396 			in_textures[frame_count++] = it->texture;
   1397 
   1398 		assert(to_average == frame_count);
   1399 
   1400 		glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count);
   1401 		/* NOTE: zero output before summing */
   1402 		glClearTexImage(aframe->texture, 0, GL_RED, GL_FLOAT, 0);
   1403 		glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT);
   1404 
   1405 		glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F);
   1406 		for (u32 i = 0; i < in_texture_count; i++) {
   1407 			glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F);
   1408 			glDispatchCompute(dispatch.x, dispatch.y, dispatch.z);
   1409 			glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
   1410 		}
   1411 
   1412 		memory_copy(aframe->voxel_transform.E,  frame->voxel_transform.E, sizeof(frame->voxel_transform));
   1413 		aframe->compound_count   = frame->compound_count;
   1414 		aframe->acquisition_kind = frame->acquisition_kind;
   1415 	}break;
   1416 	}
   1417 	#endif
   1418 }
   1419 
   1420 function void
   1421 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
   1422 {
   1423 	BeamformerComputeContext * cs = &ctx->compute_context;
   1424 	BeamformerSharedMemory *   sm = ctx->shared_memory;
   1425 
   1426 	for (BeamformWork *work = beamform_work_queue_pop(q);
   1427 	     work;
   1428 	     beamform_work_queue_pop_commit(q), work = beamform_work_queue_pop(q))
   1429 	{
   1430 		switch (work->kind) {
   1431 
   1432 		case BeamformerWorkKind_ExportBuffer:{
   1433 			/* TODO(rnp): better way of handling DispatchCompute barrier */
   1434 			post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute);
   1435 			beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)work->lock, (u32)-1);
   1436 			BeamformerExportContext *ec = &work->export_context;
   1437 			switch (ec->kind) {
   1438 			case BeamformerExportKind_BeamformedData:{
   1439 				BeamformerFrameBacklog *bl = &ctx->compute_context.backlog;
   1440 				u32 req_count = Clamp(ec->count, 1, bl->counter);
   1441 				u32 frame_idx = bl->counter - req_count;
   1442 				u8 *sm_output = beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size);
   1443 				u64 exported_size = 0;
   1444 				for (u32 export_count = 0; export_count < req_count; export_count++, frame_idx++) {
   1445 					BeamformerFrame *f = bl->frames + frame_idx % countof(bl->frames);
   1446 					u64 frame_size = beamformer_frame_byte_size(f->points, f->data_kind);
   1447 					assert((frame_size & 63) == 0);
   1448 					// NOTE(tkh) we don't want to assume that all req_count frames are the same size,
   1449 					// so we either need to count the total size of all requested frames first or
   1450 					// just fill up as much as possible.
   1451 					if (exported_size + frame_size <= ec->size) {
   1452 						vk_host_wait_timeline(VulkanTimeline_Compute, f->timeline_valid_value, -1ULL);
   1453 						vk_buffer_range_download(sm_output + exported_size, bl->buffer, f->buffer_offset, frame_size, 1);
   1454 						exported_size += frame_size;
   1455 					}
   1456 				}
   1457 			}break;
   1458 
   1459 			case BeamformerExportKind_Stats:{
   1460 				ComputeTimingTable *table = ctx->compute_timing_table;
   1461 				/* NOTE(rnp): do a little spin to let this finish updating */
   1462 				spin_wait(table->write_index != atomic_load_u32(&table->read_index));
   1463 				ComputeShaderStats *stats = ctx->compute_shader_stats;
   1464 				if (sizeof(stats->table) <= ec->size)
   1465 					memory_copy(beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
   1466 					         &stats->table, sizeof(stats->table));
   1467 			}break;
   1468 			InvalidDefaultCase;
   1469 			}
   1470 			beamformer_shared_memory_release_lock(ctx->shared_memory, work->lock);
   1471 			post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync);
   1472 		}break;
   1473 
   1474 		case BeamformerWorkKind_CreateFilter:{
   1475 			/* TODO(rnp): this should probably get deleted and moved to lazy loading */
   1476 			BeamformerCreateFilterContext *fctx = &work->create_filter_context;
   1477 			u32 block = fctx->parameter_block;
   1478 			u32 slot  = fctx->filter_slot;
   1479 			BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena);
   1480 			beamformer_filter_update(cp->filters + slot, fctx->parameters, block, slot, arena);
   1481 		}break;
   1482 
   1483 		case BeamformerWorkKind_ComputeIndirect:
   1484 		case BeamformerWorkKind_Compute:
   1485 		{
   1486 			push_compute_timing_info(ctx->compute_timing_table,
   1487 			                         (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin});
   1488 
   1489 			BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena);
   1490 			if unlikely(beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) {
   1491 				u32 block = work->compute_context.parameter_block;
   1492 				Temp scratch = temp_begin(arena);
   1493 				beamformer_commit_parameter_block(ctx, cp, block, arena);
   1494 				temp_end(scratch);
   1495 			}
   1496 
   1497 			post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute);
   1498 
   1499 			u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0);
   1500 			static_assert(BeamformerMaxComputeShaderStages <= 32, "");
   1501 			if unlikely(dirty_programs) {
   1502 				for EachBit(dirty_programs, slot) {
   1503 					assert(slot < BeamformerMaxComputeShaderStages);
   1504 					Temp scratch = temp_begin(arena);
   1505 					beamformer_reload_compute_pipeline(cp->vulkan_pipelines + slot,
   1506 					                                   cp->pipeline.shaders[slot],
   1507 					                                   cp->shader_descriptors + slot, arena);
   1508 					temp_end(scratch);
   1509 				}
   1510 			}
   1511 
   1512 			atomic_store_u32(&cs->processing_compute, 1);
   1513 
   1514 			start_renderdoc_capture();
   1515 
   1516 			i32 das_index = -1;
   1517 			b32 has_sum   = 0;
   1518 			for (u32 i = 0; i < cp->pipeline.shader_count; i++) {
   1519 				has_sum |= cp->pipeline.shaders[i] == BeamformerShaderKind_Sum;
   1520 				if (cp->pipeline.shaders[i] == BeamformerShaderKind_DAS)
   1521 					das_index = (i32)i;
   1522 			}
   1523 
   1524 			b32 das_coherent = das_index >= 0 &&
   1525 			                   (cp->shader_descriptors[das_index].compile_flags &
   1526 			                    BeamformerDASCompileFlags_CoherencyWeighting) != 0;
   1527 			u64 reserved_frame_size = 0;
   1528 
   1529 			if (has_sum)
   1530 				reserved_frame_size += beamformer_frame_byte_size(cp->output_points, cp->iq_pipeline ?
   1531 				                                                  BeamformerDataKind_Float32Complex :
   1532 				                                                  BeamformerDataKind_Float32);
   1533 
   1534 			// TODO(rnp): incoherent sum for different data kinds
   1535 			if (das_coherent)
   1536 				reserved_frame_size += beamformer_frame_byte_size(cp->output_points, BeamformerDataKind_Float32);
   1537 
   1538 			BeamformerFrame *frame  = beamformer_frame_next(cs, cp->output_points, cp->iq_pipeline, reserved_frame_size);
   1539 			frame->acquisition_kind = cp->acquisition_kind;
   1540 			frame->contrast_mode    = cp->contrast_mode;
   1541 			frame->compound_count   = cp->acquisition_count;
   1542 			frame->parameter_block  = work->compute_context.parameter_block;
   1543 			frame->view_plane_tag   = work->compute_context.view_plane;
   1544 			memory_copy(frame->voxel_transform.E, cp->voxel_transform.E, sizeof(cp->voxel_transform));
   1545 
   1546 			VulkanHandle cmd = vk_command_begin(VulkanTimeline_Compute);
   1547 			vk_command_timestamp(cmd);
   1548 
   1549 			if (das_index >= 0) {
   1550 				u64        frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind);
   1551 				GPUBuffer *backlog    = cs->backlog.buffer;
   1552 
   1553 				vk_command_clear_buffer(cmd, backlog, frame->buffer_offset, frame_size, 0);
   1554 				if (das_coherent) {
   1555 					u64 coherent_size = frame_size / beamformer_data_kind_element_count[frame->data_kind];
   1556 					vk_command_clear_buffer(cmd, backlog, backlog->size - coherent_size, coherent_size, 0);
   1557 				}
   1558 			}
   1559 
   1560 			BeamformerRFBuffer *rf = &cs->rf_buffer;
   1561 			u32 compute_index = rf->compute_index;
   1562 			u32 slot = compute_index % countof(rf->upload_complete_values);
   1563 
   1564 			if (work->kind == BeamformerWorkKind_ComputeIndirect) {
   1565 				// TODO(rnp): this shouldn't be necessary, there should be a way of communicating
   1566 				// what the value will be so that the only the command wait is needed.
   1567 				spin_wait(atomic_load_u64(&rf->insertion_index) <= compute_index);
   1568 
   1569 				/* NOTE(rnp): if the GPU supports BAR there may be no need to synchronize
   1570 				 * other than the above spin */
   1571 				if (vk_buffer_needs_sync(&rf->buffer))
   1572 					vk_command_wait_timeline(cmd, VulkanTimeline_Transfer, rf->upload_complete_values[slot]);
   1573 			} else {
   1574 				slot = (rf->compute_index - 1) % countof(rf->upload_complete_values);
   1575 			}
   1576 
   1577 			for (u32 channel_offset = 0;
   1578 			     channel_offset < cp->channel_count;
   1579 			     channel_offset += BeamformerChunkChannelCount)
   1580 			{
   1581 				u64 rf_pointer = rf->buffer.gpu_pointer + slot * rf->active_rf_size;
   1582 				rf_pointer += cp->raw_channel_byte_stride * channel_offset;
   1583 				for (u32 i = 0; i < cp->first_image_shader_index; i++) {
   1584 					do_compute_shader(ctx, cmd, cp, frame, i, channel_offset, rf_pointer);
   1585 					vk_command_timestamp(cmd);
   1586 				}
   1587 			}
   1588 
   1589 			for (u32 i = cp->first_image_shader_index; i < cp->pipeline.shader_count; i++) {
   1590 				do_compute_shader(ctx, cmd, cp, frame, i, 0, 0);
   1591 				vk_command_timestamp(cmd);
   1592 			}
   1593 
   1594 			u64 end_timeline_value = vk_command_end(cmd, (VulkanHandle){0}, (VulkanHandle){0});
   1595 			if (work->kind == BeamformerWorkKind_ComputeIndirect) {
   1596 				atomic_store_u64(rf->compute_complete_values + slot, end_timeline_value);
   1597 				atomic_add_u64(&rf->compute_index, 1);
   1598 			}
   1599 
   1600 			atomic_store_u64(&frame->timeline_valid_value, end_timeline_value);
   1601 
   1602 			Temp scratch;
   1603 			DeferLoop(scratch = temp_begin(arena), temp_end(scratch))
   1604 			{
   1605 				/* NOTE(rnp): this blocks until work completes */
   1606 				u64 *timestamps  = vk_command_read_timestamps(VulkanTimeline_Compute, arena);
   1607 
   1608 				i32 steps        = ((i32)cp->channel_count / BeamformerChunkChannelCount) - 1;
   1609 				i32 step         = 0;
   1610 				u32 shader_index = 0;
   1611 				u64 last_time    = timestamps[0] > 0 ? timestamps[1] : 0;
   1612 
   1613 				for (u64 i = 2; i < timestamps[0] + 1; i++) {
   1614 					push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){
   1615 						.kind        = ComputeTimingInfoKind_Shader,
   1616 						.shader      = cp->pipeline.shaders[shader_index],
   1617 						.shader_slot = shader_index,
   1618 						.timer_count = timestamps[i] - last_time,
   1619 					});
   1620 					last_time = timestamps[i];
   1621 
   1622 					shader_index++;
   1623 					if (shader_index == cp->first_image_shader_index && step < steps) {
   1624 						shader_index = 0;
   1625 						step++;
   1626 					}
   1627 				}
   1628 			}
   1629 
   1630 			cs->processing_progress = 1;
   1631 
   1632 			if (has_sum) {
   1633 				#if 0
   1634 				u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames));
   1635 				ctx->averaged_frames[aframe_index].view_plane_tag  = frame->view_plane_tag;
   1636 				ctx->averaged_frames[aframe_index].ready_to_present = 1;
   1637 				atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index));
   1638 				#endif
   1639 			} else {
   1640 				atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame);
   1641 			}
   1642 
   1643 			atomic_store_u32(&cs->processing_compute, 0);
   1644 
   1645 			push_compute_timing_info(ctx->compute_timing_table,
   1646 			                         (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd});
   1647 
   1648 			end_renderdoc_capture();
   1649 		}break;
   1650 		InvalidDefaultCase;
   1651 		}
   1652 	}
   1653 }
   1654 
   1655 function void
   1656 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats)
   1657 {
   1658 	/* TODO(rnp): we do not currently do anything to handle the potential for a half written
   1659 	 * info item. this could result in garbage entries but they shouldn't really matter */
   1660 
   1661 	u32 target = atomic_load_u32(&t->write_index);
   1662 	u32 stats_index = stats->latest_frame_index;
   1663 
   1664 	b32 has_rf = 0;
   1665 	f32 gpu_clocks_to_nano = 1.0e-9f * vk_gpu_info()->timestamp_period_ns;
   1666 
   1667 	// NOTE(rnp): not equal (the index may wrap)
   1668 	while (t->read_index != target) {
   1669 		ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)];
   1670 		switch (info.kind) {
   1671 
   1672 		case ComputeTimingInfoKind_ComputeFrameBegin:{
   1673 			assert(t->compute_frame_active == 0);
   1674 			t->compute_frame_active = 1;
   1675 			/* NOTE(rnp): allow multiple instances of same shader to accumulate */
   1676 			t->in_flight_shader_count = 0;
   1677 			memory_clear(t->in_flight_shader_ids, 0, sizeof(t->in_flight_shader_ids));
   1678 			memory_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index]));
   1679 		}break;
   1680 
   1681 		case ComputeTimingInfoKind_ComputeFrameEnd:{
   1682 			assert(t->compute_frame_active == 1);
   1683 			t->compute_frame_active = 0;
   1684 			stats_index = stats->latest_frame_index = (stats_index + 1) % countof(stats->table.times);
   1685 			stats->table.shader_count = t->in_flight_shader_count;
   1686 			memory_copy(stats->table.shader_ids, t->in_flight_shader_ids, sizeof(t->in_flight_shader_ids));
   1687 		}break;
   1688 
   1689 		case ComputeTimingInfoKind_Shader:{
   1690 			t->in_flight_shader_count = Max(t->in_flight_shader_count, info.shader_slot + 1u);
   1691 			t->in_flight_shader_ids[info.shader_slot] = info.shader;
   1692 			stats->table.times[stats_index][info.shader_slot] += info.timer_count * gpu_clocks_to_nano;
   1693 		}break;
   1694 
   1695 		case ComputeTimingInfoKind_RF_Data:{
   1696 			stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas);
   1697 			f32 delta = info.timer_count / (f32)os_system_info()->timer_frequency;
   1698 			stats->table.rf_time_deltas[stats->latest_rf_index] = delta;
   1699 			has_rf = 1;
   1700 		}break;
   1701 		}
   1702 		/* NOTE(rnp): do this at the end so that stats table is always in a consistent state */
   1703 		t->read_index++;
   1704 	}
   1705 
   1706 	for (u32 i = 0; i < stats->table.shader_count; i++) {
   1707 		f32 sum = 0;
   1708 		for EachElement(stats->table.times, it)
   1709 			sum += stats->table.times[it][i];
   1710 		stats->average_times[i] = sum / countof(stats->table.times);
   1711 	}
   1712 
   1713 	if (has_rf) {
   1714 		f32 sum = 0;
   1715 		for EachElement(stats->table.rf_time_deltas, i)
   1716 			sum += stats->table.rf_time_deltas[i];
   1717 		stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas);
   1718 	}
   1719 }
   1720 
   1721 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute)
   1722 {
   1723 	BeamformerSharedMemory *sm = ctx->shared_memory;
   1724 	complete_queue(ctx, &sm->external_work_queue, arena);
   1725 	complete_queue(ctx, ctx->beamform_work_queue, arena);
   1726 }
   1727 
   1728 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
   1729 {
   1730 	BeamformerSharedMemory *sm                  = ctx->shared_memory;
   1731 	BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace;
   1732 	BeamformerSharedMemoryLockKind upload_lock  = BeamformerSharedMemoryLockKind_UploadRF;
   1733 
   1734 	u64 rf_block_rf_size;
   1735 	if (atomic_load_u32(sm->locks + upload_lock) &&
   1736 	    (rf_block_rf_size = atomic_swap_u64(&sm->rf_block_rf_size, 0)))
   1737 	{
   1738 		beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)scratch_lock, (u32)-1);
   1739 
   1740 		BeamformerRFBuffer *rf = ctx->rf_buffer;
   1741 
   1742 		rf->active_rf_size = vk_round_up_to_sync_size(rf_block_rf_size & 0xFFFFFFFFULL, 64);
   1743 		if unlikely(rf->buffer.size < countof(rf->upload_complete_values) * rf->active_rf_size) {
   1744 			GPUBufferAllocateInfo allocate_info = {
   1745 				.size  = countof(rf->upload_complete_values) * rf->active_rf_size,
   1746 				.flags = VulkanUsageFlag_HostReadWrite,
   1747 				.label = str8("RawRFBuffer"),
   1748 			};
   1749 			vk_buffer_allocate(&rf->buffer, &allocate_info);
   1750 		}
   1751 
   1752 		u64 slot = rf->insertion_index % countof(rf->upload_complete_values);
   1753 
   1754 		/* NOTE(rnp): don't overwrite slot if the compute thread hasn't processed it */
   1755 		spin_wait(atomic_load_u64(&rf->compute_index) < rf->insertion_index);
   1756 		vk_host_wait_timeline(VulkanTimeline_Compute, rf->compute_complete_values[slot], -1ULL);
   1757 
   1758 		vk_buffer_range_upload(&rf->buffer, beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
   1759 		                       slot * rf->active_rf_size, rf->active_rf_size, 1);
   1760 		store_fence();
   1761 
   1762 		beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock);
   1763 		post_sync_barrier(ctx->shared_memory, upload_lock);
   1764 
   1765 		atomic_store_u64(rf->upload_complete_values + slot, vk_host_signal_timeline(VulkanTimeline_Transfer));
   1766 		atomic_add_u64(&rf->insertion_index, 1);
   1767 
   1768 		os_wake_all_waiters(ctx->compute_worker_sync);
   1769 
   1770 		u64 current_time = os_timer_count();
   1771 		push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){
   1772 			.kind        = ComputeTimingInfoKind_RF_Data,
   1773 			.timer_count = current_time - rf->timestamp,
   1774 		});
   1775 		rf->timestamp = current_time;
   1776 	}
   1777 }
   1778 
   1779 function void
   1780 beamformer_queue_compute(BeamformerCtx *ctx, BeamformerFrame *frame, u32 parameter_block)
   1781 {
   1782 	BeamformerSharedMemory *sm = ctx->shared_memory;
   1783 	BeamformerSharedMemoryLockKind dispatch_lock = BeamformerSharedMemoryLockKind_DispatchCompute;
   1784 	if (!sm->live_imaging_parameters.active && beamformer_shared_memory_take_lock(sm, (i32)dispatch_lock, 0))
   1785 	{
   1786 		BeamformWork *work = beamform_work_queue_push(ctx->beamform_work_queue);
   1787 		if (work) {
   1788 			work->kind = BeamformerWorkKind_Compute;
   1789 			work->compute_context.view_plane      = frame ? frame->view_plane_tag : 0;
   1790 			work->compute_context.parameter_block = parameter_block;
   1791 			beamform_work_queue_push_commit(ctx->beamform_work_queue);
   1792 		}
   1793 	}
   1794 	os_wake_all_waiters(&ctx->compute_worker.sync_variable);
   1795 }
   1796 
   1797 #include "ui.c"
   1798 
   1799 function void
   1800 beamformer_process_input_events(BeamformerCtx *ctx, BeamformerInput *input,
   1801                                 BeamformerInputEvent *events, u32 event_count)
   1802 {
   1803 	for (u32 index = 0; index < event_count; index++) {
   1804 		BeamformerInputEvent *event = events + index;
   1805 		switch (event->kind) {
   1806 
   1807 		// NOTE(rnp): ui will handle these
   1808 		case BeamformerInputEventKind_ButtonPress:
   1809 		case BeamformerInputEventKind_ButtonRelease:
   1810 		case BeamformerInputEventKind_MouseScroll:
   1811 		case BeamformerInputEventKind_WindowResize:
   1812 		{}break;
   1813 
   1814 		case BeamformerInputEventKind_ExecutableReload:{
   1815 			ui_init(ctx, ctx->ui_arena);
   1816 		}break;
   1817 
   1818 		case BeamformerInputEventKind_FileEvent:{
   1819 			BeamformerFileReloadContext *frc = event->file_watch_user_context;
   1820 			switch (frc->kind) {
   1821 			case BeamformerFileReloadKind_ComputeInternalShader:{
   1822 				// TODO(rnp): this could stall, better to push it onto compute once queue is better
   1823 				beamformer_reload_compute_pipeline(frc->shader_reload.pipeline, frc->shader_reload.shader, 0, ctx->arena);
   1824 			}break;
   1825 
   1826 			case BeamformerFileReloadKind_ComputeShader:{
   1827 				for EachElement(ctx->compute_context.compute_plans, block) {
   1828 					BeamformerComputePlan *cp = ctx->compute_context.compute_plans[block];
   1829 					for (u32 slot = 0; cp && slot < cp->pipeline.shader_count; slot++) {
   1830 						i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]];
   1831 						if (beamformer_reloadable_shader_kinds[shader_index] == frc->shader_reload.shader)
   1832 							atomic_or_u32(&cp->dirty_programs, 1 << slot);
   1833 					}
   1834 				}
   1835 
   1836 				// TODO(rnp): track latest parameter block
   1837 				if (ctx->latest_frame)
   1838 					beamformer_queue_compute(ctx, ctx->latest_frame, 0);
   1839 			}break;
   1840 
   1841 			case BeamformerFileReloadKind_RenderShader:{
   1842 				beamformer_reload_render_pipeline(frc->shader_reload.pipeline, frc->shader_reload.shader, ctx->arena);
   1843 				ctx->render_shader_updated = 1;
   1844 			}break;
   1845 
   1846 			InvalidDefaultCase;
   1847 			}
   1848 		}break;
   1849 
   1850 		InvalidDefaultCase;
   1851 		}
   1852 	}
   1853 }
   1854 
   1855 function void
   1856 beamformer_panel_group_insert_at(BeamformerUIPanel *group, BeamformerUIPanel *tab, u64 new_child_index)
   1857 {
   1858 	if (tab->parent) beamformer_ui_panel_unlink(tab);
   1859 	new_child_index = Min(new_child_index, group->child_count);
   1860 
   1861 	tab->parent = group;
   1862 	group->child_count++;
   1863 	if (group->kind == BeamformerPanelKind_TabGroup) group->u.tab_focus = tab;
   1864 
   1865 	BeamformerUIPanel *previous_sibling = new_child_index == 0 ? 0 : group->first_child;
   1866 	for (u64 child_index = 1; child_index < new_child_index; child_index++)
   1867 		previous_sibling = previous_sibling->next_sibling;
   1868 
   1869 	if (previous_sibling) {
   1870 		tab->previous_sibling = previous_sibling;
   1871 		tab->next_sibling     = previous_sibling->next_sibling;
   1872 		if (tab->next_sibling) tab->next_sibling->previous_sibling = tab;
   1873 		previous_sibling->next_sibling = tab;
   1874 		if (previous_sibling == group->last_child) group->last_child = tab;
   1875 	} else {
   1876 		DLLInsertFirst(0, group->first_child, group->last_child, tab, next_sibling, previous_sibling);
   1877 	}
   1878 }
   1879 
   1880 BEAMFORMER_EXPORT void
   1881 beamformer_frame_step(void *memory, BeamformerInput *input)
   1882 {
   1883 	BeamformerCtx *ctx = beamformer_context = memory;
   1884 	beamformer_input = input;
   1885 
   1886 	u64 current_time = os_timer_count();
   1887 	dt_for_frame = (f64)(current_time - ctx->frame_timestamp) / os_system_info()->timer_frequency;
   1888 	ctx->frame_timestamp = current_time;
   1889 	ctx->frame_index++;
   1890 
   1891 	coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats);
   1892 
   1893 	// NOTE(rnp): reset frame state
   1894 	{
   1895 		ctx->registers = &ctx->base_registers;
   1896 		swap(ctx->command_queues[0], ctx->command_queues[1]);
   1897 		zero_struct(ctx->command_queues + 0);
   1898 		//zero_struct(ctx->registers);
   1899 		arena_clear(beamformer_frame_arena());
   1900 	}
   1901 
   1902 	beamformer_process_input_events(ctx, input, input->event_queue, input->event_count);
   1903 
   1904 	BeamformerSharedMemory *sm = ctx->shared_memory;
   1905 	u32 live_imaging_active = atomic_load_u32(&sm->live_imaging_parameters.active);
   1906 	if (live_imaging_active != ctx->live_imaging_active) {
   1907 		if (ctx->live_imaging_active) {
   1908 			if (ctx->auto_live_control_panel) {
   1909 				BeamformerUIPanel *parent = ctx->auto_live_control_panel->parent;
   1910 				beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)ctx->auto_live_control_panel);
   1911 				if (parent->child_count == 1)
   1912 					beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent);
   1913 			}
   1914 		} else {
   1915 			if (beamformer_registers()->live_controls) {
   1916 				beamformer_command(beamformer_command_infos[BeamformerCommandKind_FocusTab].string,
   1917 				                   .tree_node = beamformer_registers()->live_controls);
   1918 			} else {
   1919 				ctx->auto_live_control_panel = beamformer_ui_push_panel(0, BeamformerPanelKind_LiveImagingControls);
   1920 				beamformer_command(beamformer_command_infos[BeamformerCommandKind_SplitTree].string,
   1921 				                   .tree_node        = (u64)ctx->auto_live_control_panel,
   1922 				                   .split_axis       = Axis2_X,
   1923 				                   .split_left_tree  = (u64)ui_context->tree,
   1924 				                   .split_right_tree = 0,
   1925 				                   .drop_target_tree = (u64)ui_context->tree);
   1926 			}
   1927 			ctx->live_imaging_active_frame = ctx->frame_index;
   1928 		}
   1929 		ctx->live_imaging_active = live_imaging_active;
   1930 	}
   1931 
   1932 	if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF))
   1933 		os_wake_all_waiters(&ctx->upload_worker.sync_variable);
   1934 	if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_DispatchCompute))
   1935 		os_wake_all_waiters(&ctx->compute_worker.sync_variable);
   1936 
   1937 	beamformer_registers()->frame = (u64)(ctx->latest_frame - ctx->compute_context.backlog.frames);
   1938 
   1939 	beamformer_ui_frame();
   1940 
   1941 	// NOTE(rnp): execute commands
   1942 	for (BeamformerCommandNode *node = ctx->command_queues[0].first;
   1943 	     node;
   1944 	     node = node == node->next ? 0 : node->next)
   1945 	{
   1946 		BeamformerRegistersScope()
   1947 		{
   1948 			memory_copy(beamformer_registers(), node->command.registers, sizeof(*node->command.registers));
   1949 			BeamformerCommandKind kind = beamformer_command_kind_from_string(node->command.name);
   1950 			switch (kind) {
   1951 			InvalidDefaultCase;
   1952 			case BeamformerCommandKind_CloseTab:{
   1953 				BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node;
   1954 				ui_kill_panel(tab);
   1955 			}break;
   1956 
   1957 			case BeamformerCommandKind_FocusTab:{
   1958 				BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node;
   1959 				assert(tab->parent->kind == BeamformerPanelKind_TabGroup);
   1960 				tab->parent->u.tab_focus = tab;
   1961 			}break;
   1962 
   1963 			case BeamformerCommandKind_MoveTab:{
   1964 				BeamformerUIPanel *move   = (BeamformerUIPanel *)beamformer_registers()->tree_node;
   1965 				BeamformerUIPanel *group  = (BeamformerUIPanel *)beamformer_registers()->drop_target_tree;
   1966 				BeamformerUIPanel *parent = move->parent;
   1967 				u64 new_child_index = beamformer_registers()->drop_child_index;
   1968 				beamformer_panel_group_insert_at(group, move, new_child_index);
   1969 
   1970 				if (move->kind == BeamformerPanelKind_LiveImagingControls) {
   1971 					beamformer_context->base_registers.v.live_controls = (u64)move;
   1972 					if (move == ctx->auto_live_control_panel)
   1973 						ctx->auto_live_control_panel = 0;
   1974 				}
   1975 
   1976 				if (parent->child_count == 0)
   1977 					beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent);
   1978 			}break;
   1979 
   1980 			case BeamformerCommandKind_OpenTab:{
   1981 				BeamformerUIPanel *panel = (BeamformerUIPanel *)beamformer_registers()->tree_node;
   1982 				assert(panel->kind == BeamformerPanelKind_TabGroup);
   1983 
   1984 				BeamformerPanelKind new_panel_kind = beamformer_panel_kind_from_string(beamformer_registers()->string);
   1985 				beamformer_ui_push_panel(panel, new_panel_kind);
   1986 			}break;
   1987 
   1988 			case BeamformerCommandKind_SplitTree:{
   1989 				BeamformerUIPanel *drag  = (BeamformerUIPanel *)beamformer_registers()->tree_node;
   1990 				BeamformerUIPanel *left  = (BeamformerUIPanel *)beamformer_registers()->split_left_tree;
   1991 				BeamformerUIPanel *right = (BeamformerUIPanel *)beamformer_registers()->split_right_tree;
   1992 				Axis2 axis = beamformer_registers()->split_axis;
   1993 
   1994 				BeamformerUIPanel *new_split     = beamformer_ui_push_panel(0, BeamformerPanelKind_Split);
   1995 				BeamformerUIPanel *new_tab_group = beamformer_ui_push_panel(0, BeamformerPanelKind_TabGroup);
   1996 				beamformer_panel_group_insert_at(new_tab_group, drag, 0);
   1997 
   1998 				BeamformerUIPanel *target = 0;
   1999 				u32 target_child_index = 0;
   2000 				f32 new_split_pct = 0.5f;
   2001 
   2002 				if (left == 0 || right == 0) {
   2003 					// NOTE(rnp): split on edge of window
   2004 					target             = left ? left : right;
   2005 					target_child_index = left ? 0 : 1;
   2006 
   2007 					if (target->kind == BeamformerPanelKind_TabGroup) {
   2008 						new_split->kind        = BeamformerPanelKind_TabGroup;
   2009 						new_split->u.tab_focus = target->u.tab_focus;
   2010 					}
   2011 
   2012 					for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) {
   2013 						next = child->previous_sibling;
   2014 						beamformer_panel_group_insert_at(new_split, child, 0);
   2015 					}
   2016 
   2017 					beamformer_panel_group_insert_at(target, new_tab_group, 0);
   2018 				} else if (((drag == left)  && right->kind == BeamformerPanelKind_Split) ||
   2019 				           ((drag == right) && left->kind  == BeamformerPanelKind_Split))
   2020 				{
   2021 					// NOTE(rnp): split on internal split
   2022 					target             = left == drag ? right : left;
   2023 					target_child_index = 1;
   2024 					new_split_pct      = 1.f / 3.f;
   2025 					beamformer_panel_group_insert_at(new_split, new_tab_group, 0);
   2026 					beamformer_panel_group_insert_at(new_split, target->last_child, 1);
   2027 				} else {
   2028 					// NOTE(rnp): TabGroup Split
   2029 					target             = left == drag ? right : left;
   2030 					target_child_index = left == drag ? 1 : 0;
   2031 					assert(target->kind == BeamformerPanelKind_TabGroup);
   2032 
   2033 					BeamformerUIPanel *focus = target->u.tab_focus;
   2034 					new_split->kind = BeamformerPanelKind_TabGroup;
   2035 					for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) {
   2036 						next = child->previous_sibling;
   2037 						beamformer_panel_group_insert_at(new_split, child, 0);
   2038 					}
   2039 					new_split->u.tab_focus = focus;
   2040 
   2041 					beamformer_panel_group_insert_at(target, new_tab_group, 0);
   2042 				}
   2043 
   2044 				beamformer_panel_group_insert_at(target, new_split, target_child_index);
   2045 				if (target->kind == BeamformerPanelKind_Split) {
   2046 					new_split->u.split.axis     = target->u.split.axis;
   2047 					new_split->u.split.fraction = target->u.split.fraction;
   2048 				}
   2049 				target->kind             = BeamformerPanelKind_Split;
   2050 				target->u.split.axis     = axis;
   2051 				target->u.split.fraction = new_split_pct;
   2052 			}break;
   2053 
   2054 			}
   2055 		}
   2056 	}
   2057 
   2058 	ctx->render_shader_updated = 0;
   2059 }