beamformer.c (13138B)
1 /* See LICENSE for license details. */ 2 3 #include "beamformer_internal.h" 4 5 /* NOTE(rnp): magic variables to force discrete GPU usage on laptops with multiple devices */ 6 EXPORT i32 NvOptimusEnablement = 1; 7 EXPORT i32 AmdPowerXpressRequestHighPerformance = 1; 8 9 #if !BEAMFORMER_DEBUG 10 #include "beamformer_core.c" 11 #else 12 13 typedef void beamformer_frame_step_fn(void *, BeamformerInput *); 14 15 #define BEAMFORMER_DEBUG_ENTRY_POINTS \ 16 X(beamformer_debug_ui_deinit) \ 17 X(beamformer_complete_compute) \ 18 X(beamformer_frame_step) \ 19 X(beamformer_rf_upload) \ 20 21 #define X(name) global name ##_fn *name; 22 BEAMFORMER_DEBUG_ENTRY_POINTS 23 #undef X 24 25 BEAMFORMER_EXPORT void 26 beamformer_debug_hot_release(void *memory, BeamformerInput *input) 27 { 28 BeamformerCtx *ctx = memory; 29 // TODO(rnp): this will deadlock if live imaging is active 30 /* NOTE(rnp): spin until compute thread finishes its work (we will probably 31 * never reload while compute is in progress but just incase). */ 32 spin_wait(atomic_load_u32(&ctx->upload_worker.awake)); 33 spin_wait(atomic_load_u32(&ctx->compute_worker.awake)); 34 } 35 36 BEAMFORMER_EXPORT void 37 beamformer_debug_hot_reload(OSLibrary library) 38 { 39 #define X(name) name = os_lookup_symbol(library, #name); 40 BEAMFORMER_DEBUG_ENTRY_POINTS 41 #undef X 42 43 str8 info = beamformer_info("reloaded main executable"); 44 os_console_log(info.data, info.length); 45 } 46 47 #endif /* BEAMFORMER_DEBUG */ 48 49 function no_return void 50 fatal(str8 message) 51 { 52 os_fatal(message.data, message.length); 53 unreachable(); 54 } 55 56 #include "vulkan.c" 57 58 // TODO(rnp): this doesn't belong here, but will be removed 59 // once vulkan migration is complete 60 void * glfwGetProcAddress(char *); 61 62 function void 63 gl_debug_logger(u32 src, u32 type, u32 id, u32 lvl, i32 len, const char *msg, const void *userctx) 64 { 65 Stream *e = (Stream *)userctx; 66 stream_append_str8s(e, str8("[OpenGL] "), (str8){.length = len, .data = (u8 *)msg}, str8("\n")); 67 os_console_log(e->data, e->widx); 68 stream_reset(e, 0); 69 } 70 71 function void 72 load_gl(Stream *err) 73 { 74 #define X(name, ret, params) name = (name##_fn *)glfwGetProcAddress(#name); 75 OGLProcedureList 76 OGLRequiredExtensionProcedureList 77 #undef X 78 79 stream_reset(err, 0); 80 #define X(name, ret, params) if (!name) stream_append_str8(err, str8("missing required GL function: " #name "\n")); 81 OGLProcedureList 82 OGLRequiredExtensionProcedureListBase 83 #if OS_WINDOWS 84 OGLRequiredExtensionProcedureListW32 85 #else 86 OGLRequiredExtensionProcedureListLinux 87 #endif 88 #undef X 89 90 if (err->widx) fatal(stream_to_str8(err)); 91 } 92 93 function void 94 beamformer_load_cuda_library(BeamformerCtx *ctx, OSLibrary cuda, Arena *scratch) 95 { 96 /* TODO(rnp): (25.10.30) registering the rf buffer with CUDA is currently 97 * causing a major performance regression. for now we are disabling its use 98 * altogether. it will be reenabled once the issue can be fixed */ 99 b32 result = 0 && vk_gpu_info()->vendor == GPUVendor_NVIDIA && ValidHandle(cuda); 100 if (result) { 101 Stream err = arena_stream(scratch); 102 103 stream_append_str8(&err, beamformer_info("loading CUDA library functions")); 104 #define X(name, symname) cuda_## name = os_lookup_symbol(cuda, symname); 105 CUDALibraryProcedureList 106 #undef X 107 108 os_console_log(err.data, err.widx); 109 } 110 111 #define X(name, symname) if (!cuda_## name) cuda_## name = cuda_ ## name ## _stub; 112 CUDALibraryProcedureList 113 #undef X 114 } 115 116 function void 117 worker_thread_sleep(GLWorkerThreadContext *ctx, BeamformerSharedMemory *sm) 118 { 119 for (;;) { 120 i32 expected = 0; 121 if (atomic_cas_u32(&ctx->sync_variable, &expected, 1) || 122 atomic_load_u32(&sm->live_imaging_parameters.active)) 123 { 124 break; 125 } 126 127 /* TODO(rnp): clean this crap up; we shouldn't need two values to communicate this */ 128 atomic_store_u32(&ctx->awake, 0); 129 os_wait_on_address(&ctx->sync_variable, 1, (u32)-1); 130 atomic_store_u32(&ctx->awake, 1); 131 } 132 } 133 134 function OS_THREAD_ENTRY_POINT_FN(compute_worker_thread_entry_point) 135 { 136 GLWorkerThreadContext *ctx = user_context; 137 138 BeamformerCtx *beamformer = (BeamformerCtx *)ctx->user_context; 139 140 for (;;) { 141 worker_thread_sleep(ctx, beamformer->shared_memory); 142 beamformer_complete_compute(beamformer, ctx->arena); 143 } 144 145 unreachable(); 146 147 return 0; 148 } 149 150 function OS_THREAD_ENTRY_POINT_FN(beamformer_upload_entry_point) 151 { 152 GLWorkerThreadContext *ctx = user_context; 153 BeamformerUploadThreadContext *up = (typeof(up))ctx->user_context; 154 155 for (;;) { 156 worker_thread_sleep(ctx, up->shared_memory); 157 beamformer_rf_upload(up); 158 } 159 160 unreachable(); 161 162 return 0; 163 } 164 165 BEAMFORMER_EXPORT void * 166 beamformer_init(BeamformerInput *input) 167 { 168 Arena *memory = arena_create(.name = "Beamformer Memory"); 169 Stream error = stream_alloc(memory, MB(1)); 170 BeamformerCtx *ctx = push_struct(memory, BeamformerCtx); 171 172 for EachElement(ctx->frame_arenas, it) 173 ctx->frame_arenas[it] = arena_create(); 174 175 str8 window_title = str8("VK Beamformer"); 176 ctx->main_window = os_window_create(window_title.data, window_title.length, 1280, 840); 177 ctx->window_size = (iv2){{1280, 840}}; 178 179 ctx->arena = memory; 180 ctx->error_stream = error; 181 ctx->ui_arena = arena_create(); 182 ctx->compute_worker.arena = arena_create(); 183 ctx->upload_worker.arena = arena_create(); 184 185 #if BEAMFORMER_RENDERDOC_HOOKS 186 start_frame_capture = input->renderdoc_start_frame_capture; 187 end_frame_capture = input->renderdoc_end_frame_capture; 188 set_capture_path_template = input->renderdoc_set_capture_file_path_template; 189 #endif 190 191 vk_load(input->vulkan_library_handle, &ctx->error_stream); 192 193 BeamformerComputeContext *cs = &ctx->compute_context; 194 195 // NOTE(rnp): allocate beamformed image ring buffer 196 { 197 u64 gpu_heap_size = vk_gpu_info()->gpu_heap_size; 198 u64 trial_sizes[] = { 199 GB(4), 200 GB(2), 201 GB(1) + MB(512), 202 GB(1), 203 }; 204 205 u32 base_index = 0; 206 for EachElement(trial_sizes, it) { 207 if (gpu_heap_size >= 2 * trial_sizes[it]) 208 break; 209 base_index++; 210 } 211 212 for (u32 i = base_index; i < countof(trial_sizes); i++) { 213 // TODO(rnp): it may be better to download data from this using the transfer queue 214 VulkanTimeline timelines[] = {VulkanTimeline_Compute, VulkanTimeline_Graphics}; 215 GPUBufferAllocateInfo allocate_info = { 216 .size = trial_sizes[i], 217 .flags = VulkanUsageFlag_TransferDestination|VulkanUsageFlag_TransferSource|VulkanUsageFlag_HostReadWrite, 218 .timeline_count = countof(timelines), 219 .timelines_used = timelines, 220 .label = str8("BeamformedData"), 221 }; 222 vk_buffer_allocate(cs->backlog.buffer, &allocate_info); 223 if (cs->backlog.buffer->size > 0) 224 break; 225 } 226 if (cs->backlog.buffer->size == 0) { 227 // NOTE(rnp): if this becomes an issue we may be able to get by in some other way 228 fatal(str8("Failed to allocate space for beamformed data\n")); 229 } 230 231 BeamformerShaderResourceInfo shader_resource_infos[] = { 232 { 233 .kind = BeamformerShaderResourceKind_Buffer, 234 .handle = cs->backlog.buffer->handle, 235 .slot = BeamformerShaderBufferSlot_BeamformedData, 236 }, 237 }; 238 vk_bind_shader_resources(shader_resource_infos, countof(shader_resource_infos)); 239 } 240 241 Arena *scratch = arena_create(); 242 beamformer_load_cuda_library(ctx, input->cuda_library_handle, scratch); 243 244 load_gl(&ctx->error_stream); 245 246 ctx->shared_memory = input->shared_memory; 247 ctx->shared_memory_size = input->shared_memory_size; 248 if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory)) 249 fatal(str8("Get more ram lol\n")); 250 zero_struct(ctx->shared_memory); 251 252 ctx->shared_memory->version = BEAMFORMER_SHARED_MEMORY_VERSION; 253 ctx->shared_memory->reserved_parameter_blocks = 1; 254 255 ctx->shared_memory->beamformed_frame_buffer_size = cs->backlog.buffer->size; 256 257 // TODO(rnp): dynamic rf data buffer slot usage 258 // NOTE(rnp): will be same as the max size we were able to get for the frame buffer 259 ctx->shared_memory->capabilities.max_rf_data_size = cs->backlog.buffer->size 260 / BeamformerMaxRawDataFramesInFlight; 261 262 ctx->shared_memory->capabilities.cuda = cuda_supported(); 263 // TODO(rnp): re-enable hilbert support, with and without cuda 264 ctx->shared_memory->capabilities.hilbert = 0; 265 266 /* TODO(rnp): I'm not sure if its a good idea to pre-reserve a bunch of semaphores 267 * on w32 but thats what we are doing for now */ 268 #if OS_WINDOWS 269 { 270 Stream sb = arena_stream(memory); 271 stream_append(&sb, input->shared_memory_name, input->shared_memory_name_length); 272 stream_append_str8(&sb, str8("_lock_")); 273 i32 start_index = sb.widx; 274 for EachElement(os_w32_shared_memory_semaphores, it) { 275 stream_reset(&sb, start_index); 276 stream_append_u64(&sb, it); 277 stream_append_byte(&sb, 0); 278 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 279 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 280 fatal(beamformer_info("init: failed to create w32 shared memory semaphore\n")); 281 282 /* NOTE(rnp): hacky garbage because CreateSemaphore will just open an existing 283 * semaphore without any indication. Sometimes the other side of the shared memory 284 * will provide incorrect parameters or will otherwise fail and its faster to 285 * restart this program than to get that application to release the semaphores */ 286 /* TODO(rnp): figure out something more robust */ 287 os_w32_semaphore_release(os_w32_shared_memory_semaphores[it], 1); 288 } 289 } 290 #endif 291 292 GLWorkerThreadContext *worker = &ctx->compute_worker; 293 /* TODO(rnp): we should lock this down after we have something working */ 294 worker->user_context = (iptr)ctx; 295 worker->handle = os_create_thread("[compute]", worker, compute_worker_thread_entry_point); 296 297 GLWorkerThreadContext *upload = &ctx->upload_worker; 298 BeamformerUploadThreadContext *upctx = push_struct(memory, typeof(*upctx)); 299 upload->user_context = (iptr)upctx; 300 upctx->rf_buffer = &cs->rf_buffer; 301 upctx->shared_memory = ctx->shared_memory; 302 upctx->shared_memory_size = ctx->shared_memory_size; 303 upctx->compute_timing_table = ctx->compute_timing_table; 304 upctx->compute_worker_sync = &ctx->compute_worker.sync_variable; 305 upload->handle = os_create_thread("[upload]", upload, beamformer_upload_entry_point); 306 307 /* NOTE: set up OpenGL debug logging */ 308 Stream *gl_error_stream = push_struct(memory, Stream); 309 *gl_error_stream = stream_alloc(memory, 1024); 310 glDebugMessageCallback(gl_debug_logger, gl_error_stream); 311 #ifdef BEAMFORMER_DEBUG 312 glEnable(GL_DEBUG_OUTPUT); 313 #endif 314 315 if (!BakeShaders) 316 { 317 for EachElement(beamformer_reloadable_compute_shader_info_indices, it) { 318 i32 index = beamformer_reloadable_compute_shader_info_indices[it]; 319 320 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 321 beamformer_reloadable_shader_files[index][0]); 322 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 323 frc->kind = BeamformerFileReloadKind_ComputeShader; 324 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 325 os_add_file_watch((char *)file.data, file.length, frc); 326 } 327 328 for EachElement(beamformer_reloadable_compute_helpers_shader_info_indices, it) { 329 i32 index = beamformer_reloadable_compute_helpers_shader_info_indices[it]; 330 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 331 beamformer_reloadable_shader_files[index][0]); 332 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 333 frc->kind = BeamformerFileReloadKind_ComputeShader; 334 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 335 os_add_file_watch((char *)file.data, file.length, frc); 336 } 337 } 338 339 arena_destroy(scratch); 340 341 ctx->state = BeamformerState_Running; 342 343 return ctx; 344 } 345 346 BEAMFORMER_EXPORT void 347 beamformer_terminate(void *memory, BeamformerInput *input) 348 { 349 /* NOTE(rnp): work around pebkac when the beamformer is closed while we are doing live 350 * imaging. if the verasonics is blocked in an external function (calling the library 351 * to start compute) it is impossible for us to get it to properly shut down which 352 * will sometimes result in us needing to power cycle the system. set the shared memory 353 * into an error state and release dispatch lock so that future calls will error instead 354 * of blocking. 355 */ 356 BeamformerCtx *ctx = memory; 357 BeamformerSharedMemory *sm = input->shared_memory; 358 if (ctx->state != BeamformerState_Terminated) { 359 if (sm) { 360 BeamformerSharedMemoryLockKind lock = BeamformerSharedMemoryLockKind_DispatchCompute; 361 atomic_store_u32(&sm->invalid, 1); 362 atomic_store_u32(&sm->external_work_queue.ridx, sm->external_work_queue.widx); 363 DEBUG_DECL(if (sm->locks[lock])) { 364 beamformer_shared_memory_release_lock(sm, (i32)lock); 365 } 366 367 atomic_or_u32(&sm->live_imaging_dirty_flags, BeamformerLiveImagingDirtyFlags_StopImaging); 368 } 369 370 beamformer_debug_ui_deinit(ctx); 371 372 ctx->state = BeamformerState_Terminated; 373 } 374 } 375 376 BEAMFORMER_EXPORT u32 377 beamformer_should_close(void *memory, BeamformerInput *input) 378 { 379 BeamformerCtx *ctx = memory; 380 if (ctx->state == BeamformerState_ShouldClose) 381 beamformer_terminate(memory, input); 382 return ctx->state == BeamformerState_Terminated; 383 }