58 explicit ComputeWindow(
const Arguments &args) :
mxvk::
VK_Window(
"-[ VK Compute CV ]-", args.width, args.height, args.fullscreen,
MXVK_VALIDATION, args.enable_vsync), assetRoot(args.path.empty() ? std::string(
compute_shader_ASSET_DIR) : args.path), inputFilename(args.filename), usingFile(!inputFilename.empty()), fastMode(args.fast), explicitResolution(args.resolutionSpecified), fullscreenMode(args.fullscreen), outputFilename(args.output), outputCrf(args.crf), encodePreset(args.encodePreset), encodeTune(args.encodeTune), encodeCodec(args.encodeCodec), encodeRealtime(args.encodeRealtime), mxwriteBlockWhenFull(args.mxwriteBlockWhenFull), repeat(args.repeat), cameraIndex(args.camera_index), requestedShaderIndex(args.shader_index), initialShaderMode(args.index > 0 ? std::clamp(args.index - 1, 0, static_cast<int>(ACIDCAM_FILTER_MODE_NAMES.size()) - 1) : 0) {
59 recordWidth = args.
width;
60 recordHeight = args.
height;
61 shaderMode = initialShaderMode;
62 initComputeResources();
67#if defined(MXVK_WITH_FFMPEG_CAPTURE)
70 destroyComputeResources();
74 bool frameUploaded =
false;
75 if (usingFile && !fastMode) {
76 throttleVideoPlayback();
79#if defined(MXVK_WITH_FFMPEG_CAPTURE)
81 frameUploaded = readFfFrameToCompute();
82 if (!frameUploaded && usingFile) {
85 frameUploaded = readFfFrameToCompute();
87 std::cout <<
"compute_shader: video file reached EOF, shutting down\n";
97 cv::cuda::GpuMat gpuFrame;
98 if (capture.readGpuRgba(gpuFrame) && !gpuFrame.empty()) {
99 frameUploaded = uploadGpuFrameToCompute(gpuFrame, capture.cudaStream());
100 if (frameUploaded && !captureUploadPathLogged) {
101 std::cout <<
"compute_shader: CUDA interop capture path active: capture -> GpuMat -> optional CUDA resize -> cudaMemcpy2DToArrayAsync -> Vulkan compute storage image (no download)\n";
102 captureUploadPathLogged =
true;
106 if (!frameUploaded && capture.readRgba(frame) && !frame.empty()) {
107 if (!captureUploadPathLogged) {
109 std::cout <<
"compute_shader: CUDA interop unavailable; fallback path active: CUDA/CPU RGBA -> optional CPU resize -> Vulkan staging upload\n";
111 std::cout <<
"compute_shader: CPU capture path active: readRgba converts to RGBA, optional CPU resize, then uploads through Vulkan staging\n";
113 captureUploadPathLogged =
true;
115 uploadCpuFrameToCompute(frame);
116 frameUploaded =
true;
117 }
else if (!frameUploaded && usingFile) {
120 if (capture.open(inputFilename)) {
121 configureVideoPlaybackRate();
122 cv::Mat restartedFrame;
123 if (capture.readRgba(restartedFrame) && !restartedFrame.empty()) {
124 if (!captureUploadPathLogged) {
126 std::cout <<
"compute_shader: CUDA interop unavailable; fallback path active: CUDA/CPU RGBA -> optional CPU resize -> Vulkan staging upload\n";
128 std::cout <<
"compute_shader: CPU capture path active: readRgba converts to RGBA, optional CPU resize, then uploads through Vulkan staging\n";
130 captureUploadPathLogged =
true;
132 uploadCpuFrameToCompute(restartedFrame);
133 frameUploaded =
true;
137 std::cout <<
"compute_shader: video file reached EOF, shutting down\n";
147 recordProcessedFrame();
149 updateFpsOverlay(frameUploaded);
154 void onRecordCustomRendering(VkCommandBuffer cmd, [[maybe_unused]] uint32_t imageIndex)
override { renderComputeOutput(cmd); }
157 if (e.type == SDL_EVENT_QUIT || (e.type == SDL_EVENT_KEY_DOWN && e.key.key == SDLK_ESCAPE)) {
162 if (e.type == SDL_EVENT_KEY_DOWN && !spvFiles.empty()) {
163 if (e.key.key == SDLK_UP) {
164 currentSpvIndex = (currentSpvIndex - 1 +
static_cast<int>(spvFiles.size())) %
static_cast<int>(spvFiles.size());
165 std::cout <<
"Current index: " << spvFiles[currentSpvIndex] <<
"\n";
167 }
else if (e.key.key == SDLK_DOWN) {
168 currentSpvIndex = (currentSpvIndex + 1) %
static_cast<int>(spvFiles.size());
169 std::cout <<
"Current index: " << spvFiles[currentSpvIndex] <<
"\n";
171 }
else if (spvFiles[currentSpvIndex] == MODE_SHADER_NAME && (e.key.key == SDLK_LEFT || e.key.key == SDLK_RIGHT)) {
172 const int delta = (e.key.key == SDLK_LEFT) ? -1 : 1;
173 shaderMode = (shaderMode + delta + 50) % 50;
174 std::cout <<
"Mode shader mode: " << shaderMode <<
"\n";
180 struct ComputeImage {
181 VkImage image = VK_NULL_HANDLE;
182 VkDeviceMemory memory = VK_NULL_HANDLE;
183 VkImageView view = VK_NULL_HANDLE;
185 VkDeviceSize cudaExportMemorySize = 0;
186 cudaExternalMemory_t cudaExternalMemory =
nullptr;
187 cudaMipmappedArray_t cudaMipmappedArray =
nullptr;
188 cudaArray_t cudaArray =
nullptr;
189 bool cudaInteropEnabled =
false;
190 bool cudaInteropUnavailableLogged =
false;
191 bool cudaUploadLogged =
false;
192 bool cudaBarrierLogged =
false;
196 std::string assetRoot;
197 mxvk::VK_Capture capture{};
198#if defined(MXVK_WITH_FFMPEG_CAPTURE)
199 mxvk::VK_FF_Capture ffCapture{};
200 bool usingFfCapture =
false;
201 std::vector<uint8_t> ffFrameRgba{};
202 int ffFramePitch = 0;
204 cv::cuda::Stream ffCudaStream{};
207 std::string inputFilename;
208 bool usingFile =
false;
209 bool fastMode =
false;
210 bool explicitResolution =
false;
211 bool fullscreenMode =
false;
212 int recordWidth = 1920;
213 int recordHeight = 1080;
214 int sourceWidth = 1920;
215 int sourceHeight = 1080;
216 std::string outputFilename;
217 std::string outputCrf;
218 std::string encodePreset;
219 std::string encodeTune;
220 std::string encodeCodec;
221 bool encodeRealtime =
false;
222 bool mxwriteBlockWhenFull =
false;
224 mxvk::Font fpsFont{};
227 int texHeight = 1080;
229 std::array<ComputeImage, 2> workImg{};
230 std::array<ComputeImage, HISTORY_SIZE> histImg{};
231 ComputeImage outImg{};
233 VkSampler computeSampler = VK_NULL_HANDLE;
235 VkBuffer stagingBuf = VK_NULL_HANDLE;
236 VkDeviceMemory stagingMem = VK_NULL_HANDLE;
237 VkBuffer readbackBuf = VK_NULL_HANDLE;
238 VkDeviceMemory readbackMem = VK_NULL_HANDLE;
240 VkDescriptorSetLayout compDSLayout = VK_NULL_HANDLE;
241 VkPipelineLayout compPipeLayout = VK_NULL_HANDLE;
242 VkPipeline compPipeline = VK_NULL_HANDLE;
243 VkDescriptorPool compDSPool = VK_NULL_HANDLE;
245 std::array<VkDescriptorSet, 2> blurDS{};
246 std::array<VkDescriptorSet, 2> blendDS{};
248 VkDescriptorSetLayout displayDSLayout = VK_NULL_HANDLE;
249 VkDescriptorPool displayDSPool = VK_NULL_HANDLE;
250 VkDescriptorSet displayDS = VK_NULL_HANDLE;
251 VkPipelineLayout displayPipeLayout = VK_NULL_HANDLE;
252 VkPipeline displayPipeline = VK_NULL_HANDLE;
253 VkBuffer displayVertexBuffer = VK_NULL_HANDLE;
254 VkDeviceMemory displayVertexMemory = VK_NULL_HANDLE;
255 VkBuffer displayIndexBuffer = VK_NULL_HANDLE;
256 VkDeviceMemory displayIndexMemory = VK_NULL_HANDLE;
258 int historyIndex = 0;
259 int historyCount = 0;
260 int currentSquare = 4;
262 int currentHistIdx = 0;
264 int requestedShaderIndex = 0;
266 int initialShaderMode = 0;
268 bool captureUploadPathLogged =
false;
269 double videoFps = 0.0;
270 double sourceFps = 0.0;
271 std::chrono::duration<double> videoFrameInterval{0.0};
272 std::chrono::steady_clock::time_point nextVideoFrameDeadline{std::chrono::steady_clock::now()};
273 double currentFps = 0.0;
274 uint32_t fpsFrameCount = 0;
275 std::chrono::steady_clock::time_point fpsSampleTime{std::chrono::steady_clock::now()};
276 std::chrono::steady_clock::time_point playbackStartTime{std::chrono::steady_clock::now()};
277 std::string fpsText =
"FPS: --";
278 uint64_t processedVideoFrames = 0;
279 bool recordingEnabled =
false;
280 [[maybe_unused]]
bool recordingWarningLogged =
false;
281 bool processedRecordPathLogged =
false;
282 std::vector<uint8_t> recordScratch{};
284 cv::cuda::Stream processedRecordStream{};
285 cv::cuda::GpuMat processedRecordGpuFrame{};
286 cv::cuda::GpuMat computeInputGpuFrame{};
288#if defined(MXWRITE_ENABLED)
289 Writer videoWriter{};
290 bool videoWriterOpen =
false;
293 std::vector<std::string> spvFiles{};
294 int currentSpvIndex = 0;
297 std::ifstream file(assetRoot +
"/data/index.txt");
298 if (!file.is_open()) {
299 throw mxvk::Exception(
"Cannot open: " + assetRoot +
"/data/index.txt");
303 while (std::getline(file, line)) {
305 spvFiles.push_back(line);
309 if (spvFiles.empty()) {
310 throw mxvk::Exception(
"index.txt contains no entries");
313 currentSpvIndex = std::clamp(requestedShaderIndex, 0,
static_cast<int>(spvFiles.size()) - 1);
316 [[nodiscard]]
double configureCameraFps() {
317 static constexpr std::array<double, 3> fpsChoices = {60.0, 30.0, 24.0};
319 for (
const double requestedFps : fpsChoices) {
320 capture.set(cv::CAP_PROP_FPS, requestedFps);
321 const double reportedFps = capture.get(cv::CAP_PROP_FPS);
322 if (reportedFps > 0.0 && reportedFps + 0.5 >= requestedFps) {
327 capture.set(cv::CAP_PROP_FPS, fpsChoices.back());
328 const double reportedFps = capture.get(cv::CAP_PROP_FPS);
329 return (reportedFps > 0.0) ? reportedFps : fpsChoices.back();
332 void resetVideoPlaybackClock() { nextVideoFrameDeadline = std::chrono::steady_clock::now(); }
334 void configureVideoPlaybackRate() {
335 double reportedFps = 0.0;
336#if defined(MXVK_WITH_FFMPEG_CAPTURE)
337 if (usingFfCapture) {
338 reportedFps = ffCapture.
fps();
342 reportedFps = capture.get(cv::CAP_PROP_FPS);
344 videoFps = (reportedFps > 0.0) ? reportedFps : 30.0;
345 sourceFps = videoFps;
346 if (videoFps <= 0.0) {
348 sourceFps = videoFps;
350 videoFrameInterval = std::chrono::duration<double>(1.0 / videoFps);
351 resetVideoPlaybackClock();
352 std::cout <<
"compute_shader: video file FPS " << videoFps <<
" fps, fast=" << (fastMode ?
"true" :
"false") <<
"\n";
355 bool openVideoSource() {
356#if defined(MXVK_WITH_FFMPEG_CAPTURE)
357 usingFfCapture =
false;
358 if (ffCapture.
open(inputFilename)) {
359 usingFfCapture =
true;
360 std::cout <<
"compute_shader: FFmpeg capture path active for file input" << (ffCapture.
using_hardware_decode() ?
" (CUDA decode)\n" :
" (software decode)\n");
363 std::cout <<
"compute_shader: FFmpeg capture failed; falling back to VK_Capture/OpenCV file input\n";
365 return capture.open(inputFilename);
368 void uploadCpuFrameToCompute(
const cv::Mat &frame) {
372 if (frame.cols == texWidth && frame.rows == texHeight) {
373 uploadToImage(frame.ptr(),
static_cast<int>(frame.step), workImg[0]);
377 cv::Mat resizedFrame;
378 cv::resize(frame, resizedFrame, cv::Size(texWidth, texHeight), 0.0, 0.0, cv::INTER_LINEAR);
379 uploadToImage(resizedFrame.ptr(),
static_cast<int>(resizedFrame.step), workImg[0]);
383 bool uploadGpuFrameToCompute(
const cv::cuda::GpuMat &gpuFrame, cv::cuda::Stream &stream) {
384 if (gpuFrame.empty()) {
387 if (gpuFrame.cols == texWidth && gpuFrame.rows == texHeight) {
388 return uploadGpuToImage(gpuFrame, stream, workImg[0]);
391 cv::cuda::resize(gpuFrame, computeInputGpuFrame, cv::Size(texWidth, texHeight), 0.0, 0.0, cv::INTER_LINEAR, stream);
392 stream.waitForCompletion();
393 return uploadGpuToImage(computeInputGpuFrame, stream, workImg[0]);
397#if defined(MXVK_WITH_FFMPEG_CAPTURE)
398 void restartFfCapture() {
400 if (ffCapture.
open(inputFilename)) {
401 usingFfCapture =
true;
402 configureVideoPlaybackRate();
406 bool readFfFrameToCompute() {
409 cv::cuda::GpuMat gpuFrame;
410 if (ffCapture.readGpuRgba(gpuFrame, ffCudaStream) && !gpuFrame.empty()) {
411 const bool uploadedWithInterop = uploadGpuFrameToCompute(gpuFrame, ffCudaStream);
412 if (uploadedWithInterop) {
413 if (!captureUploadPathLogged) {
414 std::cout <<
"compute_shader: FFmpeg CUDA path active: NVDEC/CUDA decode -> CUDA NV12/RGBA conversion -> optional CUDA resize -> cudaMemcpy2DToArrayAsync -> Vulkan compute storage image (no CPU download)\n";
415 captureUploadPathLogged =
true;
421 gpuFrame.download(cpuFrame, ffCudaStream);
422 ffCudaStream.waitForCompletion();
423 if (!cpuFrame.empty()) {
424 if (!captureUploadPathLogged) {
425 std::cout <<
"compute_shader: FFmpeg CUDA decode active, Vulkan CUDA interop unavailable; downloading RGBA for optional CPU resize and staging upload\n";
426 captureUploadPathLogged =
true;
428 uploadCpuFrameToCompute(cpuFrame);
436 if (!ffCapture.
readRgba(ffFrameRgba, frameWidth, frameHeight, ffFramePitch) || ffFrameRgba.empty()) {
439 if (frameWidth <= 0 || frameHeight <= 0) {
442 if (!captureUploadPathLogged) {
443 std::cout <<
"compute_shader: FFmpeg capture path active: decoded RGBA -> optional CPU resize -> Vulkan staging upload\n";
444 captureUploadPathLogged =
true;
446 cv::Mat frame(frameHeight, frameWidth, CV_8UC4, ffFrameRgba.data(),
static_cast<size_t>(ffFramePitch));
447 uploadCpuFrameToCompute(frame);
452 void configureRecordingDefaults() {
453 if (outputFilename.empty()) {
454 outputFilename = assetRoot +
"/compute_shader_output.mp4";
456 if (outputCrf.empty()) {
461 [[nodiscard]]
int overlayFontSizeForCanvas()
const {
462 const int canvasMinDim = std::min(texWidth, texHeight);
463 return std::clamp(canvasMinDim / 30, 8, 36);
466 void maybeResizeWindowToSource() {
467 if (usingFile && !explicitResolution && !fullscreenMode &&
getSDLWindow() !=
nullptr) {
469 std::cout <<
"compute_shader: window resized to source frame size " << texWidth <<
"x" << texHeight <<
" (pass -r/--resolution to override)\n";
473#if defined(MXWRITE_ENABLED)
474 [[nodiscard]]
int parseCrf()
const {
476 size_t parsedChars = 0;
477 const int value = std::stoi(outputCrf, &parsedChars);
478 if (parsedChars == outputCrf.size() && value >= 0 && value <= 51) {
481 }
catch (
const std::exception &) {
483 throw mxvk::Exception(
"compute_shader: invalid CRF '" + outputCrf +
"'; expected integer 0..51");
486 void openVideoWriter() {
487 if (videoWriterOpen) {
490 if (sourceFps <= 0.0) {
491 sourceFps = usingFile ? videoFps : 30.0;
493 if (sourceFps <= 0.0) {
496 EncodeOptions encodeOptions{};
497 encodeOptions.
crf = parseCrf();
498 if (!encodePreset.empty()) {
499 encodeOptions.
preset = encodePreset;
501 if (!encodeTune.empty()) {
502 encodeOptions.
tune = encodeTune;
504 if (!encodeCodec.empty()) {
505 encodeOptions.
codec = encodeCodec;
507 encodeOptions.
realtime = encodeRealtime;
510 if (!videoWriter.
open(outputFilename, recordWidth, recordHeight,
static_cast<float>(sourceFps), encodeOptions)) {
511 throw mxvk::Exception(
"compute_shader: failed to open MXWrite output file '" + outputFilename +
"'");
513 videoWriterOpen =
true;
514 std::cout <<
"compute_shader: recording to " << outputFilename <<
" at " << sourceFps <<
" fps with crf " << encodeOptions.
crf <<
", codec=" << encodeOptions.
codec <<
", preset=" << encodeOptions.
preset <<
", tune=" << (encodeOptions.
tune.empty() ?
"none" : encodeOptions.
tune) <<
", realtime=" << (encodeOptions.
realtime ?
"true" :
"false") <<
", block_when_full=" << (mxwriteBlockWhenFull ?
"true" :
"false") <<
"\n";
517 void recordFrame(
const cv::Mat &frame) {
518 if (videoWriterOpen && !frame.empty()) {
519 recordFrame(frame.ptr(), frame.cols, frame.rows,
static_cast<int>(frame.step));
523 void recordFrame(
const uint8_t *data,
int width,
int height,
int pitch) {
524 if (!videoWriterOpen || data ==
nullptr || width != recordWidth || height != recordHeight) {
528 const int tightPitch = recordWidth * 4;
529 if (pitch == tightPitch) {
530 videoWriter.
write(
const_cast<uint8_t *
>(data));
534 if (pitch < tightPitch) {
538 const int recordTightPitch = recordWidth * 4;
539 recordScratch.resize(
static_cast<size_t>(recordTightPitch) *
static_cast<size_t>(recordHeight));
540 for (
int row = 0; row < recordHeight; ++row) {
541 std::memcpy(recordScratch.data() +
static_cast<size_t>(row) *
static_cast<size_t>(tightPitch), data +
static_cast<size_t>(row) *
static_cast<size_t>(pitch),
static_cast<size_t>(recordTightPitch));
543 videoWriter.
write(recordScratch.data());
547 void recordGpuFrame(cv::cuda::GpuMat &gpuFrame) {
548 if (!videoWriterOpen || gpuFrame.empty()) {
551#if defined(MXWRITE_HAS_CUDA_COPY)
557 gpuFrame.download(cpuFrame);
558 recordFrame(cpuFrame);
562 void openVideoWriter() {
563 if (!recordingEnabled) {
566 if (!recordingWarningLogged) {
567 std::cout <<
"compute_shader: MXWrite is unavailable; video recording disabled\n";
568 recordingWarningLogged =
true;
570 recordingEnabled =
false;
573 void recordFrame(
const cv::Mat &) {}
574 void recordFrame(
const uint8_t *,
int,
int,
int) {}
576 void recordGpuFrame(cv::cuda::GpuMat &) {}
581 bool recordProcessedFrameCuda() {
582 if (!recordingEnabled || !ensureCudaInterop(outImg)) {
586 const VkCommandBuffer cmd = beginSingleTimeCommands();
587 transitionImageLayout(cmd, outImg.image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT);
588 endSingleTimeCommands(cmd);
590 processedRecordGpuFrame.create(texHeight, texWidth, CV_8UC4);
591 cudaStream_t cudaStream = mxvk::cuda_stream_handle(processedRecordStream);
592 cudaError_t cudaResult = cudaMemcpy2DFromArrayAsync(processedRecordGpuFrame.ptr(), processedRecordGpuFrame.step, outImg.cudaArray, 0, 0,
static_cast<size_t>(texWidth) * 4U,
static_cast<size_t>(texHeight), cudaMemcpyDeviceToDevice, cudaStream);
593 if (cudaResult != cudaSuccess) {
594 std::cout <<
"compute_shader: CUDA processed-frame readback failed: " << cudaGetErrorString(cudaResult) <<
"\n";
598 cudaResult = cudaStreamSynchronize(cudaStream);
599 if (cudaResult != cudaSuccess) {
600 std::cout <<
"compute_shader: CUDA processed-frame readback sync failed: " << cudaGetErrorString(cudaResult) <<
"\n";
604 if (!processedRecordPathLogged) {
605 std::cout <<
"compute_shader: processed recording path active: Vulkan compute output image -> CUDA array -> RGBA GpuMat -> MXWrite"
606#if defined(MXWRITE_HAS_CUDA_COPY)
607 <<
" CUDA ingestion when hardware encode is active"
609 <<
" CPU fallback when MXWrite CUDA ingestion is unavailable"
612 processedRecordPathLogged =
true;
614 recordGpuFrame(processedRecordGpuFrame);
619 void recordProcessedFrame() {
620 if (!recordingEnabled || readbackBuf == VK_NULL_HANDLE || readbackMem == VK_NULL_HANDLE) {
625 if (recordProcessedFrameCuda()) {
630 const int tightPitch = texWidth * 4;
631 const VkDeviceSize bytes =
static_cast<VkDeviceSize
>(tightPitch) *
static_cast<VkDeviceSize
>(texHeight);
632 const VkCommandBuffer cmd = beginSingleTimeCommands();
634 transitionImageLayout(cmd, outImg.image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
636 VkBufferImageCopy2 region{};
637 region.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2;
638 region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
639 region.imageExtent = {
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), 1};
641 VkCopyImageToBufferInfo2 copyInfo{};
642 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_BUFFER_INFO_2;
643 copyInfo.srcImage = outImg.image;
644 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
645 copyInfo.dstBuffer = readbackBuf;
646 copyInfo.regionCount = 1;
647 copyInfo.pRegions = ®ion;
648 vkCmdCopyImageToBuffer2(cmd, ©Info);
650 transitionImageLayout(cmd, outImg.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT, VK_ACCESS_2_SHADER_SAMPLED_READ_BIT);
652 endSingleTimeCommands(cmd);
654 void *mapped =
nullptr;
656 cv::Mat sourceFrame(texHeight, texWidth, CV_8UC4, mapped, tightPitch);
657 if (!processedRecordPathLogged) {
658 std::cout <<
"compute_shader: processed recording path active: Vulkan compute output image -> readback buffer -> MXWrite\n";
659 processedRecordPathLogged =
true;
661 recordFrame(sourceFrame);
662 vkUnmapMemory(
device, readbackMem);
665 void throttleVideoPlayback() {
666 if (!usingFile || fastMode || videoFps <= 0.0) {
670 const auto now = std::chrono::steady_clock::now();
671 if (nextVideoFrameDeadline > now) {
672 std::this_thread::sleep_until(nextVideoFrameDeadline);
674 nextVideoFrameDeadline += std::chrono::duration_cast<std::chrono::steady_clock::duration>(videoFrameInterval);
677 void initComputeResources() {
679 if (
device == VK_NULL_HANDLE) {
680 throw mxvk::Exception(
"Compute resources require an initialized Vulkan device");
685 setFont(assetRoot +
"/data/font.ttf", 20);
688 if (!openVideoSource()) {
689 throw mxvk::Exception(
"Failed to open video file " + inputFilename);
691 configureVideoPlaybackRate();
692 }
else if (!capture.open(cameraIndex)) {
693 throw mxvk::Exception(
"Failed to open camera " + std::to_string(cameraIndex));
697 capture.set(cv::CAP_PROP_FRAME_WIDTH, recordWidth);
698 capture.set(cv::CAP_PROP_FRAME_HEIGHT, recordHeight);
699 const double selectedFps = configureCameraFps();
700 sourceFps = selectedFps;
701 std::cout <<
"compute_shader: requested camera FPS fallback order 60 -> 30 -> 24; selected " << selectedFps <<
" fps\n";
704#if defined(MXVK_WITH_FFMPEG_CAPTURE)
705 if (usingFfCapture) {
706 sourceWidth = ffCapture.
width();
707 sourceHeight = ffCapture.
height();
708 if (sourceWidth <= 0 || sourceHeight <= 0) {
709 throw mxvk::Exception(
"Failed to query FFmpeg video dimensions");
715 if (!capture.read(frame) || frame.empty()) {
716 throw mxvk::Exception(usingFile ?
"Failed to read initial video frame" :
"Failed to read initial camera frame");
719 sourceWidth = frame.cols;
720 sourceHeight = frame.rows;
723 if (explicitResolution) {
724 texWidth = recordWidth;
725 texHeight = recordHeight;
726 std::cout <<
"compute_shader: compute canvas set from explicit resolution " << texWidth <<
"x" << texHeight <<
"; source frames are " << sourceWidth <<
"x" << sourceHeight <<
"\n";
728 texWidth = sourceWidth;
729 texHeight = sourceHeight;
730 recordWidth = texWidth;
731 recordHeight = texHeight;
734 configureRecordingDefaults();
735 recordingEnabled =
true;
737 fpsFont.reset(assetRoot +
"/data/font.ttf", overlayFontSizeForCanvas());
738 playbackStartTime = std::chrono::steady_clock::now();
739 maybeResizeWindowToSource();
741 const VkDeviceSize imgBytes =
static_cast<VkDeviceSize
>(texWidth) * texHeight * 4;
743 createBuffer(imgBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, stagingBuf, stagingMem);
744 createBuffer(imgBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, readbackBuf, readbackMem);
747 const VkCommandBuffer cmd = beginSingleTimeCommands();
749 allocCImg(workImg[0], cmd,
true);
751 allocCImg(workImg[0], cmd);
753 allocCImg(workImg[1], cmd);
754 for (ComputeImage &img : histImg) {
758 allocCImg(outImg, cmd,
true);
760 allocCImg(outImg, cmd);
762 endSingleTimeCommands(cmd);
765 VkSamplerCreateInfo samplerInfo{};
766 samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
767 samplerInfo.magFilter = VK_FILTER_LINEAR;
768 samplerInfo.minFilter = VK_FILTER_LINEAR;
769 samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
770 samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
771 samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
772 samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
773 samplerInfo.maxAnisotropy = 1.0f;
777 buildDescriptorSetLayout();
778 buildComputePipeline();
779 buildDescriptorSets();
780 createDisplayResources();
784#if defined(MXVK_WITH_FFMPEG_CAPTURE)
787 destroyComputeResources();
792 void updateFpsOverlay(
bool frameUploaded) {
793 if (!
active || !frameUploaded) {
799 }
catch (
const std::exception &ex) {
800 std::cerr <<
"compute_shader: failed to clear stale text overlay queue: " << ex.what() <<
"\n";
805 ++processedVideoFrames;
808 const auto now = std::chrono::steady_clock::now();
809 const double elapsed = std::chrono::duration<double>(now - fpsSampleTime).count();
810 if (elapsed >= 0.25) {
811 currentFps =
static_cast<double>(fpsFrameCount) / elapsed;
814 fpsText = std::format(
"FPS: {:.1f}", currentFps);
817 const auto recElapsed = now - playbackStartTime;
818 const uint64_t recTotalSeconds =
static_cast<uint64_t
>(std::chrono::duration_cast<std::chrono::seconds>(recElapsed).count());
819 const uint64_t recHours = recTotalSeconds / 3600U;
820 const uint64_t recMinutes = (recTotalSeconds / 60U) % 60U;
821 const uint64_t recSeconds = recTotalSeconds % 60U;
822 const std::string recText = std::format(
"Rec: {:02}:{:02}:{:02}", recHours, recMinutes, recSeconds);
824 const double effectiveSourceFps = (sourceFps > 0.0) ? sourceFps : videoFps;
825 const uint64_t totalTenths = (effectiveSourceFps > 0.0) ?
static_cast<uint64_t
>(std::llround((
static_cast<double>(processedVideoFrames) / effectiveSourceFps) * 10.0)) : 0U;
826 const uint64_t hours = totalTenths / 36000U;
827 const uint64_t minutes = (totalTenths / 600U) % 60U;
828 const uint64_t seconds = (totalTenths / 10U) % 60U;
829 const uint64_t tenths = totalTenths % 10U;
830 const std::string timeText = std::format(
"Output: {:02}:{:02}:{:02}.{:01}", hours, minutes, seconds, tenths);
831 const std::string overlayText = std::format(
"Source FPS: {:.1f} Current FPS: {:.1f} | {} | {}", sourceFps, currentFps, recText, timeText);
833 printText(overlayText, 18, 18, SDL_Color{255, 240, 0, 255}, fpsFont);
834 if (!spvFiles.empty()) {
835 if (spvFiles[currentSpvIndex] == MODE_SHADER_NAME) {
836 const int modeIndex = std::clamp(shaderMode, 0,
static_cast<int>(ACIDCAM_FILTER_MODE_NAMES.size()) - 1);
837 const std::string modeText = std::format(
"Mode: {} {}/{}", ACIDCAM_FILTER_MODE_NAMES[modeIndex], modeIndex + 1, ACIDCAM_FILTER_MODE_NAMES.size());
838 printText(modeText, 15, 68, SDL_Color{255, 105, 180, 255});
840 const std::string spvText = std::format(
"{}: {}", currentSpvIndex, spvFiles[currentSpvIndex]);
841 printText(spvText, 15, 68, SDL_Color{80, 160, 255, 255});
846 [[nodiscard]] uint32_t findMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties)
const {
847 VkPhysicalDeviceMemoryProperties memProperties{};
850 for (uint32_t index = 0; index < memProperties.memoryTypeCount; ++index) {
851 const bool typeMatches = (typeFilter & (1U << index)) != 0U;
852 const bool propertyMatches = (memProperties.memoryTypes[index].propertyFlags & properties) == properties;
853 if (typeMatches && propertyMatches) {
858 throw mxvk::Exception(
"Failed to find suitable memory type");
861 void createBuffer(VkDeviceSize size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties, VkBuffer &buffer, VkDeviceMemory &bufferMemory) {
862 VkBuffer newBuffer = VK_NULL_HANDLE;
863 VkDeviceMemory newMemory = VK_NULL_HANDLE;
865 VkBufferCreateInfo bufferInfo{};
866 bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
867 bufferInfo.size = size;
868 bufferInfo.usage = usage;
869 bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
874 VkMemoryRequirements memRequirements{};
875 vkGetBufferMemoryRequirements(
device, newBuffer, &memRequirements);
877 VkMemoryAllocateInfo allocInfo{};
878 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
879 allocInfo.allocationSize = memRequirements.size;
880 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, properties);
884 if (newBuffer != VK_NULL_HANDLE) {
885 vkDestroyBuffer(
device, newBuffer,
nullptr);
887 if (newMemory != VK_NULL_HANDLE) {
888 vkFreeMemory(
device, newMemory,
nullptr);
893 if (buffer != VK_NULL_HANDLE) {
894 vkDestroyBuffer(
device, buffer,
nullptr);
896 if (bufferMemory != VK_NULL_HANDLE) {
897 vkFreeMemory(
device, bufferMemory,
nullptr);
900 bufferMemory = newMemory;
903 [[nodiscard]] VkCommandBuffer beginSingleTimeCommands()
const {
904 VkCommandBufferAllocateInfo allocInfo{};
905 allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
906 allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
908 allocInfo.commandBufferCount = 1;
910 VkCommandBuffer commandBuffer = VK_NULL_HANDLE;
913 VkCommandBufferBeginInfo beginInfo{};
914 beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
915 beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
918 return commandBuffer;
921 void endSingleTimeCommands(VkCommandBuffer commandBuffer)
const {
924 VkCommandBufferSubmitInfo commandBufferInfo{};
925 commandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
926 commandBufferInfo.commandBuffer = commandBuffer;
928 VkSubmitInfo2 submitInfo{};
929 submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2;
930 submitInfo.commandBufferInfoCount = 1;
931 submitInfo.pCommandBufferInfos = &commandBufferInfo;
938 void createImage(uint32_t width, uint32_t height, VkFormat format, VkImageTiling tiling, VkImageUsageFlags usage, VkMemoryPropertyFlags properties, VkImage &image, VkDeviceMemory &imageMemory) {
939 VkImage newImage = VK_NULL_HANDLE;
940 VkDeviceMemory newMemory = VK_NULL_HANDLE;
942 VkImageCreateInfo imageInfo{};
943 imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
944 imageInfo.imageType = VK_IMAGE_TYPE_2D;
945 imageInfo.extent.width = width;
946 imageInfo.extent.height = height;
947 imageInfo.extent.depth = 1;
948 imageInfo.mipLevels = 1;
949 imageInfo.arrayLayers = 1;
950 imageInfo.format = format;
951 imageInfo.tiling = tiling;
952 imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
953 imageInfo.usage = usage;
954 imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
955 imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
960 VkMemoryRequirements memRequirements{};
961 vkGetImageMemoryRequirements(
device, newImage, &memRequirements);
963 VkMemoryAllocateInfo allocInfo{};
964 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
965 allocInfo.allocationSize = memRequirements.size;
966 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, properties);
970 if (newImage != VK_NULL_HANDLE) {
971 vkDestroyImage(
device, newImage,
nullptr);
973 if (newMemory != VK_NULL_HANDLE) {
974 vkFreeMemory(
device, newMemory,
nullptr);
979 if (image != VK_NULL_HANDLE) {
980 vkDestroyImage(
device, image,
nullptr);
982 if (imageMemory != VK_NULL_HANDLE) {
983 vkFreeMemory(
device, imageMemory,
nullptr);
986 imageMemory = newMemory;
990 void createCudaExportableImage(ComputeImage &img) {
991 std::cout <<
"compute_shader: CUDA interop init: requesting exportable compute input image " << texWidth <<
"x" << texHeight <<
" RGBA8 optimal-tiled OPAQUE_FD\n";
993 VkExternalMemoryImageCreateInfo externalImageInfo{};
994 externalImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
995 externalImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
997 VkImageCreateInfo imageInfo{};
998 imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
999 imageInfo.pNext = &externalImageInfo;
1000 imageInfo.imageType = VK_IMAGE_TYPE_2D;
1001 imageInfo.extent.width =
static_cast<uint32_t
>(texWidth);
1002 imageInfo.extent.height =
static_cast<uint32_t
>(texHeight);
1003 imageInfo.extent.depth = 1;
1004 imageInfo.mipLevels = 1;
1005 imageInfo.arrayLayers = 1;
1006 imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
1007 imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
1008 imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
1009 imageInfo.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
1010 imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
1011 imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
1015 VkMemoryRequirements memRequirements{};
1016 vkGetImageMemoryRequirements(
device, img.image, &memRequirements);
1018 VkExportMemoryAllocateInfo exportMemoryInfo{};
1019 exportMemoryInfo.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
1020 exportMemoryInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1022 VkMemoryAllocateInfo allocInfo{};
1023 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
1024 allocInfo.pNext = &exportMemoryInfo;
1025 allocInfo.allocationSize = memRequirements.size;
1028 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
1031 img.cudaExportMemorySize = memRequirements.size;
1032 img.cudaInteropUnavailableLogged =
false;
1033 std::cout <<
"compute_shader: CUDA interop init: exportable compute input image allocated (memorySize=" <<
static_cast<unsigned long long>(memRequirements.size) <<
" bytes, memoryType=" << allocInfo.memoryTypeIndex <<
"); optimal image memory will be imported as cudaArray\n";
1035 if (img.image != VK_NULL_HANDLE) {
1036 vkDestroyImage(
device, img.image,
nullptr);
1037 img.image = VK_NULL_HANDLE;
1039 if (img.memory != VK_NULL_HANDLE) {
1040 vkFreeMemory(
device, img.memory,
nullptr);
1041 img.memory = VK_NULL_HANDLE;
1043 img.cudaExportMemorySize = 0;
1048 void destroyCudaInterop(ComputeImage &img) {
1049 if (img.cudaInteropEnabled || img.cudaExternalMemory !=
nullptr || img.cudaMipmappedArray !=
nullptr) {
1050 std::cout <<
"compute_shader: CUDA interop: destroying imported compute input image resources\n";
1052 if (img.cudaMipmappedArray !=
nullptr) {
1053 cudaFreeMipmappedArray(img.cudaMipmappedArray);
1054 img.cudaMipmappedArray =
nullptr;
1055 img.cudaArray =
nullptr;
1057 if (img.cudaExternalMemory !=
nullptr) {
1058 cudaDestroyExternalMemory(img.cudaExternalMemory);
1059 img.cudaExternalMemory =
nullptr;
1061 img.cudaInteropEnabled =
false;
1062 img.cudaExportMemorySize = 0;
1063 img.cudaUploadLogged =
false;
1064 img.cudaBarrierLogged =
false;
1067 bool ensureCudaInterop(ComputeImage &img) {
1068 if (img.cudaInteropEnabled) {
1071 if (img.memory == VK_NULL_HANDLE || img.cudaExportMemorySize == 0) {
1072 if (!img.cudaInteropUnavailableLogged) {
1073 std::cout <<
"compute_shader: CUDA interop init: compute input image is not exportable\n";
1074 img.cudaInteropUnavailableLogged =
true;
1078 if (vkGetMemoryFdKHR ==
nullptr) {
1079 if (!img.cudaInteropUnavailableLogged) {
1080 std::cout <<
"compute_shader: CUDA interop init: vkGetMemoryFdKHR was not loaded\n";
1081 img.cudaInteropUnavailableLogged =
true;
1086 VkMemoryGetFdInfoKHR fdInfo{};
1087 fdInfo.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
1088 fdInfo.memory = img.memory;
1089 fdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1092 const VkResult fdResult = vkGetMemoryFdKHR(
device, &fdInfo, &memoryFd);
1093 if (fdResult != VK_SUCCESS) {
1094 if (!img.cudaInteropUnavailableLogged) {
1095 std::cout <<
"compute_shader: CUDA interop init: vkGetMemoryFdKHR failed (" <<
static_cast<int>(fdResult) <<
")\n";
1096 img.cudaInteropUnavailableLogged =
true;
1100 std::cout <<
"compute_shader: CUDA interop init: exported compute input image memory fd=" << memoryFd <<
"\n";
1102 cudaExternalMemoryHandleDesc externalMemoryDesc{};
1103 externalMemoryDesc.type = cudaExternalMemoryHandleTypeOpaqueFd;
1104 externalMemoryDesc.handle.fd = memoryFd;
1105 externalMemoryDesc.size = img.cudaExportMemorySize;
1107 cudaError_t cudaResult = cudaImportExternalMemory(&img.cudaExternalMemory, &externalMemoryDesc);
1108 if (cudaResult != cudaSuccess) {
1110 if (!img.cudaInteropUnavailableLogged) {
1111 std::cout <<
"compute_shader: CUDA interop init: cudaImportExternalMemory failed: " << cudaGetErrorString(cudaResult) <<
"\n";
1112 img.cudaInteropUnavailableLogged =
true;
1114 img.cudaExternalMemory =
nullptr;
1117 std::cout <<
"compute_shader: CUDA interop init: imported compute input image external memory into CUDA (" <<
static_cast<unsigned long long>(img.cudaExportMemorySize) <<
" bytes)\n";
1119 cudaExternalMemoryMipmappedArrayDesc arrayDesc{};
1120 arrayDesc.offset = 0;
1121 arrayDesc.formatDesc = cudaCreateChannelDesc<uchar4>();
1122 arrayDesc.extent = make_cudaExtent(
static_cast<size_t>(texWidth),
static_cast<size_t>(texHeight), 0);
1123 arrayDesc.flags = cudaArrayColorAttachment;
1124 arrayDesc.numLevels = 1;
1126 cudaResult = cudaExternalMemoryGetMappedMipmappedArray(&img.cudaMipmappedArray, img.cudaExternalMemory, &arrayDesc);
1127 if (cudaResult != cudaSuccess) {
1128 if (!img.cudaInteropUnavailableLogged) {
1129 std::cout <<
"compute_shader: CUDA interop init: cudaExternalMemoryGetMappedMipmappedArray failed: " << cudaGetErrorString(cudaResult) <<
"\n";
1130 img.cudaInteropUnavailableLogged =
true;
1132 destroyCudaInterop(img);
1135 std::cout <<
"compute_shader: CUDA interop init: mapped compute input CUDA mipmapped array " << texWidth <<
"x" << texHeight <<
" uchar4\n";
1137 cudaResult = cudaGetMipmappedArrayLevel(&img.cudaArray, img.cudaMipmappedArray, 0);
1138 if (cudaResult != cudaSuccess) {
1139 if (!img.cudaInteropUnavailableLogged) {
1140 std::cout <<
"compute_shader: CUDA interop init: cudaGetMipmappedArrayLevel failed: " << cudaGetErrorString(cudaResult) <<
"\n";
1141 img.cudaInteropUnavailableLogged =
true;
1143 destroyCudaInterop(img);
1147 img.cudaInteropEnabled =
true;
1148 std::cout <<
"compute_shader: CUDA interop init: direct CUDA-to-compute-input upload is ready\n";
1153 [[nodiscard]] VkImageView createImageView(VkImage image, VkFormat format, VkImageAspectFlags aspectFlags)
const {
1154 VkImageViewCreateInfo viewInfo{};
1155 viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
1156 viewInfo.image = image;
1157 viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
1158 viewInfo.format = format;
1159 viewInfo.subresourceRange.aspectMask = aspectFlags;
1160 viewInfo.subresourceRange.baseMipLevel = 0;
1161 viewInfo.subresourceRange.levelCount = 1;
1162 viewInfo.subresourceRange.baseArrayLayer = 0;
1163 viewInfo.subresourceRange.layerCount = 1;
1165 VkImageView imageView = VK_NULL_HANDLE;
1170 void allocCImg(ComputeImage &img, VkCommandBuffer cmd, [[maybe_unused]]
bool cudaExportable =
false) {
1172 if (cudaExportable) {
1174 createCudaExportableImage(img);
1175 }
catch (
const std::exception &ex) {
1176 std::cout <<
"compute_shader: CUDA exportable input image unavailable: " << ex.what() <<
"; using Vulkan staging fallback\n";
1177 createImage(
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, img.image, img.memory);
1180 createImage(
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, img.image, img.memory);
1183 createImage(
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, img.image, img.memory);
1185 img.view = createImageView(img.image, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT);
1187 transitionImageLayout(cmd, img.image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_NONE, VK_ACCESS_2_NONE, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1190 void reloadPipeline() {
1191 vkDeviceWaitIdle(
device);
1192 if (compPipeline != VK_NULL_HANDLE) {
1193 vkDestroyPipeline(
device, compPipeline,
nullptr);
1194 compPipeline = VK_NULL_HANDLE;
1196 if (compPipeLayout != VK_NULL_HANDLE) {
1197 vkDestroyPipelineLayout(
device, compPipeLayout,
nullptr);
1198 compPipeLayout = VK_NULL_HANDLE;
1200 shaderMode = initialShaderMode;
1201 buildComputePipeline();
1204 void buildDescriptorSetLayout() {
1205 std::array<VkDescriptorSetLayoutBinding, 3> bindings{};
1207 bindings[0].binding = 0;
1208 bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
1209 bindings[0].descriptorCount = 1;
1210 bindings[0].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1212 bindings[1].binding = 1;
1213 bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1214 bindings[1].descriptorCount = 1;
1215 bindings[1].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1217 bindings[2].binding = 2;
1218 bindings[2].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1219 bindings[2].descriptorCount = HISTORY_SIZE;
1220 bindings[2].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1222 VkDescriptorSetLayoutCreateInfo createInfo{};
1223 createInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
1224 createInfo.bindingCount =
static_cast<uint32_t
>(bindings.size());
1225 createInfo.pBindings = bindings.data();
1229 void buildComputePipeline() {
1230 const std::string spvPath = assetRoot +
"/data/" + spvFiles[currentSpvIndex];
1234 VkPushConstantRange pushConstantRange{};
1235 pushConstantRange.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1236 pushConstantRange.offset = 0;
1237 pushConstantRange.size =
sizeof(ComputePC);
1239 VkPipelineLayoutCreateInfo pipelineLayoutInfo{};
1240 pipelineLayoutInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
1241 pipelineLayoutInfo.setLayoutCount = 1;
1242 pipelineLayoutInfo.pSetLayouts = &compDSLayout;
1243 pipelineLayoutInfo.pushConstantRangeCount = 1;
1244 pipelineLayoutInfo.pPushConstantRanges = &pushConstantRange;
1247 VkComputePipelineCreateInfo pipelineInfo{};
1248 pipelineInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO;
1249 pipelineInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1250 pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
1251 pipelineInfo.stage.module =
module;
1252 pipelineInfo.stage.pName =
"main";
1253 pipelineInfo.layout = compPipeLayout;
1254 VK_CHECK_RESULT(vkCreateComputePipelines(
device, VK_NULL_HANDLE, 1, &pipelineInfo,
nullptr, &compPipeline));
1256 vkDestroyShaderModule(
device, module,
nullptr);
1259 void buildDescriptorSets() {
1260 std::array<VkDescriptorPoolSize, 2> poolSizes{};
1261 poolSizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4};
1262 poolSizes[1] = {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4 * (1 + HISTORY_SIZE)};
1264 VkDescriptorPoolCreateInfo poolInfo{};
1265 poolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
1266 poolInfo.maxSets = 4;
1267 poolInfo.poolSizeCount =
static_cast<uint32_t
>(poolSizes.size());
1268 poolInfo.pPoolSizes = poolSizes.data();
1271 std::array<VkDescriptorSetLayout, 4> layouts{};
1272 layouts.fill(compDSLayout);
1274 VkDescriptorSetAllocateInfo allocInfo{};
1275 allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
1276 allocInfo.descriptorPool = compDSPool;
1277 allocInfo.descriptorSetCount =
static_cast<uint32_t
>(layouts.size());
1278 allocInfo.pSetLayouts = layouts.data();
1280 std::array<VkDescriptorSet, 4> raw{};
1284 blendDS[0] = raw[2];
1285 blendDS[1] = raw[3];
1287 writeBlurDS(blurDS[0], workImg[0].view, workImg[1].view);
1288 writeBlurDS(blurDS[1], workImg[1].view, workImg[0].view);
1289 writeBlendDS(blendDS[0], workImg[0].view);
1290 writeBlendDS(blendDS[1], workImg[1].view);
1293 void writeBlurDS(VkDescriptorSet descriptorSet, VkImageView destView, VkImageView srcView) {
1294 VkDescriptorImageInfo destInfo{VK_NULL_HANDLE, destView, VK_IMAGE_LAYOUT_GENERAL};
1295 VkDescriptorImageInfo srcInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL};
1296 std::vector<VkDescriptorImageInfo> historyInfos(HISTORY_SIZE, VkDescriptorImageInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL});
1298 std::array<VkWriteDescriptorSet, 3> writes{};
1299 writes[0] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &destInfo,
nullptr,
nullptr};
1300 writes[1] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 1, 0, 1, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, &srcInfo,
nullptr,
nullptr};
1301 writes[2] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 2, 0, HISTORY_SIZE, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, historyInfos.data(),
nullptr,
nullptr};
1302 vkUpdateDescriptorSets(
device,
static_cast<uint32_t
>(writes.size()), writes.data(), 0,
nullptr);
1305 [[nodiscard]] std::vector<char> readDisplayShader(
const std::string &name)
const {
1306 const std::array<std::string, 3> candidates = {
1307 assetRoot +
"/data/" + name,
1308 assetRoot +
"/" + name,
1309 std::string(
"data/") + name,
1312 for (
const std::string &path : candidates) {
1313 std::ifstream file(path, std::ios::binary);
1314 if (file.is_open()) {
1319 throw mxvk::Exception(
"Cannot open compute display shader: " + name);
1322 void createDisplayBuffers() {
1323 if (displayVertexBuffer != VK_NULL_HANDLE && displayIndexBuffer != VK_NULL_HANDLE) {
1327 const std::array<float, 16> vertices = {
1345 const std::array<uint16_t, 6> indices = {0, 1, 2, 0, 2, 3};
1347 createBuffer(
sizeof(
float) * vertices.size(), VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, displayVertexBuffer, displayVertexMemory);
1348 createBuffer(
sizeof(uint16_t) * indices.size(), VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, displayIndexBuffer, displayIndexMemory);
1350 void *mapped =
nullptr;
1351 VK_CHECK_RESULT(vkMapMemory(
device, displayVertexMemory, 0,
sizeof(
float) * vertices.size(), 0, &mapped));
1352 std::memcpy(mapped, vertices.data(),
sizeof(
float) * vertices.size());
1353 vkUnmapMemory(
device, displayVertexMemory);
1355 VK_CHECK_RESULT(vkMapMemory(
device, displayIndexMemory, 0,
sizeof(uint16_t) * indices.size(), 0, &mapped));
1356 std::memcpy(mapped, indices.data(),
sizeof(uint16_t) * indices.size());
1357 vkUnmapMemory(
device, displayIndexMemory);
1360 void createDisplayDescriptorSet() {
1361 VkDescriptorSetLayoutBinding binding{};
1362 binding.binding = 0;
1363 binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1364 binding.descriptorCount = 1;
1365 binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT;
1367 VkDescriptorSetLayoutCreateInfo layoutInfo{};
1368 layoutInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
1369 layoutInfo.bindingCount = 1;
1370 layoutInfo.pBindings = &binding;
1373 VkDescriptorPoolSize poolSize{};
1374 poolSize.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1375 poolSize.descriptorCount = 1;
1377 VkDescriptorPoolCreateInfo poolInfo{};
1378 poolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
1379 poolInfo.poolSizeCount = 1;
1380 poolInfo.pPoolSizes = &poolSize;
1381 poolInfo.maxSets = 1;
1384 VkDescriptorSetAllocateInfo allocInfo{};
1385 allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
1386 allocInfo.descriptorPool = displayDSPool;
1387 allocInfo.descriptorSetCount = 1;
1388 allocInfo.pSetLayouts = &displayDSLayout;
1391 VkDescriptorImageInfo imageInfo{};
1392 imageInfo.sampler = computeSampler;
1393 imageInfo.imageView = outImg.view;
1394 imageInfo.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1396 VkWriteDescriptorSet write{};
1397 write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
1398 write.dstSet = displayDS;
1399 write.dstBinding = 0;
1400 write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1401 write.descriptorCount = 1;
1402 write.pImageInfo = &imageInfo;
1403 vkUpdateDescriptorSets(
device, 1, &write, 0,
nullptr);
1406 void rebuildDisplayPipeline() {
1407 if (displayPipeline != VK_NULL_HANDLE) {
1408 vkDestroyPipeline(
device, displayPipeline,
nullptr);
1409 displayPipeline = VK_NULL_HANDLE;
1411 if (displayPipeLayout != VK_NULL_HANDLE) {
1412 vkDestroyPipelineLayout(
device, displayPipeLayout,
nullptr);
1413 displayPipeLayout = VK_NULL_HANDLE;
1415 if (displayDSLayout == VK_NULL_HANDLE ||
swapchain_format == VK_FORMAT_UNDEFINED) {
1422 VkPipelineShaderStageCreateInfo vertStage{};
1423 vertStage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1424 vertStage.stage = VK_SHADER_STAGE_VERTEX_BIT;
1425 vertStage.module = vertModule;
1426 vertStage.pName =
"main";
1428 VkPipelineShaderStageCreateInfo fragStage{};
1429 fragStage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1430 fragStage.stage = VK_SHADER_STAGE_FRAGMENT_BIT;
1431 fragStage.module = fragModule;
1432 fragStage.pName =
"main";
1434 const std::array<VkPipelineShaderStageCreateInfo, 2> stages = {vertStage, fragStage};
1436 VkVertexInputBindingDescription binding{};
1437 binding.binding = 0;
1438 binding.stride =
sizeof(float) * 4;
1439 binding.inputRate = VK_VERTEX_INPUT_RATE_VERTEX;
1441 std::array<VkVertexInputAttributeDescription, 2> attrs{};
1442 attrs[0].binding = 0;
1443 attrs[0].location = 0;
1444 attrs[0].format = VK_FORMAT_R32G32_SFLOAT;
1445 attrs[0].offset = 0;
1446 attrs[1].binding = 0;
1447 attrs[1].location = 1;
1448 attrs[1].format = VK_FORMAT_R32G32_SFLOAT;
1449 attrs[1].offset =
sizeof(float) * 2;
1451 VkPipelineVertexInputStateCreateInfo vertexInput{};
1452 vertexInput.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO;
1453 vertexInput.vertexBindingDescriptionCount = 1;
1454 vertexInput.pVertexBindingDescriptions = &binding;
1455 vertexInput.vertexAttributeDescriptionCount =
static_cast<uint32_t
>(attrs.size());
1456 vertexInput.pVertexAttributeDescriptions = attrs.data();
1458 VkPipelineInputAssemblyStateCreateInfo inputAssembly{};
1459 inputAssembly.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO;
1460 inputAssembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
1462 const std::array<VkDynamicState, 2> dynamicStates = {VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR};
1463 VkPipelineDynamicStateCreateInfo dynamicInfo{};
1464 dynamicInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO;
1465 dynamicInfo.dynamicStateCount =
static_cast<uint32_t
>(dynamicStates.size());
1466 dynamicInfo.pDynamicStates = dynamicStates.data();
1468 VkPipelineViewportStateCreateInfo viewportState{};
1469 viewportState.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO;
1470 viewportState.viewportCount = 1;
1471 viewportState.scissorCount = 1;
1473 VkPipelineRasterizationStateCreateInfo rasterizer{};
1474 rasterizer.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
1475 rasterizer.polygonMode = VK_POLYGON_MODE_FILL;
1476 rasterizer.cullMode = VK_CULL_MODE_NONE;
1477 rasterizer.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
1478 rasterizer.lineWidth = 1.0f;
1480 VkPipelineMultisampleStateCreateInfo multisample{};
1481 multisample.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO;
1482 multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
1484 VkPipelineDepthStencilStateCreateInfo depthStencil{};
1485 depthStencil.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
1486 depthStencil.depthTestEnable = VK_FALSE;
1487 depthStencil.depthWriteEnable = VK_FALSE;
1489 VkPipelineColorBlendAttachmentState blendAttachment{};
1490 blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
1491 blendAttachment.blendEnable = VK_FALSE;
1493 VkPipelineColorBlendStateCreateInfo colorBlend{};
1494 colorBlend.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO;
1495 colorBlend.attachmentCount = 1;
1496 colorBlend.pAttachments = &blendAttachment;
1498 VkPushConstantRange pushRange{};
1499 pushRange.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT;
1500 pushRange.size =
sizeof(float) * 12;
1502 VkPipelineLayoutCreateInfo layoutInfo{};
1503 layoutInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
1504 layoutInfo.setLayoutCount = 1;
1505 layoutInfo.pSetLayouts = &displayDSLayout;
1506 layoutInfo.pushConstantRangeCount = 1;
1507 layoutInfo.pPushConstantRanges = &pushRange;
1510 VkPipelineRenderingCreateInfo renderingInfo{};
1511 renderingInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO;
1512 renderingInfo.colorAttachmentCount = 1;
1518 VkGraphicsPipelineCreateInfo pipelineInfo{};
1519 pipelineInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO;
1520 pipelineInfo.pNext = &renderingInfo;
1521 pipelineInfo.stageCount =
static_cast<uint32_t
>(stages.size());
1522 pipelineInfo.pStages = stages.data();
1523 pipelineInfo.pVertexInputState = &vertexInput;
1524 pipelineInfo.pInputAssemblyState = &inputAssembly;
1525 pipelineInfo.pViewportState = &viewportState;
1526 pipelineInfo.pRasterizationState = &rasterizer;
1527 pipelineInfo.pMultisampleState = &multisample;
1528 pipelineInfo.pDepthStencilState = &depthStencil;
1529 pipelineInfo.pColorBlendState = &colorBlend;
1530 pipelineInfo.pDynamicState = &dynamicInfo;
1531 pipelineInfo.layout = displayPipeLayout;
1532 pipelineInfo.renderPass = VK_NULL_HANDLE;
1533 VK_CHECK_RESULT(vkCreateGraphicsPipelines(
device, VK_NULL_HANDLE, 1, &pipelineInfo,
nullptr, &displayPipeline));
1535 vkDestroyShaderModule(
device, fragModule,
nullptr);
1536 vkDestroyShaderModule(
device, vertModule,
nullptr);
1539 void createDisplayResources() {
1540 createDisplayBuffers();
1541 createDisplayDescriptorSet();
1542 rebuildDisplayPipeline();
1543 std::cout <<
"compute_shader: display path active: Vulkan compute output image -> fullscreen sampled draw (no readback, no sprite upload)\n";
1546 void writeBlendDS(VkDescriptorSet descriptorSet, VkImageView srcView) {
1547 VkDescriptorImageInfo destInfo{VK_NULL_HANDLE, outImg.view, VK_IMAGE_LAYOUT_GENERAL};
1548 VkDescriptorImageInfo srcInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL};
1550 std::vector<VkDescriptorImageInfo> historyInfos(HISTORY_SIZE);
1551 for (
int index = 0; index < HISTORY_SIZE; ++index) {
1552 historyInfos[index] = {computeSampler, histImg[index].view, VK_IMAGE_LAYOUT_GENERAL};
1555 std::array<VkWriteDescriptorSet, 3> writes{};
1556 writes[0] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &destInfo,
nullptr,
nullptr};
1557 writes[1] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 1, 0, 1, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, &srcInfo,
nullptr,
nullptr};
1558 writes[2] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
nullptr, descriptorSet, 2, 0, HISTORY_SIZE, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, historyInfos.data(),
nullptr,
nullptr};
1559 vkUpdateDescriptorSets(
device,
static_cast<uint32_t
>(writes.size()), writes.data(), 0,
nullptr);
1562 void uploadToImage(
const void *data,
int srcPitch, ComputeImage &img) {
1563 const VkDeviceSize bytes =
static_cast<VkDeviceSize
>(texWidth) * texHeight * 4;
1564 const int tightPitch = texWidth * 4;
1565 if (data ==
nullptr || srcPitch < tightPitch) {
1569 void *mapped =
nullptr;
1571 if (srcPitch == tightPitch) {
1572 std::memcpy(mapped, data,
static_cast<size_t>(bytes));
1574 const auto *src =
static_cast<const uint8_t *
>(data);
1575 auto *dst =
static_cast<uint8_t *
>(mapped);
1576 for (
int row = 0; row < texHeight; ++row) {
1577 std::memcpy(dst +
static_cast<size_t>(row) * tightPitch, src +
static_cast<size_t>(row) * srcPitch,
static_cast<size_t>(tightPitch));
1580 vkUnmapMemory(
device, stagingMem);
1582 const VkCommandBuffer cmd = beginSingleTimeCommands();
1584 transitionImageLayout(cmd, img.image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
1586 VkBufferImageCopy2 region{};
1587 region.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2;
1588 region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1589 region.imageExtent = {
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), 1};
1591 VkCopyBufferToImageInfo2 copyInfo{};
1592 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_BUFFER_TO_IMAGE_INFO_2;
1593 copyInfo.srcBuffer = stagingBuf;
1594 copyInfo.dstImage = img.image;
1595 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
1596 copyInfo.regionCount = 1;
1597 copyInfo.pRegions = ®ion;
1598 vkCmdCopyBufferToImage2(cmd, ©Info);
1600 transitionImageLayout(cmd, img.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1602 endSingleTimeCommands(cmd);
1606 bool uploadGpuToImage(
const cv::cuda::GpuMat &rgba, cv::cuda::Stream &stream, ComputeImage &img) {
1607 if (rgba.empty() || rgba.type() != CV_8UC4 || rgba.cols != texWidth || rgba.rows != texHeight) {
1610 if (!ensureCudaInterop(img)) {
1614 cudaStream_t cudaStream = mxvk::cuda_stream_handle(stream);
1615 if (!img.cudaUploadLogged) {
1616 std::cout <<
"compute_shader: CUDA interop upload: copying " << rgba.cols <<
"x" << rgba.rows <<
" RGBA GpuMat to Vulkan compute storage image via cudaArray (source pitch=" <<
static_cast<unsigned long long>(rgba.step) <<
" bytes, copy row bytes=" <<
static_cast<unsigned long long>(
static_cast<size_t>(rgba.cols) * 4U) <<
")\n";
1617 img.cudaUploadLogged =
true;
1620 cudaError_t cudaResult = cudaMemcpy2DToArrayAsync(img.cudaArray, 0, 0, rgba.ptr(), rgba.step,
static_cast<size_t>(rgba.cols) * 4U,
static_cast<size_t>(rgba.rows), cudaMemcpyDeviceToDevice, cudaStream);
1621 if (cudaResult != cudaSuccess) {
1622 std::cout <<
"compute_shader: CUDA interop compute input copy failed: " << cudaGetErrorString(cudaResult) <<
"\n";
1626 cudaResult = cudaStreamSynchronize(cudaStream);
1627 if (cudaResult != cudaSuccess) {
1628 std::cout <<
"compute_shader: CUDA interop compute input sync failed: " << cudaGetErrorString(cudaResult) <<
"\n";
1632 const VkCommandBuffer cmd = beginSingleTimeCommands();
1633 transitionImageLayout(cmd, img.image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1634 endSingleTimeCommands(cmd);
1636 if (!img.cudaBarrierLogged) {
1637 std::cout <<
"compute_shader: CUDA interop sync: CUDA stream synchronized; Vulkan records GENERAL -> GENERAL memory barrier before compute sampling\n";
1638 img.cudaBarrierLogged =
true;
1644 void dispatchOne(VkCommandBuffer cmd, VkDescriptorSet descriptorSet,
int mode) {
1646 pc.
mode = (!spvFiles.empty() && spvFiles[currentSpvIndex] == MODE_SHADER_NAME) ? shaderMode : mode;
1655 vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, compPipeline);
1656 vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, compPipeLayout, 0, 1, &descriptorSet, 0,
nullptr);
1657 vkCmdPushConstants(cmd, compPipeLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0,
sizeof(pc), &pc);
1659 const uint32_t groupX = (
static_cast<uint32_t
>(texWidth) + 15) / 16;
1660 const uint32_t groupY = (
static_cast<uint32_t
>(texHeight) + 15) / 16;
1661 vkCmdDispatch(cmd, groupX, groupY, 1);
1664 void renderComputeOutput(VkCommandBuffer cmd) {
1665 if (displayPipeline == VK_NULL_HANDLE || displayPipeLayout == VK_NULL_HANDLE || displayDS == VK_NULL_HANDLE || displayVertexBuffer == VK_NULL_HANDLE || displayIndexBuffer == VK_NULL_HANDLE) {
1669 vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, displayPipeline);
1670 vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, displayPipeLayout, 0, 1, &displayDS, 0,
nullptr);
1672 const VkDeviceSize offset = 0;
1673 vkCmdBindVertexBuffers(cmd, 0, 1, &displayVertexBuffer, &offset);
1674 vkCmdBindIndexBuffer(cmd, displayIndexBuffer, 0, VK_INDEX_TYPE_UINT16);
1695 {0.0f, 0.0f, 0.0f, 0.0f},
1698 vkCmdPushConstants(cmd, displayPipeLayout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0,
sizeof(DisplayPC), &pc);
1699 vkCmdDrawIndexed(cmd, 6, 1, 0, 0, 0);
1702 void computeBarrier(VkCommandBuffer cmd, VkImage img)
const { transitionImageLayout(cmd, img, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT); }
1704 void runComputeFrame() {
1705 const VkCommandBuffer cmd = beginSingleTimeCommands();
1708 const bool isModeShader = !spvFiles.empty() && spvFiles[currentSpvIndex] == MODE_SHADER_NAME;
1711 std::array<VkImageMemoryBarrier2, 2> barriers{};
1712 barriers[0] = makeImageBarrier(workImg[srcIdx].image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
1714 barriers[1] = makeImageBarrier(histImg[historyIndex].image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
1716 VkDependencyInfo dependencyInfo{};
1717 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
1718 dependencyInfo.imageMemoryBarrierCount =
static_cast<uint32_t
>(barriers.size());
1719 dependencyInfo.pImageMemoryBarriers = barriers.data();
1720 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
1722 VkImageCopy2 copy{};
1723 copy.sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2;
1724 copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1725 copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1726 copy.extent = {
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), 1};
1728 VkCopyImageInfo2 copyInfo{};
1729 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2;
1730 copyInfo.srcImage = workImg[srcIdx].image;
1731 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
1732 copyInfo.dstImage = histImg[historyIndex].image;
1733 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
1734 copyInfo.regionCount = 1;
1735 copyInfo.pRegions = ©
1736 vkCmdCopyImage2(cmd, ©Info);
1738 barriers[0] = makeImageBarrier(workImg[srcIdx].image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1740 barriers[1] = makeImageBarrier(histImg[historyIndex].image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT);
1742 dependencyInfo = {};
1743 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
1744 dependencyInfo.imageMemoryBarrierCount =
static_cast<uint32_t
>(barriers.size());
1745 dependencyInfo.pImageMemoryBarriers = barriers.data();
1746 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
1748 if (historyCount < HISTORY_SIZE) {
1751 historyIndex = (historyIndex + 1) % HISTORY_SIZE;
1752 dispatchOne(cmd, blendDS[srcIdx], shaderMode);
1754 const int passes = 3 + (std::rand() % 7);
1755 for (
int pass = 0; pass < passes; ++pass) {
1756 dispatchOne(cmd, blurDS[dstIdx], 0);
1757 computeBarrier(cmd, workImg[dstIdx].image);
1758 std::swap(srcIdx, dstIdx);
1761 std::array<VkImageMemoryBarrier2, 2> barriers{};
1762 barriers[0] = makeImageBarrier(workImg[srcIdx].image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
1764 barriers[1] = makeImageBarrier(histImg[historyIndex].image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
1766 VkDependencyInfo dependencyInfo{};
1767 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
1768 dependencyInfo.imageMemoryBarrierCount =
static_cast<uint32_t
>(barriers.size());
1769 dependencyInfo.pImageMemoryBarriers = barriers.data();
1770 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
1772 VkImageCopy2 copy{};
1773 copy.sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2;
1774 copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1775 copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1776 copy.extent = {
static_cast<uint32_t
>(texWidth),
static_cast<uint32_t
>(texHeight), 1};
1778 VkCopyImageInfo2 copyInfo{};
1779 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2;
1780 copyInfo.srcImage = workImg[srcIdx].image;
1781 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
1782 copyInfo.dstImage = histImg[historyIndex].image;
1783 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
1784 copyInfo.regionCount = 1;
1785 copyInfo.pRegions = ©
1786 vkCmdCopyImage2(cmd, ©Info);
1788 barriers[0] = makeImageBarrier(workImg[srcIdx].image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1790 barriers[1] = makeImageBarrier(histImg[historyIndex].image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT);
1792 dependencyInfo = {};
1793 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
1794 dependencyInfo.imageMemoryBarrierCount =
static_cast<uint32_t
>(barriers.size());
1795 dependencyInfo.pImageMemoryBarriers = barriers.data();
1796 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
1798 if (historyCount < HISTORY_SIZE) {
1801 historyIndex = (historyIndex + 1) % HISTORY_SIZE;
1803 const bool isMetalMedian = !spvFiles.empty() && spvFiles[currentSpvIndex].find(
"metalmedianblend") != std::string::npos;
1804 dispatchOne(cmd, blendDS[srcIdx], isMetalMedian ? 2 : 1);
1807 transitionImageLayout(cmd, outImg.image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT, VK_ACCESS_2_SHADER_SAMPLED_READ_BIT);
1809 endSingleTimeCommands(cmd);
1812 void tickAnimState() {
1813 if (currentDir == 1) {
1814 if (++currentHistIdx >= HISTORY_SIZE - 1) {
1815 currentHistIdx = HISTORY_SIZE - 1;
1818 }
else if (--currentHistIdx <= 0) {
1823 if (squareDir == 1) {
1825 if (currentSquare >= 64) {
1831 if (currentSquare <= 2) {
1837 static int alphaDir = 1;
1838 if (alphaDir == 1) {
1840 if (alpha >= (255.0f / 32.0f)) {
1841 alpha = 255.0f / 32.0f;
1846 if (alpha <= 1.0f) {
1853 static VkImageMemoryBarrier2 makeImageBarrier(VkImage img, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags2 srcStage, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStage, VkAccessFlags2 dstAccess) {
1854 VkImageMemoryBarrier2 barrier{};
1855 barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2;
1856 barrier.srcStageMask = srcStage;
1857 barrier.srcAccessMask = srcAccess;
1858 barrier.dstStageMask = dstStage;
1859 barrier.dstAccessMask = dstAccess;
1860 barrier.oldLayout = oldLayout;
1861 barrier.newLayout = newLayout;
1862 barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
1863 barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
1864 barrier.image = img;
1865 barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
1869 static void transitionImageLayout(VkCommandBuffer cmd, VkImage img, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags2 srcStage, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStage, VkAccessFlags2 dstAccess) {
1870 const VkImageMemoryBarrier2 barrier = makeImageBarrier(img, oldLayout, newLayout, srcStage, srcAccess, dstStage, dstAccess);
1872 VkDependencyInfo dependencyInfo{};
1873 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
1874 dependencyInfo.imageMemoryBarrierCount = 1;
1875 dependencyInfo.pImageMemoryBarriers = &barrier;
1876 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
1879 void destroyDisplayResources() {
1880 if (displayPipeline != VK_NULL_HANDLE) {
1881 vkDestroyPipeline(
device, displayPipeline,
nullptr);
1882 displayPipeline = VK_NULL_HANDLE;
1884 if (displayPipeLayout != VK_NULL_HANDLE) {
1885 vkDestroyPipelineLayout(
device, displayPipeLayout,
nullptr);
1886 displayPipeLayout = VK_NULL_HANDLE;
1888 if (displayDSPool != VK_NULL_HANDLE) {
1889 vkDestroyDescriptorPool(
device, displayDSPool,
nullptr);
1890 displayDSPool = VK_NULL_HANDLE;
1891 displayDS = VK_NULL_HANDLE;
1893 if (displayDSLayout != VK_NULL_HANDLE) {
1894 vkDestroyDescriptorSetLayout(
device, displayDSLayout,
nullptr);
1895 displayDSLayout = VK_NULL_HANDLE;
1897 if (displayVertexBuffer != VK_NULL_HANDLE) {
1898 vkDestroyBuffer(
device, displayVertexBuffer,
nullptr);
1899 displayVertexBuffer = VK_NULL_HANDLE;
1901 if (displayVertexMemory != VK_NULL_HANDLE) {
1902 vkFreeMemory(
device, displayVertexMemory,
nullptr);
1903 displayVertexMemory = VK_NULL_HANDLE;
1905 if (displayIndexBuffer != VK_NULL_HANDLE) {
1906 vkDestroyBuffer(
device, displayIndexBuffer,
nullptr);
1907 displayIndexBuffer = VK_NULL_HANDLE;
1909 if (displayIndexMemory != VK_NULL_HANDLE) {
1910 vkFreeMemory(
device, displayIndexMemory,
nullptr);
1911 displayIndexMemory = VK_NULL_HANDLE;
1915 void destroyComputeResources() {
1916 if (
device == VK_NULL_HANDLE) {
1920 vkDeviceWaitIdle(
device);
1921 destroyDisplayResources();
1923 auto destroyImage = [&](ComputeImage &img) {
1925 destroyCudaInterop(img);
1927 if (img.view != VK_NULL_HANDLE) {
1928 vkDestroyImageView(
device, img.view,
nullptr);
1930 if (img.image != VK_NULL_HANDLE) {
1931 vkDestroyImage(
device, img.image,
nullptr);
1933 if (img.memory != VK_NULL_HANDLE) {
1934 vkFreeMemory(
device, img.memory,
nullptr);
1939 destroyImage(workImg[0]);
1940 destroyImage(workImg[1]);
1941 for (ComputeImage &img : histImg) {
1944 destroyImage(outImg);
1946 if (computeSampler != VK_NULL_HANDLE) {
1947 vkDestroySampler(
device, computeSampler,
nullptr);
1948 computeSampler = VK_NULL_HANDLE;
1950 if (compDSPool != VK_NULL_HANDLE) {
1951 vkDestroyDescriptorPool(
device, compDSPool,
nullptr);
1952 compDSPool = VK_NULL_HANDLE;
1954 if (compPipeline != VK_NULL_HANDLE) {
1955 vkDestroyPipeline(
device, compPipeline,
nullptr);
1956 compPipeline = VK_NULL_HANDLE;
1958 if (compPipeLayout != VK_NULL_HANDLE) {
1959 vkDestroyPipelineLayout(
device, compPipeLayout,
nullptr);
1960 compPipeLayout = VK_NULL_HANDLE;
1962 if (compDSLayout != VK_NULL_HANDLE) {
1963 vkDestroyDescriptorSetLayout(
device, compDSLayout,
nullptr);
1964 compDSLayout = VK_NULL_HANDLE;
1966 if (stagingBuf != VK_NULL_HANDLE) {
1967 vkDestroyBuffer(
device, stagingBuf,
nullptr);
1968 stagingBuf = VK_NULL_HANDLE;
1970 if (stagingMem != VK_NULL_HANDLE) {
1971 vkFreeMemory(
device, stagingMem,
nullptr);
1972 stagingMem = VK_NULL_HANDLE;
1974 if (readbackBuf != VK_NULL_HANDLE) {
1975 vkDestroyBuffer(
device, readbackBuf,
nullptr);
1976 readbackBuf = VK_NULL_HANDLE;
1978 if (readbackMem != VK_NULL_HANDLE) {
1979 vkFreeMemory(
device, readbackMem,
nullptr);
1980 readbackMem = VK_NULL_HANDLE;