MXVK Vulkan Framework 0.35.0
C++20 Vulkan rendering framework for practical 2D and 3D application development with SDL3.
Loading...
Searching...
No Matches
mxvk::VK_Capture Class Reference

Vulkan OpenCV video capture source. More...

#include <mxvk/include/mxvk/mxvk_cv.hpp>

Public Member Functions

void close ()
 Close the capture device.
bool createImage (VkDevice device, VkPhysicalDevice physDev, VkQueue gQueue, VkCommandPool cmdPool, size_t width, size_t height, const std::string &vert, const std::string &frag)
 Allocate the VKSprite and upload the initial texture.
void draw (int x, int y)
 Draw using the sprite's native dimensions.
void draw (int x, int y, int width, int height)
 Draw the current frame at a specified position and size.
double get (unsigned int option)
 Query a VideoCapture property.
VK_Sprite * getSprite ()
 Access the backing sprite.
bool grab ()
 Decode and discard the next capture frame without conversion.
bool is_open () const
 Check whether the capture device is open.
bool open (const std::string &filename)
 Open a video file.
bool open (int id, int mode=0)
 Open a camera device.
VK_Capture & operator= (const VK_Capture &)=delete
VK_Capture & operator= (VK_Capture &&)=delete
bool read ()
 Capture and upload the next video frame.
bool read (cv::Mat &frame)
 Capture the next frame into a cv::Mat.
bool readRgba (cv::Mat &rgba, bool flipY=false)
bool readToModelTexture (VKAbstractModel &model, bool flipY=false)
bool readToSprite (VK_Sprite &targetSprite)
bool readToSprite (VK_Sprite &targetSprite, bool flipY)
bool reload (size_t width, size_t height, const std::string &vert, const std::string &frag)
 Replace the vertex/fragment shaders.
void resetSprite ()
 Destroy the current sprite and its Vulkan resources.
void set (unsigned int option, double value)
 Set a VideoCapture property.
 VK_Capture ()=default
 Default constructor.
 VK_Capture (const VK_Capture &)=delete
 VK_Capture (VK_Capture &&)=delete
 ~VK_Capture ()=default
 Destructor.

Detailed Description

Vulkan OpenCV video capture source.

Opens a video file or camera, decodes frames, and uploads them to a VKSprite texture for Vulkan-accelerated rendering inside a VKWindow.

Definition at line 30 of file mxvk_cv.hpp.

Constructor & Destructor Documentation

◆ VK_Capture() [1/3]

mxvk::VK_Capture::VK_Capture ( )
default

Default constructor.

◆ ~VK_Capture()

mxvk::VK_Capture::~VK_Capture ( )
default

Destructor.

◆ VK_Capture() [2/3]

mxvk::VK_Capture::VK_Capture ( const VK_Capture & )
delete

◆ VK_Capture() [3/3]

mxvk::VK_Capture::VK_Capture ( VK_Capture && )
delete

Member Function Documentation

◆ close()

void mxvk::VK_Capture::close ( )

Close the capture device.

Definition at line 37 of file mxvk_cv.cpp.

37 {
38 cap.release();
39 std::cout << "mxvk_cv: Capture closed\n";
40 }

◆ createImage()

bool mxvk::VK_Capture::createImage ( VkDevice device,
VkPhysicalDevice physDev,
VkQueue gQueue,
VkCommandPool cmdPool,
size_t width,
size_t height,
const std::string & vert,
const std::string & frag )

Allocate the VKSprite and upload the initial texture.

Parameters
deviceLogical Vulkan device.
physDevPhysical device.
gQueueGraphics queue.
cmdPoolCommand pool.
widthDisplay width.
heightDisplay height.
vertVertex shader path.
fragFragment shader path.
Returns
true on success.

Definition at line 44 of file mxvk_cv.cpp.

44 {
45 sprite = std::make_unique<VK_Sprite>(device, physDev, gQueue, cmdPool);
46 sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
47 sprite->enableExtendedUBO();
48 sprite->rebuildPipeline();
49 std::cout << std::format("mxvk_cv: Sprite created: {}x{}\n", width, height);
50 return true;
51 }

◆ draw() [1/2]

void mxvk::VK_Capture::draw ( int x,
int y )

Draw using the sprite's native dimensions.

Parameters
xDestination X.
yDestination Y.

Definition at line 450 of file mxvk_cv.cpp.

450 {
451 if (sprite)
452 sprite->drawSprite(x, y);
453 }

◆ draw() [2/2]

void mxvk::VK_Capture::draw ( int x,
int y,
int width,
int height )

Draw the current frame at a specified position and size.

Parameters
xWidth destination X.
yWidth destination Y.
widthDisplay width.
heightDisplay height.

Definition at line 445 of file mxvk_cv.cpp.

445 {
446 if (sprite)
447 sprite->drawSpriteRect(x, y, width, height);
448 }

◆ get()

double mxvk::VK_Capture::get ( unsigned int option)

Query a VideoCapture property.

Parameters
optioncv::VideoCaptureProperties constant.
Returns
Current value.

Definition at line 457 of file mxvk_cv.cpp.

457{ return cap.get(option); }

◆ getSprite()

VK_Sprite * mxvk::VK_Capture::getSprite ( )
inline

Access the backing sprite.

Returns
Pointer to the VKSprite (may be null before createImage).

Definition at line 91 of file mxvk_cv.hpp.

91{ return sprite.get(); }

◆ grab()

bool mxvk::VK_Capture::grab ( )

Decode and discard the next capture frame without conversion.

Returns
true when the backend advanced to another frame.

Definition at line 66 of file mxvk_cv.cpp.

66{ return cap.grab(); }

◆ is_open()

bool mxvk::VK_Capture::is_open ( ) const
inline

Check whether the capture device is open.

Returns
true if the VideoCapture is opened.

Definition at line 71 of file mxvk_cv.hpp.

71{ return cap.isOpened(); }

◆ open() [1/2]

bool mxvk::VK_Capture::open ( const std::string & filename)

Open a video file.

Parameters
filenamePath to the video file.
Returns
true on success.

Definition at line 14 of file mxvk_cv.cpp.

14 {
15 const bool ok = cap.open(filename);
16 if (ok)
17 std::cout << std::format("mxvk_cv: Opened file: {}\n", filename);
18 else
19 std::cout << std::format("mxvk_cv: Failed to open file: {}\n", filename);
20 return ok;
21 }

◆ open() [2/2]

bool mxvk::VK_Capture::open ( int id,
int mode = 0 )

Open a camera device.

Parameters
idCamera index.
modeBackend hint (0 = auto).
Returns
true on success.

Definition at line 23 of file mxvk_cv.cpp.

23 {
24#ifdef __linux__
25 if (mode == 0)
26 mode = cv::CAP_V4L2;
27#endif
28 if (cap.open(id, mode)) {
29 cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'));
30 std::cout << std::format("mxvk_cv: Opened device: {}\n", id);
31 return true;
32 }
33 std::cout << std::format("mxvk_cv: Failed to open device: {}\n", id);
34 return false;
35 }

◆ operator=() [1/2]

VK_Capture & mxvk::VK_Capture::operator= ( const VK_Capture & )
delete

◆ operator=() [2/2]

VK_Capture & mxvk::VK_Capture::operator= ( VK_Capture && )
delete

◆ read() [1/2]

bool mxvk::VK_Capture::read ( )

Capture and upload the next video frame.

Returns
true if a frame was available.

Definition at line 430 of file mxvk_cv.cpp.

430 {
431#ifdef MXVK_CUDA
432 if (initializeCuda()) {
433 return readCuda();
434 }
435#endif
436 if (!cap.read(frame)) {
437 return false;
438 }
439 cv::Mat rgba;
440 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
441 sprite->updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
442 return true;
443 }

◆ read() [2/2]

bool mxvk::VK_Capture::read ( cv::Mat & frame)

Capture the next frame into a cv::Mat.

Parameters
frameOutput matrix.
Returns
true if a frame was available.

Definition at line 64 of file mxvk_cv.cpp.

64{ return cap.read(frame); }

◆ readRgba()

bool mxvk::VK_Capture::readRgba ( cv::Mat & rgba,
bool flipY = false )

Definition at line 68 of file mxvk_cv.cpp.

68 {
69#ifdef MXVK_CUDA
70 cv::Mat cpuFallbackFrame;
71
72 try {
73 if (initializeCuda()) {
74 if (cudaMappedInput) {
75 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
76 return false;
77 }
78 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
79 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
80 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
81 } else {
82 if (!cap.read(frame) || frame.empty()) {
83 return false;
84 }
85 cpuFallbackFrame = frame;
86 gpuFrame.upload(frame, cuda_stream);
87 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
88 }
89
90 if (flipY) {
91 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
92 } else {
93 gpuVulkanRgba = gpuRgba;
94 }
95
96 if (!cudaPipelineLogged) {
97 std::cout << std::format("mxvk_cv: CUDA RGBA path active: capture -> GpuMat -> cv::cuda::cvtColor(BGR to RGBA) -> {} -> download\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
98 cudaPipelineLogged = true;
99 }
100
101 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
102 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
103 cuda_stream.waitForCompletion();
104 rgba = host_mem_mat_header(pinnedRgba);
105 return true;
106 }
107 } catch (const cv::Exception &e) {
108 std::cout << std::format("mxvk_cv: CUDA RGBA path failed; falling back to CPU path: {}\n", e.what());
109 cudaAvailable = false;
110 if (cpuFallbackFrame.empty()) {
111 return false;
112 }
113 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
114 if (flipY) {
115 cv::flip(rgba, rgba, 0);
116 }
117 return true;
118 }
119#endif
120
121 if (!cap.read(frame) || frame.empty()) {
122 return false;
123 }
124 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
125 if (flipY) {
126 cv::flip(rgba, rgba, 0);
127 }
128 return true;
129 }

◆ readToModelTexture()

bool mxvk::VK_Capture::readToModelTexture ( VKAbstractModel & model,
bool flipY = false )

Definition at line 278 of file mxvk_cv.cpp.

278 {
279#ifdef MXVK_CUDA
280 if (initializeCuda()) {
281 cv::Mat cpuFallbackFrame;
282
283 try {
284 if (cudaMappedInput) {
285 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
286 return false;
287 }
288 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
289 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
290 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
291 } else {
292 if (!cap.read(frame) || frame.empty()) {
293 return false;
294 }
295 cpuFallbackFrame = frame;
296 gpuFrame.upload(frame, cuda_stream);
297 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
298 }
299
300 if (flipY) {
301 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
302 } else {
303 gpuVulkanRgba = gpuRgba;
304 }
305
306 if (model.updatePrimaryTextureCuda(gpuVulkanRgba, cuda_stream)) {
307 if (!cudaPipelineLogged) {
308 std::cout << std::format("mxvk_cv: CUDA interop active for model texture: RGBA GpuMat -> {} -> Vulkan external-memory image array (no download)\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
309 cudaPipelineLogged = true;
310 }
311 return true;
312 }
313
314 if (!cudaPipelineLogged) {
315 std::cout << "mxvk_cv: CUDA interop unavailable for model texture; using pinned CUDA download -> Vulkan staging upload\n";
316 cudaPipelineLogged = true;
317 }
318 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
319 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
320 cuda_stream.waitForCompletion();
321
322 pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
323 return model.updatePrimaryTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows, static_cast<int>(pinnedRgbaMat.step));
324 } catch (const cv::Exception &e) {
325 std::cout << std::format("mxvk_cv: CUDA model-texture path failed; falling back to CPU path: {}\n", e.what());
326 cudaAvailable = false;
327 if (cpuFallbackFrame.empty()) {
328 return false;
329 }
330 cv::Mat rgba;
331 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
332 if (flipY) {
333 cv::flip(rgba, rgba, 0);
334 }
335 return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
336 }
337 }
338#endif
339 cv::Mat rgba;
340 if (!readRgba(rgba, flipY)) {
341 return false;
342 }
343 return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
344 }
mxvk::VK_Capture::readRgba
bool readRgba(cv::Mat &rgba, bool flipY=false)
Definition mxvk_cv.cpp:68

◆ readToSprite() [1/2]

bool mxvk::VK_Capture::readToSprite ( VK_Sprite & targetSprite)

Definition at line 346 of file mxvk_cv.cpp.

346 {
347#ifdef MXVK_CUDA
348 if (initializeCuda()) {
349 return readToSprite(targetSprite, cudaFlipYForVulkan);
350 }
351#endif
352 return readToSprite(targetSprite, false);
353 }
mxvk::VK_Capture::readToSprite
bool readToSprite(VK_Sprite &targetSprite)
Definition mxvk_cv.cpp:346

◆ readToSprite() [2/2]

bool mxvk::VK_Capture::readToSprite ( VK_Sprite & targetSprite,
bool flipY )

Definition at line 355 of file mxvk_cv.cpp.

355 {
356#ifdef MXVK_CUDA
357 if (initializeCuda()) {
358 cv::Mat cpuFallbackFrame;
359
360 try {
361 if (cudaMappedInput) {
362 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
363 return false;
364 }
365 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
366 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
367 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
368 } else {
369 if (!cap.read(frame) || frame.empty()) {
370 return false;
371 }
372 cpuFallbackFrame = frame;
373 gpuFrame.upload(frame, cuda_stream);
374 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
375 }
376
377 if (flipY) {
378 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
379 } else {
380 gpuVulkanRgba = gpuRgba;
381 }
382
383 if (targetSprite.updateTextureCuda(gpuVulkanRgba, cuda_stream)) {
384 if (!cudaPipelineLogged) {
385 std::cout << std::format("mxvk_cv: CUDA interop active for external sprite: RGBA GpuMat -> {} -> Vulkan external-memory image\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
386 cudaPipelineLogged = true;
387 }
388 return true;
389 }
390
391 if (!cudaPipelineLogged) {
392 std::cout << "mxvk_cv: CUDA interop unavailable for external sprite; using pinned CUDA download -> Vulkan staging upload\n";
393 cudaPipelineLogged = true;
394 }
395 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
396 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
397 cuda_stream.waitForCompletion();
398
399 pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
400 targetSprite.updateTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows, static_cast<int>(pinnedRgbaMat.step));
401 return true;
402 } catch (const cv::Exception &e) {
403 std::cout << std::format("mxvk_cv: CUDA external-sprite path failed; falling back to CPU path: {}\n", e.what());
404 cudaAvailable = false;
405 if (cpuFallbackFrame.empty()) {
406 return false;
407 }
408 cv::Mat rgba;
409 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
410 if (flipY) {
411 cv::flip(rgba, rgba, 0);
412 }
413 targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
414 return true;
415 }
416 }
417#endif
418 if (!cap.read(frame) || frame.empty()) {
419 return false;
420 }
421 cv::Mat rgba;
422 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
423 if (flipY) {
424 cv::flip(rgba, rgba, 0);
425 }
426 targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
427 return true;
428 }

◆ reload()

bool mxvk::VK_Capture::reload ( size_t width,
size_t height,
const std::string & vert,
const std::string & frag )

Replace the vertex/fragment shaders.

Parameters
widthDisplay width.
heightDisplay height.
vertNew vertex shader path.
fragNew fragment shader path.
Returns
true on success.

Definition at line 53 of file mxvk_cv.cpp.

53 {
54 if (sprite) {
55 sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
56 sprite->enableExtendedUBO();
57 std::cout << std::format("mxvk_cv: Shader reloaded: vert={} frag={}\n", std::filesystem::path(vert).filename().string(), std::filesystem::path(frag).filename().string());
58 return true;
59 }
60 std::cout << "mxvk_cv: Reload failed: no sprite\n";
61 return false;
62 }

◆ resetSprite()

void mxvk::VK_Capture::resetSprite ( )

Destroy the current sprite and its Vulkan resources.

Call this before swapchain/command-pool teardown to avoid destroying sprite staging command buffers against an invalid command pool.

Definition at line 42 of file mxvk_cv.cpp.

42{ sprite.reset(); }

◆ set()

void mxvk::VK_Capture::set ( unsigned int option,
double value )

Set a VideoCapture property.

Parameters
optioncv::VideoCaptureProperties constant.
valueDesired value.

Definition at line 455 of file mxvk_cv.cpp.

455{ cap.set(option, value); }
The documentation for this class was generated from the following files: