PI4 Server to OpenCV Windows Client Blank Screen Problem

We are using a PI 4 to Windows 11 setup.  The Windows camera app and the OBS app works very well.

However, we can't get it to work in OpenCV.  An OpenCV expert and I have spent at least 5 hours with Gemini and ChatGPT and we still have not been able to get it to work and always get a blank screen.  

We have forced DirectShow, explicitly Set Format Before Reading, hard-Flush Frames on Startup, assure One Process Owns the Camera, among many other things.  Still blank screen.

Any help would be GREATLY appreciated.  Here is our test file:

import subprocess
import time
import sys
import signal
import cv2
import numpy as np


def open_ffmpeg_dshow(device_name: str, width: int, height: int, fps: int):
   """
   Starts FFmpeg reading from DirectShow and outputting raw BGR frames to stdout.
   This bypasses OpenCV's webcam decode path (which is where VirtualHere often breaks).
   """
   # Notes:
   # - video_size + framerate requests the mode (driver may choose closest).
   # - pixel_format "mjpeg" can help sometimes, but leave it out initially to let dshow choose.
   #   If you want to force MJPEG later, uncomment: "-input_format", "mjpeg"
   cmd = [
       "ffmpeg",
       "-hide_banner",
       "-loglevel", "error",
       "-f", "dshow",
       "-rtbufsize", "256M",
       "-framerate", str(fps),
       "-video_size", f"{width}x{height}",
       # "-input_format", "mjpeg",  # optional experiment
       "-i", f"video={device_name}",
       "-an",
       "-pix_fmt", "bgr24",
       "-f", "rawvideo",
       "pipe:1",
   ]
   # Use a new process group so we can terminate it cleanly on Windows
   creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform.startswith("win") else 0
   p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE,
                        bufsize=10**8, creationflags=creationflags)
   return p


def main():
   # === SET THESE ===
   DEVICE_NAME = "HDMI USB Camera"
   WIDTH, HEIGHT, FPS = 960, 540, 30

   frame_bytes = WIDTH * HEIGHT * 3

   print(f"Starting FFmpeg DirectShow capture: {DEVICE_NAME} @ {WIDTH}x{HEIGHT}@{FPS}", flush=True)
   p = open_ffmpeg_dshow(DEVICE_NAME, WIDTH, HEIGHT, FPS)

   win = "VirtualHere via FFmpeg (bypasses OpenCV capture)"
   cv2.namedWindow(win, cv2.WINDOW_NORMAL)
   cv2.resizeWindow(win, max(640, WIDTH), max(360, HEIGHT))
   cv2.waitKey(1)

   frames = 0
   t0 = time.time()

   try:
       while True:
           # Read exactly one frame worth of bytes
           raw = p.stdout.read(frame_bytes)
           if raw is None or len(raw) != frame_bytes:
               # If FFmpeg died, show stderr for debugging
               err = b""
               try:
                   err = p.stderr.read()
               except Exception:
                   pass
               raise RuntimeError(f"FFmpeg stream ended (got {len(raw) if raw else 0} bytes). stderr:\n{err.decode(errors='ignore')}")

           frame = np.frombuffer(raw, dtype=np.uint8).reshape((HEIGHT, WIDTH, 3))

           frames += 1
           now = time.time()
           if now - t0 >= 1.0:
               print(f"FPS: {frames}", flush=True)
               frames = 0
               t0 = now

           cv2.imshow(win, frame)
           if (cv2.waitKey(1) & 0xFF) == ord("q"):
               break

   except KeyboardInterrupt:
       print("KeyboardInterrupt. Shutting down...", flush=True)

   finally:
       # Clean shutdown: close window, terminate ffmpeg
       try:
           cv2.destroyAllWindows()
       except Exception:
           pass

       if p and p.poll() is None:
           try:
               if sys.platform.startswith("win"):
                   # Send Ctrl+Break to the process group (more graceful than terminate)
                   p.send_signal(signal.CTRL_BREAK_EVENT)
                   time.sleep(0.5)
               p.terminate()
           except Exception:
               pass

           try:
               p.wait(timeout=2.0)
           except Exception:
               try:
                   p.kill()
               except Exception:
                   pass

       print("Clean shutdown complete.", flush=True)


if __name__ == "__main__":
   main()
 

 

#2

You will need to stream as MJEPG because there will not be enough bandwidth in the network link to stream raw frames.

#3

Hi,

I have tried with MJEPG, H264, YUV2 but OpenCV is not grabbing the frames from the camera. Here is a short snippet of code that we are trying.

 

import cv2, time

IDX = 0  # try 0/1/2 etc

cap = cv2.VideoCapture(IDX, cv2.CAP_DSHOW)  # <-- key
if not cap.isOpened():
   raise RuntimeError("Could not open camera")

# Force a codec/pixel format, the below format has been tried one by one.
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
#cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"YUY2"))
#cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"H264"))
time.sleep(0.2)

# Force resolution + fps first
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)
cap.set(cv2.CAP_PROP_FPS, 30)

 

# Warm-up: throw away initial frames (black during init is common)
for _ in range(30):
   cap.read()

while True:
   ok, frame = cap.read()
   print(frame.shape)
   if not ok or frame is None:
       continue

   cv2.imshow("cam", frame)
   if cv2.waitKey(1) & 0xFF == 27:
       break

cap.release()
cv2.destroyAllWindows()
 

OpenCV Version is 4.10.0 and Python version is 3.12. OpenCV is installed via PIP.

Can you help us with what we are missing?

#4

on the pi try

sudo sh -c 'echo 256 > /sys/module/usbcore/parameters/usbfs_memory_mb'

#6

I dont know unfortunatley. Try a reasolution of 160x120 mjpeg 1fps and see if that shows anything.

#7

We made it working by using the MSMF backend instead of DSHOW.

Thanks for the help!

#8

OK great, thanks for letting me know. That might be useful for others.