Android auto streaming working
This commit is contained in:
Anonymous
@@ -34,3 +34,7 @@ protobuf-codegen = "3.7"
|
||||
# Remote ESP-IDF components (managed by idf component manager)
|
||||
[[package.metadata.esp-idf-sys.extra_components]]
|
||||
remote_component = { name = "espressif/mdns", version = "1.4" }
|
||||
|
||||
[[package.metadata.esp-idf-sys.extra_components]]
|
||||
remote_component = { name = "espressif/esp_h264", version = "1.3.0" }
|
||||
bindings_header = "src/esp_h264_bindings.h"
|
||||
|
||||
+15
-1
@@ -1,4 +1,17 @@
|
||||
dependencies:
|
||||
espressif/esp_h264:
|
||||
component_hash: 40cb64e697a8cfd787453165f6ce54e1fb8601844448eb073d677b9f9fde071e
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=4.4'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
targets:
|
||||
- esp32s3
|
||||
- esp32p4
|
||||
version: 1.3.0
|
||||
espressif/mdns:
|
||||
component_hash: d36b265164be5139f92de993f08f5ecaa0de0c0acbf84deee1f10bb5902d04ff
|
||||
dependencies:
|
||||
@@ -14,7 +27,8 @@ dependencies:
|
||||
type: idf
|
||||
version: 5.5.1
|
||||
direct_dependencies:
|
||||
- espressif/esp_h264
|
||||
- espressif/mdns
|
||||
manifest_hash: e7f1c61e6135aecdb38fc4be6c0a74a0b5dab2667145d593f168f8cd7832abc8
|
||||
manifest_hash: f7d9e17895d8211525d0fb2d64157259171e1c31e740e45b3b8dd0b09c0d010e
|
||||
target: esp32s3
|
||||
version: 2.0.0
|
||||
|
||||
+222
-91
@@ -1,18 +1,116 @@
|
||||
//! H.264 software decoder using Espressif's esp_h264 component.
|
||||
//! H.264 software decoder using Espressif's esp_h264 component (v1.3.0).
|
||||
//!
|
||||
//! Decodes H.264 NAL units from Android Auto's video channel into
|
||||
//! RGB565 framebuffer data suitable for the ST7796 display.
|
||||
//!
|
||||
//! Pipeline: H.264 NAL → esp_h264 decoder → I420 (YUV) → downscale → RGB565
|
||||
//! Pipeline: H.264 NAL → esp_h264 SW decoder → I420 (YUV) → downscale → RGB565
|
||||
//!
|
||||
//! Performance on ESP32-S3 (dual-task decoder):
|
||||
//! 320×192 → ~27 fps
|
||||
//! 640×480 → ~11 fps
|
||||
//! Performance on ESP32-S3R8 (SW decoder, from Espressif benchmarks):
|
||||
//! 640×480 → ~9 fps (mono task) / ~11 fps (dual task)
|
||||
//! 320×192 → ~23 fps (mono task) / ~27 fps (dual task)
|
||||
//!
|
||||
//! Android Auto sends 800×480 minimum, so we decode and downscale to
|
||||
//! fit the 480×320 display.
|
||||
//!
|
||||
//! # FFI bindings
|
||||
//!
|
||||
//! The `esp_h264` component is a C library registered in the Espressif IDF
|
||||
//! Component Registry (espressif/esp_h264 v1.3.0). Because it is a C
|
||||
//! component — not a Rust crate — we cannot use `extern crate`. Instead,
|
||||
//! `esp-idf-sys` generates bindgen Rust FFI declarations from
|
||||
//! `src/esp_h264_bindings.h` which includes `esp_h264_dec_sw.h`.
|
||||
//!
|
||||
//! The generated symbols live in the `esp_idf_sys` crate root after the
|
||||
//! component is linked into the final firmware binary.
|
||||
|
||||
use anyhow::{Result, bail, Context};
|
||||
use anyhow::{bail, Context, Result};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Manual FFI bindings for espressif/esp_h264 v1.3.0
|
||||
//
|
||||
// The component is a C library (not a Rust crate), compiled as part of the
|
||||
// ESP-IDF build via the IDF Component Manager. We declare the C types and
|
||||
// function prototypes here manually so the code compiles regardless of
|
||||
// whether bindgen has run yet. The linker resolves the symbols from the
|
||||
// static library produced when the component is compiled.
|
||||
//
|
||||
// Struct layouts verified against:
|
||||
// esp_h264/interface/include/esp_h264_types.h
|
||||
// esp_h264/interface/include/esp_h264_dec.h
|
||||
// esp_h264/sw/include/esp_h264_dec_sw.h
|
||||
// ---------------------------------------------------------------------------
|
||||
mod ffi {
|
||||
// Error codes (esp_h264_err_t)
|
||||
pub const ESP_H264_ERR_OK: i32 = 0;
|
||||
|
||||
// I420 pixel format — ESP_H264_4CC('Y','U','1','2') = 0x32315559
|
||||
pub const ESP_H264_RAW_FMT_I420: u32 =
|
||||
('Y' as u32) | (('U' as u32) << 8) | (('1' as u32) << 16) | (('2' as u32) << 24);
|
||||
|
||||
// esp_h264_dec_handle_t: opaque pointer to the decoder vtable struct
|
||||
pub type EspH264DecHandle = *mut core::ffi::c_void;
|
||||
|
||||
// esp_h264_pkt_t: data buffer + length
|
||||
#[repr(C)]
|
||||
pub struct EspH264Pkt {
|
||||
pub buffer: *mut u8,
|
||||
pub len: u32,
|
||||
}
|
||||
|
||||
// esp_h264_dec_cfg_t / esp_h264_dec_cfg_sw_t (identical types)
|
||||
#[repr(C)]
|
||||
pub struct EspH264DecCfg {
|
||||
pub pic_type: u32, // esp_h264_raw_format_t enum value
|
||||
}
|
||||
|
||||
// esp_h264_dec_in_frame_t
|
||||
#[repr(C)]
|
||||
pub struct EspH264DecInFrame {
|
||||
pub raw_data: EspH264Pkt, // encoded input data
|
||||
pub consume: u32, // bytes consumed on return
|
||||
pub dts: u32,
|
||||
pub pts: u32,
|
||||
}
|
||||
|
||||
// esp_h264_dec_out_frame_t
|
||||
#[repr(C)]
|
||||
pub struct EspH264DecOutFrame {
|
||||
pub frame_type: i32, // esp_h264_frame_type_t (-1=invalid, 0=IDR, 1=I, 2=P)
|
||||
pub outbuf: *mut u8, // decoded I420 data (component-owned, valid until next call)
|
||||
pub out_size: u32, // 0 for SPS/PPS/SEI, non-zero for image frames
|
||||
pub dts: u32,
|
||||
pub pts: u32,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
extern "C" {
|
||||
/// Create a new software H.264 decoder instance.
|
||||
/// Must be followed by `esp_h264_dec_open` before use.
|
||||
pub fn esp_h264_dec_sw_new(
|
||||
cfg: *const EspH264DecCfg,
|
||||
out_dec: *mut EspH264DecHandle,
|
||||
) -> i32;
|
||||
|
||||
/// Initialise the decoder (allocates internal buffers).
|
||||
pub fn esp_h264_dec_open(dec: EspH264DecHandle) -> i32;
|
||||
|
||||
/// Decode one NAL unit.
|
||||
/// `in_frame.consume` is set to the number of bytes consumed.
|
||||
/// `out_frame.outbuf` / `out_frame.out_size` are set when a full
|
||||
/// image frame is ready; `out_size == 0` for SPS/PPS/SEI.
|
||||
pub fn esp_h264_dec_process(
|
||||
dec: EspH264DecHandle,
|
||||
in_frame: *mut EspH264DecInFrame,
|
||||
out_frame: *mut EspH264DecOutFrame,
|
||||
) -> i32;
|
||||
|
||||
/// Flush and uninitialise the decoder.
|
||||
pub fn esp_h264_dec_close(dec: EspH264DecHandle) -> i32;
|
||||
|
||||
/// Free the decoder instance.
|
||||
pub fn esp_h264_dec_del(dec: EspH264DecHandle) -> i32;
|
||||
}
|
||||
}
|
||||
|
||||
/// Display dimensions (WT32-SC01 Plus: ST7796 480×320)
|
||||
pub const DISPLAY_WIDTH: u32 = 480;
|
||||
@@ -42,22 +140,25 @@ impl Default for DecoderConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// H.264 decoder state.
|
||||
/// H.264 decoder backed by the espressif/esp_h264 C component (v1.3.0).
|
||||
///
|
||||
/// Wraps the esp_h264 C decoder API via unsafe FFI.
|
||||
/// All large buffers are allocated in PSRAM (8MB) to avoid internal SRAM OOM.
|
||||
/// Lifecycle: `new()` creates and opens the decoder once.
|
||||
/// Each `decode()` call feeds one NAL unit and returns an RGB565 frame
|
||||
/// when a complete image frame is ready (IDR/P). SPS/PPS/SEI NALs return
|
||||
/// `Ok(None)` — that is normal and expected at stream start.
|
||||
///
|
||||
/// The decoder's output I420 buffer is managed internally by the C library.
|
||||
/// Only the RGB565 output framebuffer is allocated here (in PSRAM).
|
||||
pub struct H264Decoder {
|
||||
config: DecoderConfig,
|
||||
/// Decoded I420 buffer (Y + U + V planes) — in PSRAM
|
||||
i420_buf: PsramBuf,
|
||||
/// Output RGB565 framebuffer (target dimensions) — in PSRAM
|
||||
/// esp_h264 decoder handle (opaque C pointer)
|
||||
handle: ffi::EspH264DecHandle,
|
||||
/// Output RGB565 framebuffer (480×320 × 2 bytes) — in PSRAM
|
||||
rgb565_buf: PsramBuf,
|
||||
/// NAL unit accumulator (H.264 frames arrive in chunks) — in PSRAM
|
||||
nal_buf: PsramBuf,
|
||||
/// Current write position in nal_buf
|
||||
nal_len: usize,
|
||||
/// Total frames decoded (for stats)
|
||||
/// Total frames that produced image output
|
||||
frames_decoded: u64,
|
||||
/// Bytes of H.264 data received (for the periodic log in session.rs)
|
||||
total_bytes_fed: usize,
|
||||
}
|
||||
|
||||
/// A buffer allocated in PSRAM via heap_caps_malloc.
|
||||
@@ -83,14 +184,6 @@ impl PsramBuf {
|
||||
Ok(Self { ptr: ptr as *mut u8, capacity: size })
|
||||
}
|
||||
|
||||
fn as_slice(&self, len: usize) -> &[u8] {
|
||||
unsafe { std::slice::from_raw_parts(self.ptr, len.min(self.capacity)) }
|
||||
}
|
||||
|
||||
fn as_mut_slice(&mut self) -> &mut [u8] {
|
||||
unsafe { std::slice::from_raw_parts_mut(self.ptr, self.capacity) }
|
||||
}
|
||||
|
||||
fn as_u16_slice(&self, pixel_count: usize) -> &[u16] {
|
||||
unsafe { std::slice::from_raw_parts(self.ptr as *const u16, pixel_count) }
|
||||
}
|
||||
@@ -107,97 +200,133 @@ impl Drop for PsramBuf {
|
||||
}
|
||||
|
||||
impl H264Decoder {
|
||||
/// Create a new decoder instance. All large buffers go to PSRAM.
|
||||
/// Create and open a new software H.264 decoder.
|
||||
///
|
||||
/// Calls `esp_h264_dec_sw_new` + `esp_h264_dec_open`. Allocates only
|
||||
/// the RGB565 output framebuffer in PSRAM; all I420 buffers are managed
|
||||
/// internally by the C component.
|
||||
pub fn new(config: DecoderConfig) -> Result<Self> {
|
||||
let i420_size = (config.source_width * config.source_height * 3 / 2) as usize;
|
||||
let rgb565_size = (config.target_width * config.target_height * 2) as usize;
|
||||
let nal_capacity = 512 * 1024; // 512KB for keyframes
|
||||
let rgb565_buf = PsramBuf::new(rgb565_size)
|
||||
.context("allocating RGB565 output buffer in PSRAM")?;
|
||||
|
||||
log::info!(
|
||||
"H.264 decoder: {}×{} → {}×{} (I420: {}KB, RGB565: {}KB, NAL: {}KB) — all PSRAM",
|
||||
config.source_width, config.source_height,
|
||||
config.target_width, config.target_height,
|
||||
i420_size / 1024, rgb565_size / 1024, nal_capacity / 1024,
|
||||
);
|
||||
let cfg = ffi::EspH264DecCfg {
|
||||
pic_type: ffi::ESP_H264_RAW_FMT_I420,
|
||||
};
|
||||
let mut handle: ffi::EspH264DecHandle = core::ptr::null_mut();
|
||||
|
||||
let i420_buf = PsramBuf::new(i420_size).context("allocating I420 buffer in PSRAM")?;
|
||||
let rgb565_buf = PsramBuf::new(rgb565_size).context("allocating RGB565 buffer in PSRAM")?;
|
||||
let nal_buf = PsramBuf::new(nal_capacity).context("allocating NAL buffer in PSRAM")?;
|
||||
let ret = unsafe { ffi::esp_h264_dec_sw_new(&cfg, &mut handle) };
|
||||
if ret != ffi::ESP_H264_ERR_OK {
|
||||
bail!("esp_h264_dec_sw_new failed: err={}", ret);
|
||||
}
|
||||
|
||||
let ret = unsafe { ffi::esp_h264_dec_open(handle) };
|
||||
if ret != ffi::ESP_H264_ERR_OK {
|
||||
unsafe { ffi::esp_h264_dec_del(handle); }
|
||||
bail!("esp_h264_dec_open failed: err={}", ret);
|
||||
}
|
||||
|
||||
let free_psram = unsafe {
|
||||
esp_idf_sys::heap_caps_get_free_size(esp_idf_sys::MALLOC_CAP_SPIRAM)
|
||||
};
|
||||
log::info!("PSRAM free after decoder init: {} KB", free_psram / 1024);
|
||||
log::info!(
|
||||
"H.264 decoder ready: {}×{} → {}×{}, RGB565 {}KB in PSRAM, {} KB PSRAM free",
|
||||
config.source_width, config.source_height,
|
||||
config.target_width, config.target_height,
|
||||
rgb565_size / 1024,
|
||||
free_psram / 1024,
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
i420_buf,
|
||||
handle,
|
||||
rgb565_buf,
|
||||
nal_buf,
|
||||
nal_len: 0,
|
||||
frames_decoded: 0,
|
||||
total_bytes_fed: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Feed H.264 data (NAL units) from Android Auto.
|
||||
/// Returns Some(rgb565 framebuffer slice) when a complete frame is decoded.
|
||||
/// Feed one H.264 NAL unit from Android Auto and decode it.
|
||||
///
|
||||
/// Returns `Ok(Some(rgb565))` when an image frame was decoded (IDR or P).
|
||||
/// Returns `Ok(None)` for SPS / PPS / SEI NALs — this is expected and
|
||||
/// normal at the start of a stream.
|
||||
///
|
||||
/// The returned slice is valid until the next call to `decode()`.
|
||||
pub fn decode(&mut self, h264_data: &[u8]) -> Result<Option<&[u16]>> {
|
||||
// Accumulate NAL data into PSRAM buffer
|
||||
let new_len = self.nal_len + h264_data.len();
|
||||
if new_len > self.nal_buf.capacity {
|
||||
log::warn!("NAL overflow, discarding {} bytes", self.nal_len);
|
||||
self.nal_len = 0;
|
||||
self.total_bytes_fed += h264_data.len();
|
||||
|
||||
// Build input frame pointing directly at the caller's slice.
|
||||
// The C API reads (does not write) this buffer, so casting away
|
||||
// const is safe for the duration of the synchronous FFI call.
|
||||
let mut in_frame = ffi::EspH264DecInFrame {
|
||||
raw_data: ffi::EspH264Pkt {
|
||||
buffer: h264_data.as_ptr() as *mut u8,
|
||||
len: h264_data.len() as u32,
|
||||
},
|
||||
consume: 0,
|
||||
dts: 0,
|
||||
pts: 0,
|
||||
};
|
||||
let mut out_frame = ffi::EspH264DecOutFrame {
|
||||
frame_type: -1,
|
||||
outbuf: core::ptr::null_mut(),
|
||||
out_size: 0,
|
||||
dts: 0,
|
||||
pts: 0,
|
||||
};
|
||||
|
||||
let ret = unsafe {
|
||||
ffi::esp_h264_dec_process(self.handle, &mut in_frame, &mut out_frame)
|
||||
};
|
||||
if ret != ffi::ESP_H264_ERR_OK {
|
||||
log::warn!("esp_h264_dec_process error: {}", ret);
|
||||
return Ok(None);
|
||||
}
|
||||
unsafe {
|
||||
std::ptr::copy_nonoverlapping(
|
||||
h264_data.as_ptr(),
|
||||
self.nal_buf.ptr.add(self.nal_len),
|
||||
h264_data.len(),
|
||||
);
|
||||
|
||||
// SPS / PPS / SEI — no image data, completely normal
|
||||
if out_frame.out_size == 0 || out_frame.outbuf.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.nal_len = new_len;
|
||||
|
||||
// TODO: Feed to esp_h264 decoder (bindings not yet generated)
|
||||
// Generate a test pattern frame periodically to verify display pipeline
|
||||
if self.nal_len > 32 * 1024 {
|
||||
self.nal_len = 0;
|
||||
self.frames_decoded += 1;
|
||||
// outbuf points to component-owned I420 data, valid until next call
|
||||
let i420 = unsafe {
|
||||
core::slice::from_raw_parts(out_frame.outbuf, out_frame.out_size as usize)
|
||||
};
|
||||
|
||||
// Generate alternating color test pattern to prove display works
|
||||
let pixels = self.config.target_width * self.config.target_height;
|
||||
let rgb565 = self.rgb565_buf.as_u16_mut_slice(pixels as usize);
|
||||
let color = match self.frames_decoded % 4 {
|
||||
0 => 0x07E0u16, // Green
|
||||
1 => 0x001Fu16, // Blue
|
||||
2 => 0xF800u16, // Red
|
||||
_ => 0xFFFFu16, // White
|
||||
};
|
||||
for px in rgb565.iter_mut() {
|
||||
*px = color;
|
||||
}
|
||||
let pixels = (self.config.target_width * self.config.target_height) as usize;
|
||||
let rgb565 = self.rgb565_buf.as_u16_mut_slice(pixels);
|
||||
|
||||
i420_to_rgb565_downscale(
|
||||
i420,
|
||||
self.config.source_width,
|
||||
self.config.source_height,
|
||||
rgb565,
|
||||
self.config.target_width,
|
||||
self.config.target_height,
|
||||
);
|
||||
|
||||
self.frames_decoded += 1;
|
||||
if self.frames_decoded % 30 == 1 {
|
||||
log::info!(
|
||||
"🎨 Test frame #{} (color=0x{:04X}, {}x{})",
|
||||
self.frames_decoded, color,
|
||||
self.config.target_width, self.config.target_height
|
||||
"🎬 Frame #{} decoded ({}×{} I420 → {}×{} RGB565, {} KB fed total)",
|
||||
self.frames_decoded,
|
||||
self.config.source_width, self.config.source_height,
|
||||
self.config.target_width, self.config.target_height,
|
||||
self.total_bytes_fed / 1024,
|
||||
);
|
||||
return Ok(Some(self.rgb565_buf.as_u16_slice(pixels as usize)));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
Ok(Some(self.rgb565_buf.as_u16_slice(pixels)))
|
||||
}
|
||||
|
||||
/// Get the number of frames decoded so far.
|
||||
pub fn frames_decoded(&self) -> u64 {
|
||||
self.frames_decoded
|
||||
}
|
||||
/// Total image frames decoded successfully.
|
||||
pub fn frames_decoded(&self) -> u64 { self.frames_decoded }
|
||||
|
||||
/// Get the current NAL accumulator length.
|
||||
pub fn nal_len(&self) -> usize {
|
||||
self.nal_len
|
||||
}
|
||||
/// Total bytes of H.264 data fed so far (used for logging in session.rs).
|
||||
pub fn nal_len(&self) -> usize { self.total_bytes_fed }
|
||||
|
||||
/// Get the output framebuffer dimensions.
|
||||
/// Output framebuffer dimensions (width, height).
|
||||
pub fn output_dimensions(&self) -> (u32, u32) {
|
||||
(self.config.target_width, self.config.target_height)
|
||||
}
|
||||
@@ -205,11 +334,13 @@ impl H264Decoder {
|
||||
|
||||
impl Drop for H264Decoder {
|
||||
fn drop(&mut self) {
|
||||
// TODO: esp_h264_dec_close(self.decoder_handle);
|
||||
log::info!(
|
||||
"H.264 decoder closed after {} frames",
|
||||
self.frames_decoded
|
||||
);
|
||||
if !self.handle.is_null() {
|
||||
unsafe {
|
||||
ffi::esp_h264_dec_close(self.handle);
|
||||
ffi::esp_h264_dec_del(self.handle);
|
||||
}
|
||||
}
|
||||
log::info!("H.264 decoder closed after {} frames", self.frames_decoded);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
// Bindings header for esp_h264 component (v1.3.0)
|
||||
// Generates Rust FFI bindings for the software H.264 decoder API.
|
||||
// Headers are located in esp_h264/interface/include/ within the component.
|
||||
#include "esp_h264_dec_sw.h"
|
||||
Reference in New Issue
Block a user