From a3c44b30b7c95fea55e2eef3e6d778fc700a399a Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Wed, 18 Mar 2026 00:01:20 +0200 Subject: [PATCH 01/11] debug_stream: text_msg: Align to latest changes to Zephyr PR changes There were some changes to the Zephyr side exception dumpping sopport in the review phase of the PR. This commint aligns to them. Signed-off-by: Jyri Sarha --- src/debug/debug_stream/debug_stream_text_msg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug/debug_stream/debug_stream_text_msg.c b/src/debug/debug_stream/debug_stream_text_msg.c index 4ee71322a3e0..358e069b15c6 100644 --- a/src/debug/debug_stream/debug_stream_text_msg.c +++ b/src/debug/debug_stream/debug_stream_text_msg.c @@ -130,7 +130,7 @@ static void ds_exception_dump(const char *format, va_list args) static int init_exception_dump_hook(void) { - set_exception_dump_hook(ds_exception_dump, ds_exception_drain); + arch_exception_set_dump_hook(ds_exception_dump, ds_exception_drain); LOG_INF("exception_dump_hook set"); return 0; } From f2388f1e62957d15b4145f78f3e39efb1977c25d Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Thu, 26 Mar 2026 18:31:47 +0200 Subject: [PATCH 02/11] debug_stream: text_msg: collect per-CPU state into cacheline-aligned struct Replace separate per-CPU arrays (ds_buf[], reports_sent_cpu[], ds_pos[]) with a single struct ds_cpu_state table aligned to CONFIG_DCACHE_LINE_SIZE to avoid false sharing between cores. Also remove the reports_sent reset in the flush path, so that once an exception dump has been sent, the sent-flag stays set and suppresses further output. Signed-off-by: Jyri Sarha --- .../debug_stream/debug_stream_text_msg.c | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/src/debug/debug_stream/debug_stream_text_msg.c b/src/debug/debug_stream/debug_stream_text_msg.c index 358e069b15c6..9de003fb486f 100644 --- a/src/debug/debug_stream/debug_stream_text_msg.c +++ b/src/debug/debug_stream/debug_stream_text_msg.c @@ -49,39 +49,46 @@ void ds_msg(const char *format, ...) * in bursts, and sending more than one record in short time makes the * host-side decoder lose track of things. */ -static struct { - struct debug_stream_text_msg msg; - char text[640]; -} __packed ds_buf[CONFIG_MP_MAX_NUM_CPUS]; -static int reports_sent_cpu[CONFIG_MP_MAX_NUM_CPUS]; -static size_t ds_pos[CONFIG_MP_MAX_NUM_CPUS]; + +/* Per-CPU state for exception dump and assert_print(). + * Cache-line aligned to avoid false sharing between cores. + */ +static struct ds_cpu_state { + struct { + struct debug_stream_text_msg msg; + char text[640]; + } __packed buf; + int reports_sent; + size_t pos; +} __aligned(CONFIG_DCACHE_LINE_SIZE) ds_cpu[CONFIG_MP_MAX_NUM_CPUS]; static void ds_exception_drain(bool flush) { unsigned int cpu = arch_proc_id(); + struct ds_cpu_state *cs = &ds_cpu[cpu]; if (flush) { - ds_pos[cpu] = 0; - reports_sent_cpu[cpu] = 0; + cs->pos = 0; return; } - if (ds_pos[cpu] == 0) + if (cs->pos == 0) return; - if (reports_sent_cpu[cpu] > 0) + if (cs->reports_sent > 0) return; - ds_buf[cpu].msg.hdr.id = DEBUG_STREAM_RECORD_ID_TEXT_MSG; - ds_buf[cpu].msg.hdr.size_words = - SOF_DIV_ROUND_UP(sizeof(ds_buf[cpu].msg) + ds_pos[cpu], - sizeof(ds_buf[cpu].msg.hdr.data[0])); + cs->buf.msg.hdr.id = DEBUG_STREAM_RECORD_ID_TEXT_MSG; + cs->buf.msg.hdr.size_words = + SOF_DIV_ROUND_UP(sizeof(cs->buf.msg) + cs->pos, + sizeof(cs->buf.msg.hdr.data[0])); + /* Make sure the possible up to 3 extra bytes at end of msg are '\0' */ - memset(ds_buf[cpu].text + ds_pos[cpu], 0, - ds_buf[cpu].msg.hdr.size_words * sizeof(ds_buf[cpu].msg.hdr.data[0]) - ds_pos[cpu]); - debug_stream_slot_send_record(&ds_buf[cpu].msg.hdr); - reports_sent_cpu[cpu] = 1; - ds_pos[cpu] = 0; + memset(cs->buf.text + cs->pos, 0, + cs->buf.msg.hdr.size_words * sizeof(cs->buf.msg.hdr.data[0]) - cs->pos); + debug_stream_slot_send_record(&cs->buf.msg.hdr); + cs->reports_sent = 1; + cs->pos = 0; } static void ds_exception_dump(const char *format, va_list args) @@ -90,11 +97,12 @@ static void ds_exception_dump(const char *format, va_list args) size_t avail; size_t written; unsigned int cpu = arch_proc_id(); + struct ds_cpu_state *cs = &ds_cpu[cpu]; - if (reports_sent_cpu[cpu] > 0) + if (cs->reports_sent > 0) return; - avail = sizeof(ds_buf[cpu].text) - ds_pos[cpu]; + avail = sizeof(cs->buf.text) - cs->pos; if (avail == 0) { ds_exception_drain(false); return; @@ -108,9 +116,9 @@ static void ds_exception_dump(const char *format, va_list args) format[0] == ' ' && format[1] == '*' && format[2] == '*' && format[3] == ' ') format += 4; - len = vsnprintf(ds_buf[cpu].text + ds_pos[cpu], avail, format, args); + len = vsnprintf(cs->buf.text + cs->pos, avail, format, args); if (len < 0) { - ds_pos[cpu] = 0; + cs->pos = 0; return; } @@ -122,9 +130,9 @@ static void ds_exception_dump(const char *format, va_list args) else written = (size_t)len; - ds_pos[cpu] += written; + cs->pos += written; - if (ds_pos[cpu] >= sizeof(ds_buf[cpu].text)) + if (cs->pos >= sizeof(cs->buf.text)) ds_exception_drain(false); } From 8b8450f5e4d151177f907e0b62e5ad4f977da58c Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Thu, 19 Mar 2026 21:27:39 +0200 Subject: [PATCH 03/11] debug_stream: text_msg: Add CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS The CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS option limit the number of debug stream slot section to lower number than the actual number of cores. In some situations a high number of cpu sections shrinks the circular buffer size so much that it limit debugging. With this option its possible to use fewer sections. The downside is that the cpus above the number of sections can not send any debug stream messages. Signed-off-by: Jyri Sarha --- src/debug/debug_stream/Kconfig | 11 +++++++++ src/debug/debug_stream/debug_stream_slot.c | 27 ++++++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/debug/debug_stream/Kconfig b/src/debug/debug_stream/Kconfig index 8756c83ef8da..9a55ccefcaab 100644 --- a/src/debug/debug_stream/Kconfig +++ b/src/debug/debug_stream/Kconfig @@ -45,6 +45,17 @@ config SOF_DEBUG_STREAM_THREAD_INFO_INTERVAL Decides how often thread info runs and checks execution cycle statistics and stack usage. +config SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS + int "Number of cpu sections slot is divided to" + default MP_MAX_NUM_CPUS + range 1 MP_MAX_NUM_CPUS + help + In some situations a high number of cpu sections shrinks the + circular buffer size so much that it limit debugging. With + this option its possible to use fewer sections. The downside + is that the cpus above the number of sections can not send + any debug stream messages. + config SOF_DEBUG_STREAM_TEXT_MSG bool "Enable text message sending through Debug-Stream" help diff --git a/src/debug/debug_stream/debug_stream_slot.c b/src/debug/debug_stream/debug_stream_slot.c index a55c1c1e116d..fece904edb5c 100644 --- a/src/debug/debug_stream/debug_stream_slot.c +++ b/src/debug/debug_stream/debug_stream_slot.c @@ -19,7 +19,7 @@ struct cpu_mutex { } __aligned(CONFIG_DCACHE_LINE_SIZE); /* CPU specific mutexes for each circular buffer */ -static struct cpu_mutex cpu_mutex[CONFIG_MP_MAX_NUM_CPUS]; +static struct cpu_mutex cpu_mutex[CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS]; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER static struct debug_stream_slot_hdr *dbg_stream_slot; @@ -54,6 +54,12 @@ debug_stream_get_circular_buffer(struct debug_stream_section_descriptor *desc, u return NULL; } + if (core >= CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS) { + LOG_DBG("No section for cpu %u >= %u ", core, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS); + return NULL; + } + *desc = hdr->section_desc[core]; LOG_DBG("Section %u (desc %u %u %u)", core, desc->core_id, desc->buf_words, desc->offset); @@ -116,18 +122,19 @@ int debug_stream_slot_send_record(struct debug_stream_record *rec) static int debug_stream_slot_init(void) { struct debug_stream_slot_hdr *hdr = debug_stream_get_slot(); - size_t hdr_size = ALIGN_UP(offsetof(struct debug_stream_slot_hdr, - section_desc[CONFIG_MP_MAX_NUM_CPUS]), - CONFIG_DCACHE_LINE_SIZE); + size_t hdr_size = ALIGN_UP( + offsetof(struct debug_stream_slot_hdr, + section_desc[CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS]), + CONFIG_DCACHE_LINE_SIZE); size_t section_area_size = ADSP_DW_SLOT_SIZE - hdr_size; size_t section_size = ALIGN_DOWN(section_area_size / - CONFIG_MP_MAX_NUM_CPUS, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS, CONFIG_DCACHE_LINE_SIZE); size_t offset = hdr_size; int i; LOG_INF("%u sections of %u bytes, hdr %u, section area %u", - CONFIG_MP_MAX_NUM_CPUS, section_size, hdr_size, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS, section_size, hdr_size, section_area_size); #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER @@ -142,9 +149,9 @@ static int debug_stream_slot_init(void) hdr->hdr.magic = DEBUG_STREAM_IDENTIFIER; hdr->hdr.hdr_size = hdr_size; - hdr->total_size = hdr_size + CONFIG_MP_MAX_NUM_CPUS * section_size; - hdr->num_sections = CONFIG_MP_MAX_NUM_CPUS; - for (i = 0; i < CONFIG_MP_MAX_NUM_CPUS; i++) { + hdr->total_size = hdr_size + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS * section_size; + hdr->num_sections = CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; + for (i = 0; i < CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; i++) { hdr->section_desc[i].core_id = i; hdr->section_desc[i].buf_words = (section_size - offsetof(struct debug_stream_circular_buf, data[0]))/ @@ -154,7 +161,7 @@ static int debug_stream_slot_init(void) i, section_size, offset); offset += section_size; } - for (i = 0; i < CONFIG_MP_MAX_NUM_CPUS; i++) { + for (i = 0; i < CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; i++) { struct debug_stream_section_descriptor desc = { 0 }; struct debug_stream_circular_buf *buf = debug_stream_get_circular_buffer(&desc, i); From cd337175030f470aeaa8e11375c7efc47f1299bc Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Thu, 12 Mar 2026 19:17:29 +0200 Subject: [PATCH 04/11] debug_stream: text_msg: add va_list-based logging helper Add ds_vamsg(const char *format, va_list ap) to the debug stream text message API and implementation. Refactor ds_msg() to forward to ds_vamsg() so formatting and record emission are handled in one shared path. This enables callers that already operate on va_list to emit debug stream text messages without rebuilding variadic arguments, and keeps the message construction logic centralized. Signed-off-by: Jyri Sarha --- src/debug/debug_stream/debug_stream_text_msg.c | 14 ++++++++++---- src/include/user/debug_stream_text_msg.h | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/debug/debug_stream/debug_stream_text_msg.c b/src/debug/debug_stream/debug_stream_text_msg.c index 9de003fb486f..ae771f3c281f 100644 --- a/src/debug/debug_stream/debug_stream_text_msg.c +++ b/src/debug/debug_stream/debug_stream_text_msg.c @@ -15,18 +15,15 @@ LOG_MODULE_REGISTER(debug_stream_text_msg); -void ds_msg(const char *format, ...) +void ds_vamsg(const char *format, va_list args) { - va_list args; struct { struct debug_stream_text_msg msg; char text[128]; } __packed buf = { 0 }; ssize_t len; - va_start(args, format); len = vsnprintf(buf.text, sizeof(buf.text), format, args); - va_end(args); if (len < 0) return; @@ -38,6 +35,15 @@ void ds_msg(const char *format, ...) debug_stream_slot_send_record(&buf.msg.hdr); } +void ds_msg(const char *format, ...) +{ + va_list args; + + va_start(args, format); + ds_vamsg(format, args); + va_end(args); +} + #if defined(CONFIG_EXCEPTION_DUMP_HOOK) /* The debug stream debug window slot is 4k, and when it is split * between the cores and the header/other overhead is removed, with 5 diff --git a/src/include/user/debug_stream_text_msg.h b/src/include/user/debug_stream_text_msg.h index 3d246e305fc3..debfaad7042e 100644 --- a/src/include/user/debug_stream_text_msg.h +++ b/src/include/user/debug_stream_text_msg.h @@ -7,6 +7,7 @@ #define __SOC_DEBUG_STREAM_TEXT_MSG_H__ #include +#include /* * Debug Stream text message. @@ -21,5 +22,6 @@ struct debug_stream_text_msg { * CONFIG_SOF_DEBUG_STREAM_TEXT_MSG to enable this function. */ void ds_msg(const char *format, ...); +void ds_vamsg(const char *format, va_list ap); #endif /* __SOC_DEBUG_STREAM_TEXT_MSG_H__ */ From 0a3e1fbf0ef47acfb6c47a6d7297d3cf49bb9b82 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Tue, 17 Mar 2026 19:51:11 +0200 Subject: [PATCH 05/11] debug_stream: text_msg: add optional assert_print() forwarding Add optional support to route assert output through debug stream text messages. Introduce Kconfig option SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT (depends on EXCEPTION_DUMP_HOOK and ASSERT/ASSERT_VERBOSE). Implement assert_print(const char *fmt, ...) to emit via ds_vamsg(), and mirror to vprintk() when CONFIG_EXCEPTION_DUMP_HOOK_ONLY is not set. Avoid duplicate post-fault prints by skipping assert output after an exception report has already been sent on the current CPU. Export assert_print for use by assert paths. Signed-off-by: Jyri Sarha --- src/debug/debug_stream/Kconfig | 10 +++++++ .../debug_stream/debug_stream_text_msg.c | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/debug/debug_stream/Kconfig b/src/debug/debug_stream/Kconfig index 9a55ccefcaab..6e831ae7a7aa 100644 --- a/src/debug/debug_stream/Kconfig +++ b/src/debug/debug_stream/Kconfig @@ -65,4 +65,14 @@ config SOF_DEBUG_STREAM_TEXT_MSG ds_msg(). See include/user/debug_stream_text_msg.h for prototype. +config SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT + bool "Enable assert print sending through Debug-Stream" + depends on EXCEPTION_DUMP_HOOK && (ASSERT || ASSERT_VERBOSE) + select SOF_DEBUG_STREAM_TEXT_MSG + help + Enable assert print sending over debug stream as text + message. This feature is also sensitive to Zephyr option + CONFIG_EXCEPTION_DUMP_HOOK_ONLY. If that is set then the + asserts are not printed through printk interface. + endif diff --git a/src/debug/debug_stream/debug_stream_text_msg.c b/src/debug/debug_stream/debug_stream_text_msg.c index ae771f3c281f..97c209d42c94 100644 --- a/src/debug/debug_stream/debug_stream_text_msg.c +++ b/src/debug/debug_stream/debug_stream_text_msg.c @@ -150,4 +150,32 @@ static int init_exception_dump_hook(void) } SYS_INIT(init_exception_dump_hook, APPLICATION, CONFIG_KERNEL_INIT_PRIORITY_DEFAULT); + +#if defined(CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT) +void assert_print(const char *fmt, ...) +{ + va_list ap; + + /* Do not print assert after exception has been dumped */ + if (ds_cpu[arch_proc_id()].reports_sent > 0) + return; + + va_start(ap, fmt); +#if !defined(CONFIG_EXCEPTION_DUMP_HOOK_ONLY) + { + va_list ap2; + + va_copy(ap2, ap); +#endif + ds_vamsg(fmt, ap); +#if !defined(CONFIG_EXCEPTION_DUMP_HOOK_ONLY) + vprintk(fmt, ap2); + va_end(ap2); + } +#endif + ds_vamsg(fmt, ap); + va_end(ap); +} +EXPORT_SYMBOL(assert_print); +#endif #endif From ae17802fb7eb11a5922e5c9859c49836b1d02024 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Thu, 26 Mar 2026 00:06:32 +0200 Subject: [PATCH 06/11] app: debug_stream_overlay.conf: Add new debug_stream Kconfig options There is two new options, CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS and CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT. CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS limits the amount of debug_stream support to a lower number of cores than what is available. As the number of supported cores shrink, the available circular buffer size increases. This means increased bandwidth. This is particularly useful, when debugging a problem on core 0. By defaut this option is commented out. CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT adds optional support to route assert output through debug stream text messages. This option obeys CONFIG_EXCEPTION_DUMP_HOOK_ONLY. If it is selected the assert print is sent only to debug stream. Without it the assert prints are printed with vprintk too, Signed-off-by: Jyri Sarha --- app/debug_stream_overlay.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/debug_stream_overlay.conf b/app/debug_stream_overlay.conf index cd310b9a7fa8..7edd6bf6709c 100644 --- a/app/debug_stream_overlay.conf +++ b/app/debug_stream_overlay.conf @@ -8,6 +8,17 @@ CONFIG_SOF_DEBUG_STREAM_THREAD_INFO=y CONFIG_THREAD_NAME=y # For Zephyr to compile with thread names on PTL we need to increase THREAD_BYTES CONFIG_MAX_THREAD_BYTES=4 +# Shrink number of CPU sections +# As the number of supported cores shrink, the available circular +# buffer size increases. This means increased bandwidth. This is +# particularly useful, when debugging a problem on core 0. +#CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS=1 + +# Direct the assert prints to debug stream +# This option obeys CONFIG_EXCEPTION_DUMP_HOOK_ONLY. If it is selected +# the assert print is sent only to debug stream. Without it the assert +# prints are printed with vprintk too, +CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT=y # Debug window slot configuration 1 # The CONFIG_SOF_TELEMETRY uses slot 2, but with performance and IO-performance From 8bbf67aeddf3d11baae84387923b6b3ce3951140 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 26 Mar 2026 15:34:08 +0100 Subject: [PATCH 07/11] idc: fix a race Under Zephyr SOF IDC uses Zephyr's P4WQ for message passing. In it k_p4wq_submit() checks that the new work item isn't already on the queue, and it does that by checking its .thread member. That pointer is set to NULL in p4wq_loop() after calling the work handler with no locks held. So, the work handler could've completed handling an IPC, a response could have been sent and a new IPC could arrive, while p4wq_loop() still hasn't cleared the .thread pointer. To fix this use a mutex in idc_send_msg() to protect the work items. Signed-off-by: Guennadi Liakhovetski --- src/idc/zephyr_idc.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index f1af9d53a85b..2f5c048f82a4 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -115,23 +115,38 @@ static void idc_handler(struct k_p4wq_work *work) /* * Used for *target* CPUs, since the initiator (usually core 0) can launch - * several IDC messages at once + * several IDC messages at once. Also we need 2 work items per target core, + * because the p4wq thread might just have returned from the work handler, but + * hasn't released the work buffer yet (hasn't set thread pointer to NULL). + * Then submitting the same work item again can result in an assertion failure. */ -static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT]; +static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT * 2]; +/* Protect the above array */ +static K_MUTEX_DEFINE(idc_mutex); int idc_send_msg(struct idc_msg *msg, uint32_t mode) { struct idc *idc = *idc_get(); struct idc_payload *payload = idc_payload_get(idc, msg->core); unsigned int target_cpu = msg->core; - struct zephyr_idc_msg *zmsg = idc_work + target_cpu; + struct zephyr_idc_msg *zmsg = idc_work + target_cpu * 2; struct idc_msg *msg_cp = &zmsg->msg; struct k_p4wq_work *work = &zmsg->work; int ret; int idc_send_memcpy_err __unused; + k_mutex_lock(&idc_mutex, K_FOREVER); + + if (unlikely(work->thread)) { + /* See comment above the idc_work[] array. */ + zmsg++; + work = &zmsg->work; + msg_cp = &zmsg->msg; + } + idc_send_memcpy_err = memcpy_s(msg_cp, sizeof(*msg_cp), msg, sizeof(*msg)); assert(!idc_send_memcpy_err); + /* Same priority as the IPC thread which is an EDF task and under Zephyr */ work->priority = CONFIG_EDF_THREAD_PRIORITY; work->deadline = 0; @@ -158,6 +173,8 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode) k_p4wq_submit(q_zephyr_idc + target_cpu, work); + k_mutex_unlock(&idc_mutex); + #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS /* Increment performance counters */ io_perf_monitor_update_data(idc->io_perf_out_msg_count, 1); From e65c6d14a05c06e048202d7ee02996922265734a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 29 Jul 2025 17:04:13 +0100 Subject: [PATCH 08/11] userspace: split ipc files into user and kernel features. REVISIT memcpy This is a mostly mechanical split of initial ipc logic into kernel and user files. This is the 1st stage in supporting both privileged kernel and non privileged userspace IPC commands and security surfaces. At a high level library loading and PM will reside as kernel IPC and pipeline and module will become user IPCs. There will be no impact for devices without MMU. Signed-off-by: Liam Girdwood --- src/include/sof/ipc/common.h | 44 ++ src/ipc/ipc4/CMakeLists.txt | 3 +- src/ipc/ipc4/handler-kernel.c | 675 +++++++++++++++++++++ src/ipc/ipc4/{handler.c => handler-user.c} | 578 +----------------- 4 files changed, 737 insertions(+), 563 deletions(-) create mode 100644 src/ipc/ipc4/handler-kernel.c rename src/ipc/ipc4/{handler.c => handler-user.c} (68%) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index e46fc10b9521..4260a9383ab5 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -24,6 +24,7 @@ struct dma_sg_elem_array; struct ipc_msg; +struct ipc4_message_request; /* validates internal non tail structures within IPC command structure */ #define IPC_IS_SIZE_INVALID(object) \ @@ -166,6 +167,49 @@ struct dai_data; */ int ipc_dai_data_config(struct dai_data *dd, struct comp_dev *dev); +/** + * \brief Processes IPC4 userspace module message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/** + * \brief Processes IPC4 userspace global message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/** + * \brief Increment the IPC compound message pre-start counter. + * @param[in] msg_id IPC message ID. + */ +void ipc_compound_pre_start(int msg_id); + +/** + * \brief Decrement the IPC compound message pre-start counter on return value status. + * @param[in] msg_id IPC message ID. + * @param[in] ret Return value of the IPC command. + * @param[in] delayed True if the reply is delayed. + */ +void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); + +/** + * \brief Complete the IPC compound message. + * @param[in] msg_id IPC message ID. + * @param[in] error Error code of the IPC command. + */ +void ipc_compound_msg_done(uint32_t msg_id, int error); + +/** + * \brief Wait for the IPC compound message to complete. + * @return 0 on success, error code otherwise on timeout. + */ +int ipc_wait_for_compound_msg(void); + /** * \brief create a IPC boot complete message. * @param[in] header header. diff --git a/src/ipc/ipc4/CMakeLists.txt b/src/ipc/ipc4/CMakeLists.txt index 02c56130ed11..52908ed2c88a 100644 --- a/src/ipc/ipc4/CMakeLists.txt +++ b/src/ipc/ipc4/CMakeLists.txt @@ -3,7 +3,8 @@ # Files common to Zephyr and XTOS add_local_sources(sof dai.c - handler.c + handler-user.c + handler-kernel.c helper.c logging.c notification.c diff --git a/src/ipc/ipc4/handler-kernel.c b/src/ipc/ipc4/handler-kernel.c new file mode 100644 index 000000000000..0b7b1633ed53 --- /dev/null +++ b/src/ipc/ipc4/handler-kernel.c @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if CONFIG_SOF_BOOT_TEST +/* CONFIG_SOF_BOOT_TEST depends on Zephyr */ +#include +#endif + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +struct ipc4_msg_data { + struct ipc_cmd_hdr msg_in; /* local copy of current message from host header */ + struct ipc_cmd_hdr msg_out; /* local copy of current message to host header */ + atomic_t delayed_reply; + uint32_t delayed_error; +}; + +static struct ipc4_msg_data msg_data; + +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg msg_reply = {0, 0, 0, 0, LIST_INIT(msg_reply.list)}; + +static struct ipc_msg msg_notify = {0, 0, 0, 0, LIST_INIT(msg_notify.list)}; + +#if CONFIG_LIBRARY +static inline struct ipc4_message_request *ipc4_get_message_request(void) +{ + struct ipc *ipc = ipc_get(); + + return (struct ipc4_message_request *)ipc->comp_data; +} + +static inline void ipc4_send_reply(struct ipc4_message_reply *reply) +{ + struct ipc *ipc = ipc_get(); + int ret; + + /* copy the extension from the message reply */ + reply->extension.dat = msg_reply.extension; + ret = memcpy_s(ipc->comp_data, sizeof(*reply), reply, sizeof(*reply)); + assert(!ret); +} + +static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) +{ + const struct ipc4_pipeline_set_state_data *ppl_data; + struct ipc *ipc = ipc_get(); + + ppl_data = (const struct ipc4_pipeline_set_state_data *)ipc->comp_data; + + return ppl_data; +} +#else +static inline struct ipc4_message_request *ipc4_get_message_request(void) +{ + /* ignoring _hdr as it does not contain valid data in IPC4/IDC case */ + return ipc_from_hdr(&msg_data.msg_in); +} + +static inline void ipc4_send_reply(struct ipc4_message_reply *reply) +{ + struct ipc *ipc = ipc_get(); + char *data = ipc->comp_data; + + ipc_msg_send(&msg_reply, data, true); +} + +#endif + +__cold static bool is_any_ppl_active(void) +{ + struct ipc_comp_dev *icd; + struct list_item *clist; + + assert_can_be_cold(); + + list_for_item(clist, &ipc_get()->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->type != COMP_TYPE_PIPELINE) + continue; + + if (icd->pipeline->status == COMP_STATE_ACTIVE) + return true; + } + + return false; +} + +void ipc_compound_pre_start(int msg_id) +{ + /* ipc thread will wait for all scheduled tasks to be complete + * Use a reference count to check status of these tasks. + */ + atomic_add(&msg_data.delayed_reply, 1); +} + +void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) +{ + if (ret) { + ipc_cmd_err(&ipc_tr, "failed to process msg %d status %d", msg_id, ret); + atomic_set(&msg_data.delayed_reply, 0); + return; + } + + /* decrease counter if it is not scheduled by another thread */ + if (!delayed) + atomic_sub(&msg_data.delayed_reply, 1); +} + +void ipc_compound_msg_done(uint32_t msg_id, int error) +{ + if (!atomic_read(&msg_data.delayed_reply)) { + ipc_cmd_err(&ipc_tr, "unexpected delayed reply"); + return; + } + + atomic_sub(&msg_data.delayed_reply, 1); + + /* error reported in delayed pipeline task */ + if (error < 0) { + if (msg_id == SOF_IPC4_GLB_SET_PIPELINE_STATE) + msg_data.delayed_error = IPC4_PIPELINE_STATE_NOT_SET; + } +} + +#if CONFIG_LIBRARY +/* There is no parallel execution in testbench for scheduler and pipelines, so the result would + * be always IPC4_FAILURE. Therefore the compound messages handling is simplified. The pipeline + * triggers will require an explicit scheduler call to get the components to desired state. + */ +int ipc_wait_for_compound_msg(void) +{ + atomic_set(&msg_data.delayed_reply, 0); + return IPC4_SUCCESS; +} +#else +int ipc_wait_for_compound_msg(void) +{ + int try_count = 30; + + while (atomic_read(&msg_data.delayed_reply)) { + k_sleep(Z_TIMEOUT_US(250)); + + if (!try_count--) { + atomic_set(&msg_data.delayed_reply, 0); + ipc_cmd_err(&ipc_tr, "ipc4: failed to wait schedule thread"); + return IPC4_FAILURE; + } + } + + return IPC4_SUCCESS; +} +#endif + +#if CONFIG_LIBRARY_MANAGER +__cold static int ipc4_load_library(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_load_library library; + int ret; + + assert_can_be_cold(); + + library.header.dat = ipc4->primary.dat; + + ret = lib_manager_load_library(library.header.r.dma_id, library.header.r.lib_id, + ipc4->primary.r.type); + if (ret != 0) + return (ret == -EINVAL) ? IPC4_ERROR_INVALID_PARAM : IPC4_FAILURE; + + return IPC4_SUCCESS; +} +#endif + +static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) +{ + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + + switch (type) { + + /* Loads library (using Code Load or HD/A Host Output DMA) */ +#ifdef CONFIG_LIBRARY_MANAGER + case SOF_IPC4_GLB_LOAD_LIBRARY: + ret = ipc4_load_library(ipc4); + break; + case SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE: + ret = ipc4_load_library(ipc4); + break; +#endif + /* not a kernel level IPC message */ + default: + /* try and handle as a user IPC message */ + ret = ipc4_user_process_glb_message(ipc4, &msg_reply); + break; + } + + return ret; +} + +/* disable power gating on core 0 */ +__cold static int ipc4_module_process_d0ix(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_set_d0ix d0ix; + uint32_t module_id, instance_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&d0ix, sizeof(d0ix), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + module_id = d0ix.primary.r.module_id; + instance_id = d0ix.primary.r.instance_id; + + tr_dbg(&ipc_tr, "%x : %x", module_id, instance_id); + + /* only module 0 can be used to set d0ix state */ + if (d0ix.primary.r.module_id || d0ix.primary.r.instance_id) { + ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); + return IPC4_INVALID_RESOURCE_ID; + } + + if (d0ix.extension.r.prevent_power_gating) + pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); + else + pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); + + return 0; +} + +/* enable/disable cores according to the state mask */ +__cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_set_dx dx; + struct ipc4_dx_state_info dx_info; + uint32_t module_id, instance_id; + uint32_t core_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&dx, sizeof(dx), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + module_id = dx.primary.r.module_id; + instance_id = dx.primary.r.instance_id; + + /* only module 0 can be used to set dx state */ + if (module_id || instance_id) { + ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); + return IPC4_INVALID_RESOURCE_ID; + } + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + sizeof(dx_info)); + ret = memcpy_s(&dx_info, sizeof(dx_info), + (const void *)MAILBOX_HOSTBOX_BASE, sizeof(dx_info)); + if (ret < 0) + return IPC4_FAILURE; + + /* check if core enable mask is valid */ + if (dx_info.core_mask > MASK(CONFIG_CORE_COUNT - 1, 0)) { + ipc_cmd_err(&ipc_tr, "CONFIG_CORE_COUNT: %d < core enable mask: %d", + CONFIG_CORE_COUNT, dx_info.core_mask); + return IPC4_ERROR_INVALID_PARAM; + } + + /* check primary core first */ + if ((dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) && + (dx_info.dx_mask & BIT(PLATFORM_PRIMARY_CORE_ID))) { + /* core0 can't be activated more, it's already active since we got here */ + ipc_cmd_err(&ipc_tr, "Core0 is already active"); + return IPC4_BAD_STATE; + } + + /* Activate/deactivate requested cores */ + for (core_id = 1; core_id < CONFIG_CORE_COUNT; core_id++) { + if ((dx_info.core_mask & BIT(core_id)) == 0) + continue; + + if (dx_info.dx_mask & BIT(core_id)) { + ret = cpu_enable_core(core_id); + if (ret != 0) { + ipc_cmd_err(&ipc_tr, "failed to enable core %d", core_id); + return IPC4_FAILURE; + } + } else { + cpu_disable_core(core_id); + if (cpu_is_core_enabled(core_id)) { + ipc_cmd_err(&ipc_tr, "failed to disable core %d", core_id); + return IPC4_FAILURE; + } + } + } + + /* Deactivating primary core if requested. */ + if (dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) { + if (cpu_enabled_cores() & ~BIT(PLATFORM_PRIMARY_CORE_ID)) { + ipc_cmd_err(&ipc_tr, "secondary cores 0x%x still active", + cpu_enabled_cores()); + return IPC4_BUSY; + } + + if (is_any_ppl_active()) { + ipc_cmd_err(&ipc_tr, "some pipelines are still active"); + return IPC4_BUSY; + } + +#if !CONFIG_ADSP_IMR_CONTEXT_SAVE + ret = llext_manager_store_to_dram(); + if (ret < 0) + ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", + ret); + +#if CONFIG_L3_HEAP + l3_heap_save(); +#endif +#endif + +#if defined(CONFIG_PM) + ipc_get()->task_mask |= IPC_TASK_POWERDOWN; +#endif + /* do platform specific suspending */ + platform_context_save(sof_get()); + +#if !defined(CONFIG_LIBRARY) && !defined(CONFIG_ZEPHYR_NATIVE_DRIVERS) + arch_irq_lock(); + platform_timer_stop(timer_get()); +#endif + ipc_get()->pm_prepare_D3 = 1; + } + + return IPC4_SUCCESS; +} + +__cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) +{ + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + + switch (type) { + case SOF_IPC4_MOD_SET_D0IX: + ret = ipc4_module_process_d0ix(ipc4); + break; + case SOF_IPC4_MOD_SET_DX: + ret = ipc4_module_process_dx(ipc4); + break; + default: + /* try and handle as a user IPC message */ + ret = ipc4_user_process_module_message(ipc4, &msg_reply); + break; + } + + return ret; +} + +__cold struct ipc_cmd_hdr *mailbox_validate(void) +{ + struct ipc_cmd_hdr *hdr = ipc_get()->comp_data; + + assert_can_be_cold(); + + return hdr; +} + +struct ipc_cmd_hdr *ipc_compact_read_msg(void) +{ + int words; + + words = ipc_platform_compact_read_msg(&msg_data.msg_in, 2); + if (!words) + return mailbox_validate(); + + return &msg_data.msg_in; +} + +struct ipc_cmd_hdr *ipc_prepare_to_send(const struct ipc_msg *msg) +{ + msg_data.msg_out.pri = msg->header; + msg_data.msg_out.ext = msg->extension; + + if (msg->tx_size) + mailbox_dspbox_write(0, (uint32_t *)msg->tx_data, msg->tx_size); + + return &msg_data.msg_out; +} + +__cold void ipc_boot_complete_msg(struct ipc_cmd_hdr *header, uint32_t data) +{ + assert_can_be_cold(); + + header->pri = SOF_IPC4_FW_READY; + header->ext = 0; +} + +#if defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) +__cold void ipc_send_failed_power_transition_response(void) +{ + struct ipc4_message_request *request = ipc_from_hdr(&msg_data.msg_in); + struct ipc4_message_reply response; + + assert_can_be_cold(); + + response.primary.r.status = IPC4_POWER_TRANSITION_FAILED; + response.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; + response.primary.r.msg_tgt = request->primary.r.msg_tgt; + response.primary.r.type = request->primary.r.type; + + msg_reply.header = response.primary.dat; + list_init(&msg_reply.list); + + ipc_msg_send_direct(&msg_reply, NULL); +} +#endif /* defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) */ + +__cold void ipc_send_panic_notification(void) +{ + assert_can_be_cold(); + + msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_EXCEPTION_CAUGHT); + msg_notify.extension = cpu_get_id(); + msg_notify.tx_size = 0; + msg_notify.tx_data = NULL; + list_init(&msg_notify.list); + + ipc_msg_send_direct(&msg_notify, NULL); +} + +#ifdef CONFIG_LOG_BACKEND_ADSP_MTRACE + +static bool is_notification_queued(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + bool queued = false; + + key = k_spin_lock(&ipc->lock); + if (!list_is_empty(&msg->list)) + queued = true; + k_spin_unlock(&ipc->lock, key); + + return queued; +} + +/* Called from ipc_send_buffer_status_notify(), which is currently "hot" */ +void ipc_send_buffer_status_notify(void) +{ + /* a single msg_notify object is used */ + if (is_notification_queued(&msg_notify)) + return; + + msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); + msg_notify.extension = 0; + msg_notify.tx_size = 0; + + tr_dbg(&ipc_tr, "tx-notify\t: %#x|%#x", msg_notify.header, msg_notify.extension); + + ipc_msg_send(&msg_notify, NULL, true); +} +#endif + +void ipc_msg_reply(struct sof_ipc_reply *reply) +{ + struct ipc4_message_request in; + + in.primary.dat = msg_data.msg_in.pri; + ipc_compound_msg_done(in.primary.r.type, reply->error); +} + +void ipc_cmd(struct ipc_cmd_hdr *_hdr) +{ + struct ipc4_message_request *in = ipc4_get_message_request(); + enum ipc4_message_target target; +#ifdef CONFIG_DEBUG_IPC_TIMINGS + struct ipc4_message_request req; + uint64_t tstamp; +#endif + int err; + +#ifdef CONFIG_DEBUG_IPC_TIMINGS + req = *in; + tstamp = sof_cycle_get_64(); +#else + if (cpu_is_primary(cpu_get_id())) + tr_info(&ipc_tr, "rx\t: %#x|%#x", in->primary.dat, in->extension.dat); +#endif + /* no process on scheduled thread */ + atomic_set(&msg_data.delayed_reply, 0); + msg_data.delayed_error = 0; + msg_reply.tx_data = NULL; + msg_reply.tx_size = 0; + msg_reply.header = in->primary.dat; + msg_reply.extension = in->extension.dat; + + target = in->primary.r.msg_tgt; + + switch (target) { + case SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG: + err = ipc4_process_glb_message(in); + if (err) + ipc_cmd_err(&ipc_tr, "ipc4: FW_GEN_MSG failed with err %d", err); + break; + case SOF_IPC4_MESSAGE_TARGET_MODULE_MSG: + err = ipc4_process_module_message(in); + if (err) + ipc_cmd_err(&ipc_tr, "ipc4: MODULE_MSG failed with err %d", err); + break; + default: + /* should not reach here as we only have 2 message types */ + ipc_cmd_err(&ipc_tr, "ipc4: invalid target %d", target); + err = IPC4_UNKNOWN_MESSAGE_TYPE; + } + + /* FW sends an ipc message to host if request bit is clear */ + if (in->primary.r.rsp == SOF_IPC4_MESSAGE_DIR_MSG_REQUEST) { + struct ipc *ipc = ipc_get(); + struct ipc4_message_reply reply = {{0}}; + + /* Process flow and time stamp for IPC4 msg processed on secondary core : + * core 0 (primary core) core x (secondary core) + * # IPC msg thread #IPC delayed worker #core x idc thread + * ipc_task_ops.run() + * ipc_do_cmd() + * msg_reply.header = in->primary.dat + * ipc4_process_on_core(x) + * mask |= SECONDARY_CORE + * idc_send_message() + * Case 1: + * // Ipc msg processed by secondary core idc_ipc() + * if ((mask & SECONDARY_CORE)) ipc_cmd() + * return; ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * ipc_platform_send_msg + * ---------------------------------------------------------------------------- + * Case 2: + * idc_ipc() + * ipc_cmd() + * //Prepare reply msg + * msg_reply.header = + * reply.primary.dat; + * ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * if ((mask & IPC_TASK_SECONDARY_CORE)) + * return; + * // Ipc reply msg was prepared, so return + * if (msg_reply.header != in->primary.dat) + * return; + * ipc_platform_send_msg + * ---------------------------------------------------------------------------- + * Case 3: + * idc_ipc() + * ipc_cmd() + * //Prepare reply msg + * msg_reply.header = + * reply.primary.dat; + * ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * ipc_platform_send_msg + * + * if ((mask & IPC_TASK_SECONDARY_CORE)) + * return; + * // Ipc reply msg was prepared, so return + * if (msg_reply.header != in->primary.dat) + * return; + */ + + /* Reply prepared by secondary core */ + if ((ipc->task_mask & IPC_TASK_SECONDARY_CORE) && cpu_is_primary(cpu_get_id())) + return; + /* Reply has been prepared by secondary core */ + if (msg_reply.header != in->primary.dat) + return; + + /* Do not send reply for SET_DX if we are going to enter D3 + * The reply is going to be sent as part of the power down + * sequence + */ + if (ipc->task_mask & IPC_TASK_POWERDOWN) + return; + + if (ipc_wait_for_compound_msg() != 0) { + ipc_cmd_err(&ipc_tr, "ipc4: failed to send delayed reply"); + err = IPC4_FAILURE; + } + + /* copy contents of message received */ + reply.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; + reply.primary.r.msg_tgt = in->primary.r.msg_tgt; + reply.primary.r.type = in->primary.r.type; + if (msg_data.delayed_error) + reply.primary.r.status = msg_data.delayed_error; + else + reply.primary.r.status = err; + + msg_reply.header = reply.primary.dat; + +#ifdef CONFIG_DEBUG_IPC_TIMINGS + tr_info(&ipc_tr, "tx-reply\t: %#x|%#x to %#x|%#x in %llu us", msg_reply.header, + msg_reply.extension, req.primary.dat, req.extension.dat, + k_cyc_to_us_near64(sof_cycle_get_64() - tstamp)); +#else + tr_dbg(&ipc_tr, "tx-reply\t: %#x|%#x", msg_reply.header, + msg_reply.extension); +#endif + ipc4_send_reply(&reply); + } +} diff --git a/src/ipc/ipc4/handler.c b/src/ipc/ipc4/handler-user.c similarity index 68% rename from src/ipc/ipc4/handler.c rename to src/ipc/ipc4/handler-user.c index fc64c904ef80..f5940dbec271 100644 --- a/src/ipc/ipc4/handler.c +++ b/src/ipc/ipc4/handler-user.c @@ -58,36 +58,11 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); -struct ipc4_msg_data { - struct ipc_cmd_hdr msg_in; /* local copy of current message from host header */ - struct ipc_cmd_hdr msg_out; /* local copy of current message to host header */ - atomic_t delayed_reply; - uint32_t delayed_error; -}; - -static struct ipc4_msg_data msg_data; - +/* Userspace message context, copied in/out by kernel IPC thread. */ /* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg msg_reply = {0, 0, 0, 0, LIST_INIT(msg_reply.list)}; - -static struct ipc_msg msg_notify = {0, 0, 0, 0, LIST_INIT(msg_notify.list)}; +static struct ipc_msg *msg_reply; #if CONFIG_LIBRARY -static inline struct ipc4_message_request *ipc4_get_message_request(void) -{ - struct ipc *ipc = ipc_get(); - - return (struct ipc4_message_request *)ipc->comp_data; -} - -static inline void ipc4_send_reply(struct ipc4_message_reply *reply) -{ - struct ipc *ipc = ipc_get(); - - /* copy the extension from the message reply */ - reply->extension.dat = msg_reply.extension; - memcpy((char *)ipc->comp_data, reply, sizeof(*reply)); -} static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) { @@ -99,19 +74,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( return ppl_data; } #else -static inline struct ipc4_message_request *ipc4_get_message_request(void) -{ - /* ignoring _hdr as it does not contain valid data in IPC4/IDC case */ - return ipc_from_hdr(&msg_data.msg_in); -} - -static inline void ipc4_send_reply(struct ipc4_message_reply *reply) -{ - struct ipc *ipc = ipc_get(); - char *data = ipc->comp_data; - - ipc_msg_send(&msg_reply, data, true); -} static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) { @@ -180,25 +142,6 @@ static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) return err; } -__cold static bool is_any_ppl_active(void) -{ - struct ipc_comp_dev *icd; - struct list_item *clist; - - assert_can_be_cold(); - - list_for_item(clist, &ipc_get()->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->type != COMP_TYPE_PIPELINE) - continue; - - if (icd->pipeline->status == COMP_STATE_ACTIVE) - return true; - } - - return false; -} - static struct ipc_comp_dev *pipeline_get_host_dev(struct ipc_comp_dev *ppl_icd) { struct ipc_comp_dev *host_dev; @@ -468,72 +411,6 @@ int ipc4_pipeline_trigger(struct ipc_comp_dev *ppl_icd, uint32_t cmd, bool *dela return ret; } -static void ipc_compound_pre_start(int msg_id) -{ - /* ipc thread will wait for all scheduled tasks to be complete - * Use a reference count to check status of these tasks. - */ - atomic_add(&msg_data.delayed_reply, 1); -} - -static void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) -{ - if (ret) { - ipc_cmd_err(&ipc_tr, "failed to process msg %d status %d", msg_id, ret); - atomic_set(&msg_data.delayed_reply, 0); - return; - } - - /* decrease counter if it is not scheduled by another thread */ - if (!delayed) - atomic_sub(&msg_data.delayed_reply, 1); -} - -static void ipc_compound_msg_done(uint32_t msg_id, int error) -{ - if (!atomic_read(&msg_data.delayed_reply)) { - ipc_cmd_err(&ipc_tr, "unexpected delayed reply"); - return; - } - - atomic_sub(&msg_data.delayed_reply, 1); - - /* error reported in delayed pipeline task */ - if (error < 0) { - if (msg_id == SOF_IPC4_GLB_SET_PIPELINE_STATE) - msg_data.delayed_error = IPC4_PIPELINE_STATE_NOT_SET; - } -} - -#if CONFIG_LIBRARY -/* There is no parallel execution in testbench for scheduler and pipelines, so the result would - * be always IPC4_FAILURE. Therefore the compound messages handling is simplified. The pipeline - * triggers will require an explicit scheduler call to get the components to desired state. - */ -static int ipc_wait_for_compound_msg(void) -{ - atomic_set(&msg_data.delayed_reply, 0); - return IPC4_SUCCESS; -} -#else -static int ipc_wait_for_compound_msg(void) -{ - int try_count = 30; - - while (atomic_read(&msg_data.delayed_reply)) { - k_sleep(Z_TIMEOUT_US(250)); - - if (!try_count--) { - atomic_set(&msg_data.delayed_reply, 0); - ipc_cmd_err(&ipc_tr, "ipc4: failed to wait schedule thread"); - return IPC4_FAILURE; - } - } - - return IPC4_SUCCESS; -} -#endif - __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper(void) { assert_can_be_cold(); @@ -670,25 +547,6 @@ static int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) return ret; } -#if CONFIG_LIBRARY_MANAGER -__cold static int ipc4_load_library(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_load_library library; - int ret; - - assert_can_be_cold(); - - library.header.dat = ipc4->primary.dat; - - ret = lib_manager_load_library(library.header.r.dma_id, library.header.r.lib_id, - ipc4->primary.r.type); - if (ret != 0) - return (ret == -EINVAL) ? IPC4_ERROR_INVALID_PARAM : IPC4_FAILURE; - - return IPC4_SUCCESS; -} -#endif - __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) { assert_can_be_cold(); @@ -758,11 +616,11 @@ __cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, &reply_size); /* reply size is returned in header extension dword */ - msg_reply.extension = reply_size; + msg_reply->extension = reply_size; if (reply_size > 0) { - msg_reply.tx_data = ipc->comp_data; - msg_reply.tx_size = reply_size; + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; } return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; @@ -782,12 +640,14 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) #endif } -static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) { uint32_t type; int ret; type = ipc4->primary.r.type; + msg_reply = reply; switch (type) { case SOF_IPC4_GLB_BOOT_CONFIG: @@ -822,15 +682,6 @@ static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) ret = IPC4_UNAVAILABLE; break; - /* Loads library (using Code Load or HD/A Host Output DMA) */ -#ifdef CONFIG_LIBRARY_MANAGER - case SOF_IPC4_GLB_LOAD_LIBRARY: - ret = ipc4_load_library(ipc4); - break; - case SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE: - ret = ipc4_load_library(ipc4); - break; -#endif case SOF_IPC4_GLB_INTERNAL_MESSAGE: ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); ret = IPC4_UNAVAILABLE; @@ -985,7 +836,7 @@ static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4 } if (!set) - msg_reply.extension = config->extension.dat; + msg_reply->extension = config->extension.dat; return ret; } @@ -1020,7 +871,7 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, +__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, bool init_block, bool final_block, @@ -1207,9 +1058,9 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ if (ret) return ret; - msg_reply.extension = reply.extension.dat; - msg_reply.tx_size = data_offset; - msg_reply.tx_data = data; + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; return ret; } @@ -1377,145 +1228,8 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) return ret; } -/* disable power gating on core 0 */ -__cold static int ipc4_module_process_d0ix(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_set_d0ix d0ix; - uint32_t module_id, instance_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&d0ix, sizeof(d0ix), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - module_id = d0ix.primary.r.module_id; - instance_id = d0ix.primary.r.instance_id; - - tr_dbg(&ipc_tr, "%x : %x", module_id, instance_id); - - /* only module 0 can be used to set d0ix state */ - if (d0ix.primary.r.module_id || d0ix.primary.r.instance_id) { - ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); - return IPC4_INVALID_RESOURCE_ID; - } - - if (d0ix.extension.r.prevent_power_gating) - pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); - else - pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); - - return 0; -} - -/* enable/disable cores according to the state mask */ -__cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_set_dx dx; - struct ipc4_dx_state_info dx_info; - uint32_t module_id, instance_id; - uint32_t core_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&dx, sizeof(dx), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - module_id = dx.primary.r.module_id; - instance_id = dx.primary.r.instance_id; - - /* only module 0 can be used to set dx state */ - if (module_id || instance_id) { - ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); - return IPC4_INVALID_RESOURCE_ID; - } - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - sizeof(dx_info)); - ret = memcpy_s(&dx_info, sizeof(dx_info), - (const void *)MAILBOX_HOSTBOX_BASE, sizeof(dx_info)); - if (ret < 0) - return IPC4_FAILURE; - - /* check if core enable mask is valid */ - if (dx_info.core_mask > MASK(CONFIG_CORE_COUNT - 1, 0)) { - ipc_cmd_err(&ipc_tr, "CONFIG_CORE_COUNT: %d < core enable mask: %d", - CONFIG_CORE_COUNT, dx_info.core_mask); - return IPC4_ERROR_INVALID_PARAM; - } - - /* check primary core first */ - if ((dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) && - (dx_info.dx_mask & BIT(PLATFORM_PRIMARY_CORE_ID))) { - /* core0 can't be activated more, it's already active since we got here */ - ipc_cmd_err(&ipc_tr, "Core0 is already active"); - return IPC4_BAD_STATE; - } - - /* Activate/deactivate requested cores */ - for (core_id = 1; core_id < CONFIG_CORE_COUNT; core_id++) { - if ((dx_info.core_mask & BIT(core_id)) == 0) - continue; - - if (dx_info.dx_mask & BIT(core_id)) { - ret = cpu_enable_core(core_id); - if (ret != 0) { - ipc_cmd_err(&ipc_tr, "failed to enable core %d", core_id); - return IPC4_FAILURE; - } - } else { - cpu_disable_core(core_id); - if (cpu_is_core_enabled(core_id)) { - ipc_cmd_err(&ipc_tr, "failed to disable core %d", core_id); - return IPC4_FAILURE; - } - } - } - - /* Deactivating primary core if requested. */ - if (dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) { - if (cpu_enabled_cores() & ~BIT(PLATFORM_PRIMARY_CORE_ID)) { - ipc_cmd_err(&ipc_tr, "secondary cores 0x%x still active", - cpu_enabled_cores()); - return IPC4_BUSY; - } - - if (is_any_ppl_active()) { - ipc_cmd_err(&ipc_tr, "some pipelines are still active"); - return IPC4_BUSY; - } - -#if !CONFIG_ADSP_IMR_CONTEXT_SAVE - ret = llext_manager_store_to_dram(); - if (ret < 0) - ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", - ret); - -#if CONFIG_L3_HEAP - l3_heap_save(); -#endif -#endif - -#if defined(CONFIG_PM) - ipc_get()->task_mask |= IPC_TASK_POWERDOWN; -#endif - /* do platform specific suspending */ - platform_context_save(sof_get()); - -#if !defined(CONFIG_LIBRARY) && !defined(CONFIG_ZEPHYR_NATIVE_DRIVERS) - arch_irq_lock(); - platform_timer_stop(timer_get()); -#endif - ipc_get()->pm_prepare_D3 = 1; - } - - return IPC4_SUCCESS; -} - -__cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) { uint32_t type; int ret; @@ -1523,6 +1237,7 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) assert_can_be_cold(); type = ipc4->primary.r.type; + msg_reply = reply; switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: @@ -1549,12 +1264,6 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) case SOF_IPC4_MOD_DELETE_INSTANCE: ret = ipc4_delete_module_instance(ipc4); break; - case SOF_IPC4_MOD_SET_D0IX: - ret = ipc4_module_process_d0ix(ipc4); - break; - case SOF_IPC4_MOD_SET_DX: - ret = ipc4_module_process_dx(ipc4); - break; case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: ret = IPC4_UNAVAILABLE; @@ -1566,258 +1275,3 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) return ret; } - -__cold struct ipc_cmd_hdr *mailbox_validate(void) -{ - struct ipc_cmd_hdr *hdr = ipc_get()->comp_data; - - assert_can_be_cold(); - - return hdr; -} - -struct ipc_cmd_hdr *ipc_compact_read_msg(void) -{ - int words; - - words = ipc_platform_compact_read_msg(&msg_data.msg_in, 2); - if (!words) - return mailbox_validate(); - - return &msg_data.msg_in; -} - -struct ipc_cmd_hdr *ipc_prepare_to_send(const struct ipc_msg *msg) -{ - msg_data.msg_out.pri = msg->header; - msg_data.msg_out.ext = msg->extension; - - if (msg->tx_size) - mailbox_dspbox_write(0, (uint32_t *)msg->tx_data, msg->tx_size); - - return &msg_data.msg_out; -} - -__cold void ipc_boot_complete_msg(struct ipc_cmd_hdr *header, uint32_t data) -{ - assert_can_be_cold(); - - header->pri = SOF_IPC4_FW_READY; - header->ext = 0; -} - -#if defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) -__cold void ipc_send_failed_power_transition_response(void) -{ - struct ipc4_message_request *request = ipc_from_hdr(&msg_data.msg_in); - struct ipc4_message_reply response; - - assert_can_be_cold(); - - response.primary.r.status = IPC4_POWER_TRANSITION_FAILED; - response.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; - response.primary.r.msg_tgt = request->primary.r.msg_tgt; - response.primary.r.type = request->primary.r.type; - - msg_reply.header = response.primary.dat; - list_init(&msg_reply.list); - - ipc_msg_send_direct(&msg_reply, NULL); -} -#endif /* defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) */ - -__cold void ipc_send_panic_notification(void) -{ - assert_can_be_cold(); - - msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_EXCEPTION_CAUGHT); - msg_notify.extension = cpu_get_id(); - msg_notify.tx_size = 0; - msg_notify.tx_data = NULL; - list_init(&msg_notify.list); - - ipc_msg_send_direct(&msg_notify, NULL); -} - -#ifdef CONFIG_LOG_BACKEND_ADSP_MTRACE - -static bool is_notification_queued(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - bool queued = false; - - key = k_spin_lock(&ipc->lock); - if (!list_is_empty(&msg->list)) - queued = true; - k_spin_unlock(&ipc->lock, key); - - return queued; -} - -/* Called from ipc_send_buffer_status_notify(), which is currently "hot" */ -void ipc_send_buffer_status_notify(void) -{ - /* a single msg_notify object is used */ - if (is_notification_queued(&msg_notify)) - return; - - msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); - msg_notify.extension = 0; - msg_notify.tx_size = 0; - - tr_dbg(&ipc_tr, "tx-notify\t: %#x|%#x", msg_notify.header, msg_notify.extension); - - ipc_msg_send(&msg_notify, NULL, true); -} -#endif - -void ipc_msg_reply(struct sof_ipc_reply *reply) -{ - struct ipc4_message_request in; - - in.primary.dat = msg_data.msg_in.pri; - ipc_compound_msg_done(in.primary.r.type, reply->error); -} - -void ipc_cmd(struct ipc_cmd_hdr *_hdr) -{ - struct ipc4_message_request *in = ipc4_get_message_request(); - enum ipc4_message_target target; -#ifdef CONFIG_DEBUG_IPC_TIMINGS - struct ipc4_message_request req; - uint64_t tstamp; -#endif - int err; - -#ifdef CONFIG_DEBUG_IPC_TIMINGS - req = *in; - tstamp = sof_cycle_get_64(); -#else - if (cpu_is_primary(cpu_get_id())) - tr_info(&ipc_tr, "rx\t: %#x|%#x", in->primary.dat, in->extension.dat); -#endif - /* no process on scheduled thread */ - atomic_set(&msg_data.delayed_reply, 0); - msg_data.delayed_error = 0; - msg_reply.tx_data = NULL; - msg_reply.tx_size = 0; - msg_reply.header = in->primary.dat; - msg_reply.extension = in->extension.dat; - - target = in->primary.r.msg_tgt; - - switch (target) { - case SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG: - err = ipc4_process_glb_message(in); - if (err) - ipc_cmd_err(&ipc_tr, "ipc4: FW_GEN_MSG failed with err %d", err); - break; - case SOF_IPC4_MESSAGE_TARGET_MODULE_MSG: - err = ipc4_process_module_message(in); - if (err) - ipc_cmd_err(&ipc_tr, "ipc4: MODULE_MSG failed with err %d", err); - break; - default: - /* should not reach here as we only have 2 message types */ - ipc_cmd_err(&ipc_tr, "ipc4: invalid target %d", target); - err = IPC4_UNKNOWN_MESSAGE_TYPE; - } - - /* FW sends an ipc message to host if request bit is clear */ - if (in->primary.r.rsp == SOF_IPC4_MESSAGE_DIR_MSG_REQUEST) { - struct ipc *ipc = ipc_get(); - struct ipc4_message_reply reply = {{0}}; - - /* Process flow and time stamp for IPC4 msg processed on secondary core : - * core 0 (primary core) core x (secondary core) - * # IPC msg thread #IPC delayed worker #core x idc thread - * ipc_task_ops.run() - * ipc_do_cmd() - * msg_reply.header = in->primary.dat - * ipc4_process_on_core(x) - * mask |= SECONDARY_CORE - * idc_send_message() - * Case 1: - * // Ipc msg processed by secondary core idc_ipc() - * if ((mask & SECONDARY_CORE)) ipc_cmd() - * return; ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * ipc_platform_send_msg - * ---------------------------------------------------------------------------- - * Case 2: - * idc_ipc() - * ipc_cmd() - * //Prepare reply msg - * msg_reply.header = - * reply.primary.dat; - * ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * if ((mask & IPC_TASK_SECONDARY_CORE)) - * return; - * // Ipc reply msg was prepared, so return - * if (msg_reply.header != in->primary.dat) - * return; - * ipc_platform_send_msg - * ---------------------------------------------------------------------------- - * Case 3: - * idc_ipc() - * ipc_cmd() - * //Prepare reply msg - * msg_reply.header = - * reply.primary.dat; - * ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * ipc_platform_send_msg - * - * if ((mask & IPC_TASK_SECONDARY_CORE)) - * return; - * // Ipc reply msg was prepared, so return - * if (msg_reply.header != in->primary.dat) - * return; - */ - - /* Reply prepared by secondary core */ - if ((ipc->task_mask & IPC_TASK_SECONDARY_CORE) && cpu_is_primary(cpu_get_id())) - return; - /* Reply has been prepared by secondary core */ - if (msg_reply.header != in->primary.dat) - return; - - /* Do not send reply for SET_DX if we are going to enter D3 - * The reply is going to be sent as part of the power down - * sequence - */ - if (ipc->task_mask & IPC_TASK_POWERDOWN) - return; - - if (ipc_wait_for_compound_msg() != 0) { - ipc_cmd_err(&ipc_tr, "ipc4: failed to send delayed reply"); - err = IPC4_FAILURE; - } - - /* copy contents of message received */ - reply.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; - reply.primary.r.msg_tgt = in->primary.r.msg_tgt; - reply.primary.r.type = in->primary.r.type; - if (msg_data.delayed_error) - reply.primary.r.status = msg_data.delayed_error; - else - reply.primary.r.status = err; - - msg_reply.header = reply.primary.dat; - -#ifdef CONFIG_DEBUG_IPC_TIMINGS - tr_info(&ipc_tr, "tx-reply\t: %#x|%#x to %#x|%#x in %llu us", msg_reply.header, - msg_reply.extension, req.primary.dat, req.extension.dat, - k_cyc_to_us_near64(sof_cycle_get_64() - tstamp)); -#else - tr_dbg(&ipc_tr, "tx-reply\t: %#x|%#x", msg_reply.header, - msg_reply.extension); -#endif - ipc4_send_reply(&reply); - } -} From 00df82d2c8f3893d49268563b1a3a8bdd8461238 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Thu, 12 Mar 2026 17:19:12 +0200 Subject: [PATCH 09/11] ipc: add simulated userspace IPC thread Add a dedicated kernel thread to simulate the future userspace IPC processing path. The thread is gated behind Kconfig option SOF_IPC_USER_THREAD (default y, depends on IPC_MAJOR_4) with a configurable stack size SOF_IPC_USER_THREAD_STACK_SIZE (default 4096). The thread uses a semaphore pair (req_sem / reply_sem) for synchronization and is pinned to the primary core. Currently it only echoes requests back as replies; actual DSP to host IPC message sending from user thread will be handled in subsequent commits. Signed-off-by: Jyri Sarha --- src/include/sof/ipc/common.h | 11 ++++++++++ src/ipc/ipc-common.c | 39 +++++++++++++++++++++++++++++++++++- zephyr/Kconfig | 16 +++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 4260a9383ab5..ddafb26d6279 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -54,6 +54,14 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_SECONDARY_CORE BIT(2) #define IPC_TASK_POWERDOWN BIT(3) +#if defined(__ZEPHYR__) && CONFIG_SOF_IPC_USER_THREAD +struct ipc_user_thread_data { + struct k_thread thread; + struct k_sem req_sem; + struct k_sem reply_sem; +}; +#endif + struct ipc { struct k_spinlock lock; /* locking mechanism */ void *comp_data; @@ -84,6 +92,9 @@ struct ipc { #ifdef __ZEPHYR__ struct k_work_delayable z_delayed_work; struct k_work_q ipc_send_wq; +#if CONFIG_SOF_IPC_USER_THREAD + struct ipc_user_thread_data ipc_user; +#endif #endif void *private; diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 10f241784625..67371aab0f25 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -179,6 +179,40 @@ void ipc_send_queued_msg(void) #ifdef __ZEPHYR__ static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); +#if CONFIG_SOF_IPC_USER_THREAD +static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +static void ipc_user_thread(void *arg1, void *arg2, void *arg3); + +static void ipc_init_user_thread(struct ipc *ipc) +{ + struct k_thread *user_thread = &ipc->ipc_user.thread; + + k_sem_init(&ipc->ipc_user.req_sem, 0, 1); + k_sem_init(&ipc->ipc_user.reply_sem, 0, 1); + + k_thread_create(user_thread, ipc_user_stack, + K_THREAD_STACK_SIZEOF(ipc_user_stack), + ipc_user_thread, ipc, NULL, NULL, + 1, 0, K_FOREVER); + + k_thread_cpu_pin(user_thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_name_set(user_thread, "ipc_user"); + k_thread_start(user_thread); +} + +static void ipc_user_thread(void *arg1, void *arg2, void *arg3) +{ + struct ipc *ipc = arg1; + + ARG_UNUSED(arg2); + ARG_UNUSED(arg3); + + while (1) { + k_sem_take(&ipc->ipc_user.req_sem, K_FOREVER); + k_sem_give(&ipc->ipc_user.reply_sem); + } +} +#endif /* CONFIG_SOF_IPC_USER_THREAD */ #endif static void schedule_ipc_worker(void) @@ -339,10 +373,13 @@ __cold int ipc_init(struct sof *sof) k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); k_thread_name_set(thread, "ipc_send_wq"); - k_thread_resume(thread); k_work_init_delayable(&sof->ipc->z_delayed_work, ipc_work_handler); + + #if CONFIG_SOF_IPC_USER_THREAD + ipc_init_user_thread(sof->ipc); + #endif #endif return platform_ipc_init(sof->ipc); diff --git a/zephyr/Kconfig b/zephyr/Kconfig index c744b965a07f..b4d7a70e81c3 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -256,3 +256,19 @@ config STACK_SIZE_IPC_TX default 2048 help IPC sender work-queue thread stack size. Keep a power of 2. + +config SOF_IPC_USER_THREAD + bool "Enable IPC user thread" + default y + depends on IPC_MAJOR_4 + help + Enable a dedicated IPC user thread and related synchronization + primitives in IPC common code. + +config SOF_IPC_USER_THREAD_STACK_SIZE + int "IPC user thread stack size" + default 4096 + depends on SOF_IPC_USER_THREAD + help + Stack size for the simulated userspace IPC thread. + Keep a power of 2. From 82167c71f23477ae8e34832a3a1a86fdb868b6d8 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Fri, 20 Mar 2026 00:42:22 +0200 Subject: [PATCH 10/11] ipc4: route user messages through optional ipc_user thread Add a synchronous dispatch wrapper for IPC4 global/module user messages and route kernel handler default paths through it. When CONFIG_SOF_IPC_USER_THREAD is enabled, requests are passed to ipc_user thread via req/reply semaphores and protected shared state. When CONFIG_SOF_IPC_USER_THREAD is disabled, the same wrapper functions fall back to direct ipc4_user_process_* calls in the main IPC thread. Signed-off-by: Jyri Sarha --- src/include/sof/ipc/common.h | 29 ++++++++++ src/ipc/ipc-common.c | 99 +++++++++++++++++++++++++++++++++++ src/ipc/ipc4/handler-kernel.c | 4 +- 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index ddafb26d6279..80e2a0a819a2 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -54,11 +54,22 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_SECONDARY_CORE BIT(2) #define IPC_TASK_POWERDOWN BIT(3) +enum ipc4_user_thread_req { + IPC4_USER_THREAD_REQ_NONE, + IPC4_USER_THREAD_REQ_GLB, + IPC4_USER_THREAD_REQ_MODULE, +}; + #if defined(__ZEPHYR__) && CONFIG_SOF_IPC_USER_THREAD struct ipc_user_thread_data { struct k_thread thread; + struct k_spinlock lock; struct k_sem req_sem; struct k_sem reply_sem; + enum ipc4_user_thread_req req; + struct ipc4_message_request *msg; + struct ipc_msg *reply; + int ret; }; #endif @@ -194,6 +205,24 @@ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct i */ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Processes IPC4 global message in IPC user thread. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply); + +/** + * \brief Processes IPC4 module message in IPC user thread. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply); + /** * \brief Increment the IPC compound message pre-start counter. * @param[in] msg_id IPC message ID. diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 67371aab0f25..d028a3f45060 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -187,8 +187,13 @@ static void ipc_init_user_thread(struct ipc *ipc) { struct k_thread *user_thread = &ipc->ipc_user.thread; + k_spinlock_init(&ipc->ipc_user.lock); k_sem_init(&ipc->ipc_user.req_sem, 0, 1); k_sem_init(&ipc->ipc_user.reply_sem, 0, 1); + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + ipc->ipc_user.msg = NULL; + ipc->ipc_user.reply = NULL; + ipc->ipc_user.ret = 0; k_thread_create(user_thread, ipc_user_stack, K_THREAD_STACK_SIZEOF(ipc_user_stack), @@ -200,18 +205,112 @@ static void ipc_init_user_thread(struct ipc *ipc) k_thread_start(user_thread); } +static int ipc4_process_in_user_thread(struct ipc *ipc, enum ipc4_user_thread_req req, + struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + k_spinlock_key_t key; + int ret; + + if (!ipc || !ipc4 || !reply) + return -EINVAL; + + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.req = req; + ipc->ipc_user.msg = ipc4; + ipc->ipc_user.reply = reply; + k_spin_unlock(&ipc->ipc_user.lock, key); + + k_sem_give(&ipc->ipc_user.req_sem); + k_sem_take(&ipc->ipc_user.reply_sem, K_FOREVER); + + key = k_spin_lock(&ipc->ipc_user.lock); + req = ipc->ipc_user.req; + ret = ipc->ipc_user.ret; + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + k_spin_unlock(&ipc->ipc_user.lock, key); + + if (req != IPC4_USER_THREAD_REQ_NONE) + return -EINVAL; + + if (ret) + tr_err(&ipc_tr, "%#x %#x reply %d", ipc4->primary.dat, ipc4->extension.dat, + ret); + + return ret; +} + +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_process_in_user_thread(ipc_get(), IPC4_USER_THREAD_REQ_GLB, ipc4, reply); +} + +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_process_in_user_thread(ipc_get(), IPC4_USER_THREAD_REQ_MODULE, ipc4, reply); +} + static void ipc_user_thread(void *arg1, void *arg2, void *arg3) { struct ipc *ipc = arg1; + struct ipc4_message_request *ipc4; + struct ipc_msg *reply; + enum ipc4_user_thread_req req; + int ret; + k_spinlock_key_t key; ARG_UNUSED(arg2); ARG_UNUSED(arg3); + tr_info(&ipc_tr, "user thread started"); while (1) { k_sem_take(&ipc->ipc_user.req_sem, K_FOREVER); + + key = k_spin_lock(&ipc->ipc_user.lock); + req = ipc->ipc_user.req; + ipc4 = ipc->ipc_user.msg; + reply = ipc->ipc_user.reply; + k_spin_unlock(&ipc->ipc_user.lock, key); + + switch (req) { + case IPC4_USER_THREAD_REQ_GLB: + ret = ipc4_user_process_glb_message(ipc4, reply); + break; + case IPC4_USER_THREAD_REQ_MODULE: + ret = ipc4_user_process_module_message(ipc4, reply); + break; + default: + ret = -EINVAL; + break; + } + + if (ret) + tr_err(&ipc_tr, "%#x %#x failed: %d", + ipc4->primary.dat, ipc4->extension.dat, ret); + + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.ret = ret; + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + k_spin_unlock(&ipc->ipc_user.lock, key); + k_sem_give(&ipc->ipc_user.reply_sem); } } +#else /* CONFIG_SOF_IPC_USER_THREAD */ + +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_user_process_glb_message(ipc4, reply); +} + +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_user_process_module_message(ipc4, reply); +} #endif /* CONFIG_SOF_IPC_USER_THREAD */ #endif diff --git a/src/ipc/ipc4/handler-kernel.c b/src/ipc/ipc4/handler-kernel.c index 0b7b1633ed53..a6003922ed14 100644 --- a/src/ipc/ipc4/handler-kernel.c +++ b/src/ipc/ipc4/handler-kernel.c @@ -249,7 +249,7 @@ static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) /* not a kernel level IPC message */ default: /* try and handle as a user IPC message */ - ret = ipc4_user_process_glb_message(ipc4, &msg_reply); + ret = ipc4_process_glb_message_in_user_thread(ipc4, &msg_reply); break; } @@ -412,7 +412,7 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) break; default: /* try and handle as a user IPC message */ - ret = ipc4_user_process_module_message(ipc4, &msg_reply); + ret = ipc4_process_module_message_in_user_thread(ipc4, &msg_reply); break; } From 320178f2e6f0f44dace0c2aff7551e9a58c92d80 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Fri, 20 Mar 2026 01:17:44 +0200 Subject: [PATCH 11/11] ipc: user: defer ipc_send_wq scheduling from ipc_user thread When CONFIG_SOF_IPC_USER_THREAD is enabled, avoid calling k_work_schedule_for_queue() directly from ipc_user thread. Record a per-ipc_user pending scheduling flag instead, and flush that pending request in ipc4_process_in_user_thread() after the request/reply handshake returns to the kernel IPC context. This keeps ipc_send_wq scheduling in kernel IPC thread context while preserving existing behavior for configurations without CONFIG_SOF_IPC_USER_THREAD. Signed-off-by: Jyri Sarha --- src/include/sof/ipc/common.h | 1 + src/ipc/ipc-common.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 80e2a0a819a2..f72cc10273d1 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -66,6 +66,7 @@ struct ipc_user_thread_data { struct k_spinlock lock; struct k_sem req_sem; struct k_sem reply_sem; + bool schedule_pending; enum ipc4_user_thread_req req; struct ipc4_message_request *msg; struct ipc_msg *reply; diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index d028a3f45060..cbd9917f66c2 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -190,6 +190,7 @@ static void ipc_init_user_thread(struct ipc *ipc) k_spinlock_init(&ipc->ipc_user.lock); k_sem_init(&ipc->ipc_user.req_sem, 0, 1); k_sem_init(&ipc->ipc_user.reply_sem, 0, 1); + ipc->ipc_user.schedule_pending = false; ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; ipc->ipc_user.msg = NULL; ipc->ipc_user.reply = NULL; @@ -210,6 +211,8 @@ static int ipc4_process_in_user_thread(struct ipc *ipc, enum ipc4_user_thread_re struct ipc_msg *reply) { k_spinlock_key_t key; + k_spinlock_key_t ipc_key; + bool schedule_pending; int ret; if (!ipc || !ipc4 || !reply) @@ -225,11 +228,20 @@ static int ipc4_process_in_user_thread(struct ipc *ipc, enum ipc4_user_thread_re k_sem_take(&ipc->ipc_user.reply_sem, K_FOREVER); key = k_spin_lock(&ipc->ipc_user.lock); + schedule_pending = ipc->ipc_user.schedule_pending; + ipc->ipc_user.schedule_pending = false; req = ipc->ipc_user.req; ret = ipc->ipc_user.ret; ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; k_spin_unlock(&ipc->ipc_user.lock, key); + if (schedule_pending) { + ipc_key = k_spin_lock(&ipc->lock); + k_work_schedule_for_queue(&ipc->ipc_send_wq, &ipc->z_delayed_work, + K_USEC(IPC_PERIOD_USEC)); + k_spin_unlock(&ipc->lock, ipc_key); + } + if (req != IPC4_USER_THREAD_REQ_NONE) return -EINVAL; @@ -322,6 +334,16 @@ static void schedule_ipc_worker(void) */ #ifdef __ZEPHYR__ struct ipc *ipc = ipc_get(); + #if CONFIG_SOF_IPC_USER_THREAD + k_spinlock_key_t key; + + if (k_current_get() == &ipc->ipc_user.thread) { + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.schedule_pending = true; + k_spin_unlock(&ipc->ipc_user.lock, key); + return; + } + #endif k_work_schedule_for_queue(&ipc->ipc_send_wq, &ipc->z_delayed_work, K_USEC(IPC_PERIOD_USEC)); #endif