diff --git a/app/debug_stream_overlay.conf b/app/debug_stream_overlay.conf index cd310b9a7fa8..7edd6bf6709c 100644 --- a/app/debug_stream_overlay.conf +++ b/app/debug_stream_overlay.conf @@ -8,6 +8,17 @@ CONFIG_SOF_DEBUG_STREAM_THREAD_INFO=y CONFIG_THREAD_NAME=y # For Zephyr to compile with thread names on PTL we need to increase THREAD_BYTES CONFIG_MAX_THREAD_BYTES=4 +# Shrink number of CPU sections +# As the number of supported cores shrink, the available circular +# buffer size increases. This means increased bandwidth. This is +# particularly useful, when debugging a problem on core 0. +#CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS=1 + +# Direct the assert prints to debug stream +# This option obeys CONFIG_EXCEPTION_DUMP_HOOK_ONLY. If it is selected +# the assert print is sent only to debug stream. Without it the assert +# prints are printed with vprintk too, +CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT=y # Debug window slot configuration 1 # The CONFIG_SOF_TELEMETRY uses slot 2, but with performance and IO-performance diff --git a/src/debug/debug_stream/Kconfig b/src/debug/debug_stream/Kconfig index 8756c83ef8da..6e831ae7a7aa 100644 --- a/src/debug/debug_stream/Kconfig +++ b/src/debug/debug_stream/Kconfig @@ -45,6 +45,17 @@ config SOF_DEBUG_STREAM_THREAD_INFO_INTERVAL Decides how often thread info runs and checks execution cycle statistics and stack usage. +config SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS + int "Number of cpu sections slot is divided to" + default MP_MAX_NUM_CPUS + range 1 MP_MAX_NUM_CPUS + help + In some situations a high number of cpu sections shrinks the + circular buffer size so much that it limit debugging. With + this option its possible to use fewer sections. The downside + is that the cpus above the number of sections can not send + any debug stream messages. + config SOF_DEBUG_STREAM_TEXT_MSG bool "Enable text message sending through Debug-Stream" help @@ -54,4 +65,14 @@ config SOF_DEBUG_STREAM_TEXT_MSG ds_msg(). See include/user/debug_stream_text_msg.h for prototype. +config SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT + bool "Enable assert print sending through Debug-Stream" + depends on EXCEPTION_DUMP_HOOK && (ASSERT || ASSERT_VERBOSE) + select SOF_DEBUG_STREAM_TEXT_MSG + help + Enable assert print sending over debug stream as text + message. This feature is also sensitive to Zephyr option + CONFIG_EXCEPTION_DUMP_HOOK_ONLY. If that is set then the + asserts are not printed through printk interface. + endif diff --git a/src/debug/debug_stream/debug_stream_slot.c b/src/debug/debug_stream/debug_stream_slot.c index a55c1c1e116d..fece904edb5c 100644 --- a/src/debug/debug_stream/debug_stream_slot.c +++ b/src/debug/debug_stream/debug_stream_slot.c @@ -19,7 +19,7 @@ struct cpu_mutex { } __aligned(CONFIG_DCACHE_LINE_SIZE); /* CPU specific mutexes for each circular buffer */ -static struct cpu_mutex cpu_mutex[CONFIG_MP_MAX_NUM_CPUS]; +static struct cpu_mutex cpu_mutex[CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS]; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER static struct debug_stream_slot_hdr *dbg_stream_slot; @@ -54,6 +54,12 @@ debug_stream_get_circular_buffer(struct debug_stream_section_descriptor *desc, u return NULL; } + if (core >= CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS) { + LOG_DBG("No section for cpu %u >= %u ", core, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS); + return NULL; + } + *desc = hdr->section_desc[core]; LOG_DBG("Section %u (desc %u %u %u)", core, desc->core_id, desc->buf_words, desc->offset); @@ -116,18 +122,19 @@ int debug_stream_slot_send_record(struct debug_stream_record *rec) static int debug_stream_slot_init(void) { struct debug_stream_slot_hdr *hdr = debug_stream_get_slot(); - size_t hdr_size = ALIGN_UP(offsetof(struct debug_stream_slot_hdr, - section_desc[CONFIG_MP_MAX_NUM_CPUS]), - CONFIG_DCACHE_LINE_SIZE); + size_t hdr_size = ALIGN_UP( + offsetof(struct debug_stream_slot_hdr, + section_desc[CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS]), + CONFIG_DCACHE_LINE_SIZE); size_t section_area_size = ADSP_DW_SLOT_SIZE - hdr_size; size_t section_size = ALIGN_DOWN(section_area_size / - CONFIG_MP_MAX_NUM_CPUS, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS, CONFIG_DCACHE_LINE_SIZE); size_t offset = hdr_size; int i; LOG_INF("%u sections of %u bytes, hdr %u, section area %u", - CONFIG_MP_MAX_NUM_CPUS, section_size, hdr_size, + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS, section_size, hdr_size, section_area_size); #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER @@ -142,9 +149,9 @@ static int debug_stream_slot_init(void) hdr->hdr.magic = DEBUG_STREAM_IDENTIFIER; hdr->hdr.hdr_size = hdr_size; - hdr->total_size = hdr_size + CONFIG_MP_MAX_NUM_CPUS * section_size; - hdr->num_sections = CONFIG_MP_MAX_NUM_CPUS; - for (i = 0; i < CONFIG_MP_MAX_NUM_CPUS; i++) { + hdr->total_size = hdr_size + CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS * section_size; + hdr->num_sections = CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; + for (i = 0; i < CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; i++) { hdr->section_desc[i].core_id = i; hdr->section_desc[i].buf_words = (section_size - offsetof(struct debug_stream_circular_buf, data[0]))/ @@ -154,7 +161,7 @@ static int debug_stream_slot_init(void) i, section_size, offset); offset += section_size; } - for (i = 0; i < CONFIG_MP_MAX_NUM_CPUS; i++) { + for (i = 0; i < CONFIG_SOF_DEBUG_STREAM_SLOT_FORCE_MAX_CPUS; i++) { struct debug_stream_section_descriptor desc = { 0 }; struct debug_stream_circular_buf *buf = debug_stream_get_circular_buffer(&desc, i); diff --git a/src/debug/debug_stream/debug_stream_text_msg.c b/src/debug/debug_stream/debug_stream_text_msg.c index 4ee71322a3e0..97c209d42c94 100644 --- a/src/debug/debug_stream/debug_stream_text_msg.c +++ b/src/debug/debug_stream/debug_stream_text_msg.c @@ -15,18 +15,15 @@ LOG_MODULE_REGISTER(debug_stream_text_msg); -void ds_msg(const char *format, ...) +void ds_vamsg(const char *format, va_list args) { - va_list args; struct { struct debug_stream_text_msg msg; char text[128]; } __packed buf = { 0 }; ssize_t len; - va_start(args, format); len = vsnprintf(buf.text, sizeof(buf.text), format, args); - va_end(args); if (len < 0) return; @@ -38,6 +35,15 @@ void ds_msg(const char *format, ...) debug_stream_slot_send_record(&buf.msg.hdr); } +void ds_msg(const char *format, ...) +{ + va_list args; + + va_start(args, format); + ds_vamsg(format, args); + va_end(args); +} + #if defined(CONFIG_EXCEPTION_DUMP_HOOK) /* The debug stream debug window slot is 4k, and when it is split * between the cores and the header/other overhead is removed, with 5 @@ -49,39 +55,46 @@ void ds_msg(const char *format, ...) * in bursts, and sending more than one record in short time makes the * host-side decoder lose track of things. */ -static struct { - struct debug_stream_text_msg msg; - char text[640]; -} __packed ds_buf[CONFIG_MP_MAX_NUM_CPUS]; -static int reports_sent_cpu[CONFIG_MP_MAX_NUM_CPUS]; -static size_t ds_pos[CONFIG_MP_MAX_NUM_CPUS]; + +/* Per-CPU state for exception dump and assert_print(). + * Cache-line aligned to avoid false sharing between cores. + */ +static struct ds_cpu_state { + struct { + struct debug_stream_text_msg msg; + char text[640]; + } __packed buf; + int reports_sent; + size_t pos; +} __aligned(CONFIG_DCACHE_LINE_SIZE) ds_cpu[CONFIG_MP_MAX_NUM_CPUS]; static void ds_exception_drain(bool flush) { unsigned int cpu = arch_proc_id(); + struct ds_cpu_state *cs = &ds_cpu[cpu]; if (flush) { - ds_pos[cpu] = 0; - reports_sent_cpu[cpu] = 0; + cs->pos = 0; return; } - if (ds_pos[cpu] == 0) + if (cs->pos == 0) return; - if (reports_sent_cpu[cpu] > 0) + if (cs->reports_sent > 0) return; - ds_buf[cpu].msg.hdr.id = DEBUG_STREAM_RECORD_ID_TEXT_MSG; - ds_buf[cpu].msg.hdr.size_words = - SOF_DIV_ROUND_UP(sizeof(ds_buf[cpu].msg) + ds_pos[cpu], - sizeof(ds_buf[cpu].msg.hdr.data[0])); + cs->buf.msg.hdr.id = DEBUG_STREAM_RECORD_ID_TEXT_MSG; + cs->buf.msg.hdr.size_words = + SOF_DIV_ROUND_UP(sizeof(cs->buf.msg) + cs->pos, + sizeof(cs->buf.msg.hdr.data[0])); + /* Make sure the possible up to 3 extra bytes at end of msg are '\0' */ - memset(ds_buf[cpu].text + ds_pos[cpu], 0, - ds_buf[cpu].msg.hdr.size_words * sizeof(ds_buf[cpu].msg.hdr.data[0]) - ds_pos[cpu]); - debug_stream_slot_send_record(&ds_buf[cpu].msg.hdr); - reports_sent_cpu[cpu] = 1; - ds_pos[cpu] = 0; + memset(cs->buf.text + cs->pos, 0, + cs->buf.msg.hdr.size_words * sizeof(cs->buf.msg.hdr.data[0]) - cs->pos); + debug_stream_slot_send_record(&cs->buf.msg.hdr); + cs->reports_sent = 1; + cs->pos = 0; } static void ds_exception_dump(const char *format, va_list args) @@ -90,11 +103,12 @@ static void ds_exception_dump(const char *format, va_list args) size_t avail; size_t written; unsigned int cpu = arch_proc_id(); + struct ds_cpu_state *cs = &ds_cpu[cpu]; - if (reports_sent_cpu[cpu] > 0) + if (cs->reports_sent > 0) return; - avail = sizeof(ds_buf[cpu].text) - ds_pos[cpu]; + avail = sizeof(cs->buf.text) - cs->pos; if (avail == 0) { ds_exception_drain(false); return; @@ -108,9 +122,9 @@ static void ds_exception_dump(const char *format, va_list args) format[0] == ' ' && format[1] == '*' && format[2] == '*' && format[3] == ' ') format += 4; - len = vsnprintf(ds_buf[cpu].text + ds_pos[cpu], avail, format, args); + len = vsnprintf(cs->buf.text + cs->pos, avail, format, args); if (len < 0) { - ds_pos[cpu] = 0; + cs->pos = 0; return; } @@ -122,18 +136,46 @@ static void ds_exception_dump(const char *format, va_list args) else written = (size_t)len; - ds_pos[cpu] += written; + cs->pos += written; - if (ds_pos[cpu] >= sizeof(ds_buf[cpu].text)) + if (cs->pos >= sizeof(cs->buf.text)) ds_exception_drain(false); } static int init_exception_dump_hook(void) { - set_exception_dump_hook(ds_exception_dump, ds_exception_drain); + arch_exception_set_dump_hook(ds_exception_dump, ds_exception_drain); LOG_INF("exception_dump_hook set"); return 0; } SYS_INIT(init_exception_dump_hook, APPLICATION, CONFIG_KERNEL_INIT_PRIORITY_DEFAULT); + +#if defined(CONFIG_SOF_DEBUG_STREAM_TEXT_MSG_ASSERT_PRINT) +void assert_print(const char *fmt, ...) +{ + va_list ap; + + /* Do not print assert after exception has been dumped */ + if (ds_cpu[arch_proc_id()].reports_sent > 0) + return; + + va_start(ap, fmt); +#if !defined(CONFIG_EXCEPTION_DUMP_HOOK_ONLY) + { + va_list ap2; + + va_copy(ap2, ap); +#endif + ds_vamsg(fmt, ap); +#if !defined(CONFIG_EXCEPTION_DUMP_HOOK_ONLY) + vprintk(fmt, ap2); + va_end(ap2); + } +#endif + ds_vamsg(fmt, ap); + va_end(ap); +} +EXPORT_SYMBOL(assert_print); +#endif #endif diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index f1af9d53a85b..2f5c048f82a4 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -115,23 +115,38 @@ static void idc_handler(struct k_p4wq_work *work) /* * Used for *target* CPUs, since the initiator (usually core 0) can launch - * several IDC messages at once + * several IDC messages at once. Also we need 2 work items per target core, + * because the p4wq thread might just have returned from the work handler, but + * hasn't released the work buffer yet (hasn't set thread pointer to NULL). + * Then submitting the same work item again can result in an assertion failure. */ -static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT]; +static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT * 2]; +/* Protect the above array */ +static K_MUTEX_DEFINE(idc_mutex); int idc_send_msg(struct idc_msg *msg, uint32_t mode) { struct idc *idc = *idc_get(); struct idc_payload *payload = idc_payload_get(idc, msg->core); unsigned int target_cpu = msg->core; - struct zephyr_idc_msg *zmsg = idc_work + target_cpu; + struct zephyr_idc_msg *zmsg = idc_work + target_cpu * 2; struct idc_msg *msg_cp = &zmsg->msg; struct k_p4wq_work *work = &zmsg->work; int ret; int idc_send_memcpy_err __unused; + k_mutex_lock(&idc_mutex, K_FOREVER); + + if (unlikely(work->thread)) { + /* See comment above the idc_work[] array. */ + zmsg++; + work = &zmsg->work; + msg_cp = &zmsg->msg; + } + idc_send_memcpy_err = memcpy_s(msg_cp, sizeof(*msg_cp), msg, sizeof(*msg)); assert(!idc_send_memcpy_err); + /* Same priority as the IPC thread which is an EDF task and under Zephyr */ work->priority = CONFIG_EDF_THREAD_PRIORITY; work->deadline = 0; @@ -158,6 +173,8 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode) k_p4wq_submit(q_zephyr_idc + target_cpu, work); + k_mutex_unlock(&idc_mutex); + #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS /* Increment performance counters */ io_perf_monitor_update_data(idc->io_perf_out_msg_count, 1); diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index e46fc10b9521..f72cc10273d1 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -24,6 +24,7 @@ struct dma_sg_elem_array; struct ipc_msg; +struct ipc4_message_request; /* validates internal non tail structures within IPC command structure */ #define IPC_IS_SIZE_INVALID(object) \ @@ -53,6 +54,26 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_SECONDARY_CORE BIT(2) #define IPC_TASK_POWERDOWN BIT(3) +enum ipc4_user_thread_req { + IPC4_USER_THREAD_REQ_NONE, + IPC4_USER_THREAD_REQ_GLB, + IPC4_USER_THREAD_REQ_MODULE, +}; + +#if defined(__ZEPHYR__) && CONFIG_SOF_IPC_USER_THREAD +struct ipc_user_thread_data { + struct k_thread thread; + struct k_spinlock lock; + struct k_sem req_sem; + struct k_sem reply_sem; + bool schedule_pending; + enum ipc4_user_thread_req req; + struct ipc4_message_request *msg; + struct ipc_msg *reply; + int ret; +}; +#endif + struct ipc { struct k_spinlock lock; /* locking mechanism */ void *comp_data; @@ -83,6 +104,9 @@ struct ipc { #ifdef __ZEPHYR__ struct k_work_delayable z_delayed_work; struct k_work_q ipc_send_wq; +#if CONFIG_SOF_IPC_USER_THREAD + struct ipc_user_thread_data ipc_user; +#endif #endif void *private; @@ -166,6 +190,67 @@ struct dai_data; */ int ipc_dai_data_config(struct dai_data *dd, struct comp_dev *dev); +/** + * \brief Processes IPC4 userspace module message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/** + * \brief Processes IPC4 userspace global message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/** + * \brief Processes IPC4 global message in IPC user thread. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply); + +/** + * \brief Processes IPC4 module message in IPC user thread. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply); + +/** + * \brief Increment the IPC compound message pre-start counter. + * @param[in] msg_id IPC message ID. + */ +void ipc_compound_pre_start(int msg_id); + +/** + * \brief Decrement the IPC compound message pre-start counter on return value status. + * @param[in] msg_id IPC message ID. + * @param[in] ret Return value of the IPC command. + * @param[in] delayed True if the reply is delayed. + */ +void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); + +/** + * \brief Complete the IPC compound message. + * @param[in] msg_id IPC message ID. + * @param[in] error Error code of the IPC command. + */ +void ipc_compound_msg_done(uint32_t msg_id, int error); + +/** + * \brief Wait for the IPC compound message to complete. + * @return 0 on success, error code otherwise on timeout. + */ +int ipc_wait_for_compound_msg(void); + /** * \brief create a IPC boot complete message. * @param[in] header header. diff --git a/src/include/user/debug_stream_text_msg.h b/src/include/user/debug_stream_text_msg.h index 3d246e305fc3..debfaad7042e 100644 --- a/src/include/user/debug_stream_text_msg.h +++ b/src/include/user/debug_stream_text_msg.h @@ -7,6 +7,7 @@ #define __SOC_DEBUG_STREAM_TEXT_MSG_H__ #include +#include /* * Debug Stream text message. @@ -21,5 +22,6 @@ struct debug_stream_text_msg { * CONFIG_SOF_DEBUG_STREAM_TEXT_MSG to enable this function. */ void ds_msg(const char *format, ...); +void ds_vamsg(const char *format, va_list ap); #endif /* __SOC_DEBUG_STREAM_TEXT_MSG_H__ */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 10f241784625..cbd9917f66c2 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -179,6 +179,151 @@ void ipc_send_queued_msg(void) #ifdef __ZEPHYR__ static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); +#if CONFIG_SOF_IPC_USER_THREAD +static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +static void ipc_user_thread(void *arg1, void *arg2, void *arg3); + +static void ipc_init_user_thread(struct ipc *ipc) +{ + struct k_thread *user_thread = &ipc->ipc_user.thread; + + k_spinlock_init(&ipc->ipc_user.lock); + k_sem_init(&ipc->ipc_user.req_sem, 0, 1); + k_sem_init(&ipc->ipc_user.reply_sem, 0, 1); + ipc->ipc_user.schedule_pending = false; + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + ipc->ipc_user.msg = NULL; + ipc->ipc_user.reply = NULL; + ipc->ipc_user.ret = 0; + + k_thread_create(user_thread, ipc_user_stack, + K_THREAD_STACK_SIZEOF(ipc_user_stack), + ipc_user_thread, ipc, NULL, NULL, + 1, 0, K_FOREVER); + + k_thread_cpu_pin(user_thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_name_set(user_thread, "ipc_user"); + k_thread_start(user_thread); +} + +static int ipc4_process_in_user_thread(struct ipc *ipc, enum ipc4_user_thread_req req, + struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + k_spinlock_key_t key; + k_spinlock_key_t ipc_key; + bool schedule_pending; + int ret; + + if (!ipc || !ipc4 || !reply) + return -EINVAL; + + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.req = req; + ipc->ipc_user.msg = ipc4; + ipc->ipc_user.reply = reply; + k_spin_unlock(&ipc->ipc_user.lock, key); + + k_sem_give(&ipc->ipc_user.req_sem); + k_sem_take(&ipc->ipc_user.reply_sem, K_FOREVER); + + key = k_spin_lock(&ipc->ipc_user.lock); + schedule_pending = ipc->ipc_user.schedule_pending; + ipc->ipc_user.schedule_pending = false; + req = ipc->ipc_user.req; + ret = ipc->ipc_user.ret; + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + k_spin_unlock(&ipc->ipc_user.lock, key); + + if (schedule_pending) { + ipc_key = k_spin_lock(&ipc->lock); + k_work_schedule_for_queue(&ipc->ipc_send_wq, &ipc->z_delayed_work, + K_USEC(IPC_PERIOD_USEC)); + k_spin_unlock(&ipc->lock, ipc_key); + } + + if (req != IPC4_USER_THREAD_REQ_NONE) + return -EINVAL; + + if (ret) + tr_err(&ipc_tr, "%#x %#x reply %d", ipc4->primary.dat, ipc4->extension.dat, + ret); + + return ret; +} + +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_process_in_user_thread(ipc_get(), IPC4_USER_THREAD_REQ_GLB, ipc4, reply); +} + +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_process_in_user_thread(ipc_get(), IPC4_USER_THREAD_REQ_MODULE, ipc4, reply); +} + +static void ipc_user_thread(void *arg1, void *arg2, void *arg3) +{ + struct ipc *ipc = arg1; + struct ipc4_message_request *ipc4; + struct ipc_msg *reply; + enum ipc4_user_thread_req req; + int ret; + k_spinlock_key_t key; + + ARG_UNUSED(arg2); + ARG_UNUSED(arg3); + tr_info(&ipc_tr, "user thread started"); + + while (1) { + k_sem_take(&ipc->ipc_user.req_sem, K_FOREVER); + + key = k_spin_lock(&ipc->ipc_user.lock); + req = ipc->ipc_user.req; + ipc4 = ipc->ipc_user.msg; + reply = ipc->ipc_user.reply; + k_spin_unlock(&ipc->ipc_user.lock, key); + + switch (req) { + case IPC4_USER_THREAD_REQ_GLB: + ret = ipc4_user_process_glb_message(ipc4, reply); + break; + case IPC4_USER_THREAD_REQ_MODULE: + ret = ipc4_user_process_module_message(ipc4, reply); + break; + default: + ret = -EINVAL; + break; + } + + if (ret) + tr_err(&ipc_tr, "%#x %#x failed: %d", + ipc4->primary.dat, ipc4->extension.dat, ret); + + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.ret = ret; + ipc->ipc_user.req = IPC4_USER_THREAD_REQ_NONE; + k_spin_unlock(&ipc->ipc_user.lock, key); + + k_sem_give(&ipc->ipc_user.reply_sem); + } +} +#else /* CONFIG_SOF_IPC_USER_THREAD */ + +int ipc4_process_glb_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_user_process_glb_message(ipc4, reply); +} + +int ipc4_process_module_message_in_user_thread(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + return ipc4_user_process_module_message(ipc4, reply); +} +#endif /* CONFIG_SOF_IPC_USER_THREAD */ #endif static void schedule_ipc_worker(void) @@ -189,6 +334,16 @@ static void schedule_ipc_worker(void) */ #ifdef __ZEPHYR__ struct ipc *ipc = ipc_get(); + #if CONFIG_SOF_IPC_USER_THREAD + k_spinlock_key_t key; + + if (k_current_get() == &ipc->ipc_user.thread) { + key = k_spin_lock(&ipc->ipc_user.lock); + ipc->ipc_user.schedule_pending = true; + k_spin_unlock(&ipc->ipc_user.lock, key); + return; + } + #endif k_work_schedule_for_queue(&ipc->ipc_send_wq, &ipc->z_delayed_work, K_USEC(IPC_PERIOD_USEC)); #endif @@ -339,10 +494,13 @@ __cold int ipc_init(struct sof *sof) k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); k_thread_name_set(thread, "ipc_send_wq"); - k_thread_resume(thread); k_work_init_delayable(&sof->ipc->z_delayed_work, ipc_work_handler); + + #if CONFIG_SOF_IPC_USER_THREAD + ipc_init_user_thread(sof->ipc); + #endif #endif return platform_ipc_init(sof->ipc); diff --git a/src/ipc/ipc4/CMakeLists.txt b/src/ipc/ipc4/CMakeLists.txt index 02c56130ed11..52908ed2c88a 100644 --- a/src/ipc/ipc4/CMakeLists.txt +++ b/src/ipc/ipc4/CMakeLists.txt @@ -3,7 +3,8 @@ # Files common to Zephyr and XTOS add_local_sources(sof dai.c - handler.c + handler-user.c + handler-kernel.c helper.c logging.c notification.c diff --git a/src/ipc/ipc4/handler-kernel.c b/src/ipc/ipc4/handler-kernel.c new file mode 100644 index 000000000000..a6003922ed14 --- /dev/null +++ b/src/ipc/ipc4/handler-kernel.c @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if CONFIG_SOF_BOOT_TEST +/* CONFIG_SOF_BOOT_TEST depends on Zephyr */ +#include +#endif + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +struct ipc4_msg_data { + struct ipc_cmd_hdr msg_in; /* local copy of current message from host header */ + struct ipc_cmd_hdr msg_out; /* local copy of current message to host header */ + atomic_t delayed_reply; + uint32_t delayed_error; +}; + +static struct ipc4_msg_data msg_data; + +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg msg_reply = {0, 0, 0, 0, LIST_INIT(msg_reply.list)}; + +static struct ipc_msg msg_notify = {0, 0, 0, 0, LIST_INIT(msg_notify.list)}; + +#if CONFIG_LIBRARY +static inline struct ipc4_message_request *ipc4_get_message_request(void) +{ + struct ipc *ipc = ipc_get(); + + return (struct ipc4_message_request *)ipc->comp_data; +} + +static inline void ipc4_send_reply(struct ipc4_message_reply *reply) +{ + struct ipc *ipc = ipc_get(); + int ret; + + /* copy the extension from the message reply */ + reply->extension.dat = msg_reply.extension; + ret = memcpy_s(ipc->comp_data, sizeof(*reply), reply, sizeof(*reply)); + assert(!ret); +} + +static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) +{ + const struct ipc4_pipeline_set_state_data *ppl_data; + struct ipc *ipc = ipc_get(); + + ppl_data = (const struct ipc4_pipeline_set_state_data *)ipc->comp_data; + + return ppl_data; +} +#else +static inline struct ipc4_message_request *ipc4_get_message_request(void) +{ + /* ignoring _hdr as it does not contain valid data in IPC4/IDC case */ + return ipc_from_hdr(&msg_data.msg_in); +} + +static inline void ipc4_send_reply(struct ipc4_message_reply *reply) +{ + struct ipc *ipc = ipc_get(); + char *data = ipc->comp_data; + + ipc_msg_send(&msg_reply, data, true); +} + +#endif + +__cold static bool is_any_ppl_active(void) +{ + struct ipc_comp_dev *icd; + struct list_item *clist; + + assert_can_be_cold(); + + list_for_item(clist, &ipc_get()->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->type != COMP_TYPE_PIPELINE) + continue; + + if (icd->pipeline->status == COMP_STATE_ACTIVE) + return true; + } + + return false; +} + +void ipc_compound_pre_start(int msg_id) +{ + /* ipc thread will wait for all scheduled tasks to be complete + * Use a reference count to check status of these tasks. + */ + atomic_add(&msg_data.delayed_reply, 1); +} + +void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) +{ + if (ret) { + ipc_cmd_err(&ipc_tr, "failed to process msg %d status %d", msg_id, ret); + atomic_set(&msg_data.delayed_reply, 0); + return; + } + + /* decrease counter if it is not scheduled by another thread */ + if (!delayed) + atomic_sub(&msg_data.delayed_reply, 1); +} + +void ipc_compound_msg_done(uint32_t msg_id, int error) +{ + if (!atomic_read(&msg_data.delayed_reply)) { + ipc_cmd_err(&ipc_tr, "unexpected delayed reply"); + return; + } + + atomic_sub(&msg_data.delayed_reply, 1); + + /* error reported in delayed pipeline task */ + if (error < 0) { + if (msg_id == SOF_IPC4_GLB_SET_PIPELINE_STATE) + msg_data.delayed_error = IPC4_PIPELINE_STATE_NOT_SET; + } +} + +#if CONFIG_LIBRARY +/* There is no parallel execution in testbench for scheduler and pipelines, so the result would + * be always IPC4_FAILURE. Therefore the compound messages handling is simplified. The pipeline + * triggers will require an explicit scheduler call to get the components to desired state. + */ +int ipc_wait_for_compound_msg(void) +{ + atomic_set(&msg_data.delayed_reply, 0); + return IPC4_SUCCESS; +} +#else +int ipc_wait_for_compound_msg(void) +{ + int try_count = 30; + + while (atomic_read(&msg_data.delayed_reply)) { + k_sleep(Z_TIMEOUT_US(250)); + + if (!try_count--) { + atomic_set(&msg_data.delayed_reply, 0); + ipc_cmd_err(&ipc_tr, "ipc4: failed to wait schedule thread"); + return IPC4_FAILURE; + } + } + + return IPC4_SUCCESS; +} +#endif + +#if CONFIG_LIBRARY_MANAGER +__cold static int ipc4_load_library(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_load_library library; + int ret; + + assert_can_be_cold(); + + library.header.dat = ipc4->primary.dat; + + ret = lib_manager_load_library(library.header.r.dma_id, library.header.r.lib_id, + ipc4->primary.r.type); + if (ret != 0) + return (ret == -EINVAL) ? IPC4_ERROR_INVALID_PARAM : IPC4_FAILURE; + + return IPC4_SUCCESS; +} +#endif + +static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) +{ + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + + switch (type) { + + /* Loads library (using Code Load or HD/A Host Output DMA) */ +#ifdef CONFIG_LIBRARY_MANAGER + case SOF_IPC4_GLB_LOAD_LIBRARY: + ret = ipc4_load_library(ipc4); + break; + case SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE: + ret = ipc4_load_library(ipc4); + break; +#endif + /* not a kernel level IPC message */ + default: + /* try and handle as a user IPC message */ + ret = ipc4_process_glb_message_in_user_thread(ipc4, &msg_reply); + break; + } + + return ret; +} + +/* disable power gating on core 0 */ +__cold static int ipc4_module_process_d0ix(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_set_d0ix d0ix; + uint32_t module_id, instance_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&d0ix, sizeof(d0ix), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + module_id = d0ix.primary.r.module_id; + instance_id = d0ix.primary.r.instance_id; + + tr_dbg(&ipc_tr, "%x : %x", module_id, instance_id); + + /* only module 0 can be used to set d0ix state */ + if (d0ix.primary.r.module_id || d0ix.primary.r.instance_id) { + ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); + return IPC4_INVALID_RESOURCE_ID; + } + + if (d0ix.extension.r.prevent_power_gating) + pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); + else + pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); + + return 0; +} + +/* enable/disable cores according to the state mask */ +__cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_set_dx dx; + struct ipc4_dx_state_info dx_info; + uint32_t module_id, instance_id; + uint32_t core_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&dx, sizeof(dx), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + module_id = dx.primary.r.module_id; + instance_id = dx.primary.r.instance_id; + + /* only module 0 can be used to set dx state */ + if (module_id || instance_id) { + ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); + return IPC4_INVALID_RESOURCE_ID; + } + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + sizeof(dx_info)); + ret = memcpy_s(&dx_info, sizeof(dx_info), + (const void *)MAILBOX_HOSTBOX_BASE, sizeof(dx_info)); + if (ret < 0) + return IPC4_FAILURE; + + /* check if core enable mask is valid */ + if (dx_info.core_mask > MASK(CONFIG_CORE_COUNT - 1, 0)) { + ipc_cmd_err(&ipc_tr, "CONFIG_CORE_COUNT: %d < core enable mask: %d", + CONFIG_CORE_COUNT, dx_info.core_mask); + return IPC4_ERROR_INVALID_PARAM; + } + + /* check primary core first */ + if ((dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) && + (dx_info.dx_mask & BIT(PLATFORM_PRIMARY_CORE_ID))) { + /* core0 can't be activated more, it's already active since we got here */ + ipc_cmd_err(&ipc_tr, "Core0 is already active"); + return IPC4_BAD_STATE; + } + + /* Activate/deactivate requested cores */ + for (core_id = 1; core_id < CONFIG_CORE_COUNT; core_id++) { + if ((dx_info.core_mask & BIT(core_id)) == 0) + continue; + + if (dx_info.dx_mask & BIT(core_id)) { + ret = cpu_enable_core(core_id); + if (ret != 0) { + ipc_cmd_err(&ipc_tr, "failed to enable core %d", core_id); + return IPC4_FAILURE; + } + } else { + cpu_disable_core(core_id); + if (cpu_is_core_enabled(core_id)) { + ipc_cmd_err(&ipc_tr, "failed to disable core %d", core_id); + return IPC4_FAILURE; + } + } + } + + /* Deactivating primary core if requested. */ + if (dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) { + if (cpu_enabled_cores() & ~BIT(PLATFORM_PRIMARY_CORE_ID)) { + ipc_cmd_err(&ipc_tr, "secondary cores 0x%x still active", + cpu_enabled_cores()); + return IPC4_BUSY; + } + + if (is_any_ppl_active()) { + ipc_cmd_err(&ipc_tr, "some pipelines are still active"); + return IPC4_BUSY; + } + +#if !CONFIG_ADSP_IMR_CONTEXT_SAVE + ret = llext_manager_store_to_dram(); + if (ret < 0) + ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", + ret); + +#if CONFIG_L3_HEAP + l3_heap_save(); +#endif +#endif + +#if defined(CONFIG_PM) + ipc_get()->task_mask |= IPC_TASK_POWERDOWN; +#endif + /* do platform specific suspending */ + platform_context_save(sof_get()); + +#if !defined(CONFIG_LIBRARY) && !defined(CONFIG_ZEPHYR_NATIVE_DRIVERS) + arch_irq_lock(); + platform_timer_stop(timer_get()); +#endif + ipc_get()->pm_prepare_D3 = 1; + } + + return IPC4_SUCCESS; +} + +__cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) +{ + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + + switch (type) { + case SOF_IPC4_MOD_SET_D0IX: + ret = ipc4_module_process_d0ix(ipc4); + break; + case SOF_IPC4_MOD_SET_DX: + ret = ipc4_module_process_dx(ipc4); + break; + default: + /* try and handle as a user IPC message */ + ret = ipc4_process_module_message_in_user_thread(ipc4, &msg_reply); + break; + } + + return ret; +} + +__cold struct ipc_cmd_hdr *mailbox_validate(void) +{ + struct ipc_cmd_hdr *hdr = ipc_get()->comp_data; + + assert_can_be_cold(); + + return hdr; +} + +struct ipc_cmd_hdr *ipc_compact_read_msg(void) +{ + int words; + + words = ipc_platform_compact_read_msg(&msg_data.msg_in, 2); + if (!words) + return mailbox_validate(); + + return &msg_data.msg_in; +} + +struct ipc_cmd_hdr *ipc_prepare_to_send(const struct ipc_msg *msg) +{ + msg_data.msg_out.pri = msg->header; + msg_data.msg_out.ext = msg->extension; + + if (msg->tx_size) + mailbox_dspbox_write(0, (uint32_t *)msg->tx_data, msg->tx_size); + + return &msg_data.msg_out; +} + +__cold void ipc_boot_complete_msg(struct ipc_cmd_hdr *header, uint32_t data) +{ + assert_can_be_cold(); + + header->pri = SOF_IPC4_FW_READY; + header->ext = 0; +} + +#if defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) +__cold void ipc_send_failed_power_transition_response(void) +{ + struct ipc4_message_request *request = ipc_from_hdr(&msg_data.msg_in); + struct ipc4_message_reply response; + + assert_can_be_cold(); + + response.primary.r.status = IPC4_POWER_TRANSITION_FAILED; + response.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; + response.primary.r.msg_tgt = request->primary.r.msg_tgt; + response.primary.r.type = request->primary.r.type; + + msg_reply.header = response.primary.dat; + list_init(&msg_reply.list); + + ipc_msg_send_direct(&msg_reply, NULL); +} +#endif /* defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) */ + +__cold void ipc_send_panic_notification(void) +{ + assert_can_be_cold(); + + msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_EXCEPTION_CAUGHT); + msg_notify.extension = cpu_get_id(); + msg_notify.tx_size = 0; + msg_notify.tx_data = NULL; + list_init(&msg_notify.list); + + ipc_msg_send_direct(&msg_notify, NULL); +} + +#ifdef CONFIG_LOG_BACKEND_ADSP_MTRACE + +static bool is_notification_queued(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + bool queued = false; + + key = k_spin_lock(&ipc->lock); + if (!list_is_empty(&msg->list)) + queued = true; + k_spin_unlock(&ipc->lock, key); + + return queued; +} + +/* Called from ipc_send_buffer_status_notify(), which is currently "hot" */ +void ipc_send_buffer_status_notify(void) +{ + /* a single msg_notify object is used */ + if (is_notification_queued(&msg_notify)) + return; + + msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); + msg_notify.extension = 0; + msg_notify.tx_size = 0; + + tr_dbg(&ipc_tr, "tx-notify\t: %#x|%#x", msg_notify.header, msg_notify.extension); + + ipc_msg_send(&msg_notify, NULL, true); +} +#endif + +void ipc_msg_reply(struct sof_ipc_reply *reply) +{ + struct ipc4_message_request in; + + in.primary.dat = msg_data.msg_in.pri; + ipc_compound_msg_done(in.primary.r.type, reply->error); +} + +void ipc_cmd(struct ipc_cmd_hdr *_hdr) +{ + struct ipc4_message_request *in = ipc4_get_message_request(); + enum ipc4_message_target target; +#ifdef CONFIG_DEBUG_IPC_TIMINGS + struct ipc4_message_request req; + uint64_t tstamp; +#endif + int err; + +#ifdef CONFIG_DEBUG_IPC_TIMINGS + req = *in; + tstamp = sof_cycle_get_64(); +#else + if (cpu_is_primary(cpu_get_id())) + tr_info(&ipc_tr, "rx\t: %#x|%#x", in->primary.dat, in->extension.dat); +#endif + /* no process on scheduled thread */ + atomic_set(&msg_data.delayed_reply, 0); + msg_data.delayed_error = 0; + msg_reply.tx_data = NULL; + msg_reply.tx_size = 0; + msg_reply.header = in->primary.dat; + msg_reply.extension = in->extension.dat; + + target = in->primary.r.msg_tgt; + + switch (target) { + case SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG: + err = ipc4_process_glb_message(in); + if (err) + ipc_cmd_err(&ipc_tr, "ipc4: FW_GEN_MSG failed with err %d", err); + break; + case SOF_IPC4_MESSAGE_TARGET_MODULE_MSG: + err = ipc4_process_module_message(in); + if (err) + ipc_cmd_err(&ipc_tr, "ipc4: MODULE_MSG failed with err %d", err); + break; + default: + /* should not reach here as we only have 2 message types */ + ipc_cmd_err(&ipc_tr, "ipc4: invalid target %d", target); + err = IPC4_UNKNOWN_MESSAGE_TYPE; + } + + /* FW sends an ipc message to host if request bit is clear */ + if (in->primary.r.rsp == SOF_IPC4_MESSAGE_DIR_MSG_REQUEST) { + struct ipc *ipc = ipc_get(); + struct ipc4_message_reply reply = {{0}}; + + /* Process flow and time stamp for IPC4 msg processed on secondary core : + * core 0 (primary core) core x (secondary core) + * # IPC msg thread #IPC delayed worker #core x idc thread + * ipc_task_ops.run() + * ipc_do_cmd() + * msg_reply.header = in->primary.dat + * ipc4_process_on_core(x) + * mask |= SECONDARY_CORE + * idc_send_message() + * Case 1: + * // Ipc msg processed by secondary core idc_ipc() + * if ((mask & SECONDARY_CORE)) ipc_cmd() + * return; ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * ipc_platform_send_msg + * ---------------------------------------------------------------------------- + * Case 2: + * idc_ipc() + * ipc_cmd() + * //Prepare reply msg + * msg_reply.header = + * reply.primary.dat; + * ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * if ((mask & IPC_TASK_SECONDARY_CORE)) + * return; + * // Ipc reply msg was prepared, so return + * if (msg_reply.header != in->primary.dat) + * return; + * ipc_platform_send_msg + * ---------------------------------------------------------------------------- + * Case 3: + * idc_ipc() + * ipc_cmd() + * //Prepare reply msg + * msg_reply.header = + * reply.primary.dat; + * ipc_msg_send() + * mask &= ~SECONDARY_CORE + * + * ipc_platform_send_msg + * + * if ((mask & IPC_TASK_SECONDARY_CORE)) + * return; + * // Ipc reply msg was prepared, so return + * if (msg_reply.header != in->primary.dat) + * return; + */ + + /* Reply prepared by secondary core */ + if ((ipc->task_mask & IPC_TASK_SECONDARY_CORE) && cpu_is_primary(cpu_get_id())) + return; + /* Reply has been prepared by secondary core */ + if (msg_reply.header != in->primary.dat) + return; + + /* Do not send reply for SET_DX if we are going to enter D3 + * The reply is going to be sent as part of the power down + * sequence + */ + if (ipc->task_mask & IPC_TASK_POWERDOWN) + return; + + if (ipc_wait_for_compound_msg() != 0) { + ipc_cmd_err(&ipc_tr, "ipc4: failed to send delayed reply"); + err = IPC4_FAILURE; + } + + /* copy contents of message received */ + reply.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; + reply.primary.r.msg_tgt = in->primary.r.msg_tgt; + reply.primary.r.type = in->primary.r.type; + if (msg_data.delayed_error) + reply.primary.r.status = msg_data.delayed_error; + else + reply.primary.r.status = err; + + msg_reply.header = reply.primary.dat; + +#ifdef CONFIG_DEBUG_IPC_TIMINGS + tr_info(&ipc_tr, "tx-reply\t: %#x|%#x to %#x|%#x in %llu us", msg_reply.header, + msg_reply.extension, req.primary.dat, req.extension.dat, + k_cyc_to_us_near64(sof_cycle_get_64() - tstamp)); +#else + tr_dbg(&ipc_tr, "tx-reply\t: %#x|%#x", msg_reply.header, + msg_reply.extension); +#endif + ipc4_send_reply(&reply); + } +} diff --git a/src/ipc/ipc4/handler.c b/src/ipc/ipc4/handler-user.c similarity index 68% rename from src/ipc/ipc4/handler.c rename to src/ipc/ipc4/handler-user.c index fc64c904ef80..f5940dbec271 100644 --- a/src/ipc/ipc4/handler.c +++ b/src/ipc/ipc4/handler-user.c @@ -58,36 +58,11 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); -struct ipc4_msg_data { - struct ipc_cmd_hdr msg_in; /* local copy of current message from host header */ - struct ipc_cmd_hdr msg_out; /* local copy of current message to host header */ - atomic_t delayed_reply; - uint32_t delayed_error; -}; - -static struct ipc4_msg_data msg_data; - +/* Userspace message context, copied in/out by kernel IPC thread. */ /* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg msg_reply = {0, 0, 0, 0, LIST_INIT(msg_reply.list)}; - -static struct ipc_msg msg_notify = {0, 0, 0, 0, LIST_INIT(msg_notify.list)}; +static struct ipc_msg *msg_reply; #if CONFIG_LIBRARY -static inline struct ipc4_message_request *ipc4_get_message_request(void) -{ - struct ipc *ipc = ipc_get(); - - return (struct ipc4_message_request *)ipc->comp_data; -} - -static inline void ipc4_send_reply(struct ipc4_message_reply *reply) -{ - struct ipc *ipc = ipc_get(); - - /* copy the extension from the message reply */ - reply->extension.dat = msg_reply.extension; - memcpy((char *)ipc->comp_data, reply, sizeof(*reply)); -} static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) { @@ -99,19 +74,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( return ppl_data; } #else -static inline struct ipc4_message_request *ipc4_get_message_request(void) -{ - /* ignoring _hdr as it does not contain valid data in IPC4/IDC case */ - return ipc_from_hdr(&msg_data.msg_in); -} - -static inline void ipc4_send_reply(struct ipc4_message_reply *reply) -{ - struct ipc *ipc = ipc_get(); - char *data = ipc->comp_data; - - ipc_msg_send(&msg_reply, data, true); -} static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data(void) { @@ -180,25 +142,6 @@ static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) return err; } -__cold static bool is_any_ppl_active(void) -{ - struct ipc_comp_dev *icd; - struct list_item *clist; - - assert_can_be_cold(); - - list_for_item(clist, &ipc_get()->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->type != COMP_TYPE_PIPELINE) - continue; - - if (icd->pipeline->status == COMP_STATE_ACTIVE) - return true; - } - - return false; -} - static struct ipc_comp_dev *pipeline_get_host_dev(struct ipc_comp_dev *ppl_icd) { struct ipc_comp_dev *host_dev; @@ -468,72 +411,6 @@ int ipc4_pipeline_trigger(struct ipc_comp_dev *ppl_icd, uint32_t cmd, bool *dela return ret; } -static void ipc_compound_pre_start(int msg_id) -{ - /* ipc thread will wait for all scheduled tasks to be complete - * Use a reference count to check status of these tasks. - */ - atomic_add(&msg_data.delayed_reply, 1); -} - -static void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) -{ - if (ret) { - ipc_cmd_err(&ipc_tr, "failed to process msg %d status %d", msg_id, ret); - atomic_set(&msg_data.delayed_reply, 0); - return; - } - - /* decrease counter if it is not scheduled by another thread */ - if (!delayed) - atomic_sub(&msg_data.delayed_reply, 1); -} - -static void ipc_compound_msg_done(uint32_t msg_id, int error) -{ - if (!atomic_read(&msg_data.delayed_reply)) { - ipc_cmd_err(&ipc_tr, "unexpected delayed reply"); - return; - } - - atomic_sub(&msg_data.delayed_reply, 1); - - /* error reported in delayed pipeline task */ - if (error < 0) { - if (msg_id == SOF_IPC4_GLB_SET_PIPELINE_STATE) - msg_data.delayed_error = IPC4_PIPELINE_STATE_NOT_SET; - } -} - -#if CONFIG_LIBRARY -/* There is no parallel execution in testbench for scheduler and pipelines, so the result would - * be always IPC4_FAILURE. Therefore the compound messages handling is simplified. The pipeline - * triggers will require an explicit scheduler call to get the components to desired state. - */ -static int ipc_wait_for_compound_msg(void) -{ - atomic_set(&msg_data.delayed_reply, 0); - return IPC4_SUCCESS; -} -#else -static int ipc_wait_for_compound_msg(void) -{ - int try_count = 30; - - while (atomic_read(&msg_data.delayed_reply)) { - k_sleep(Z_TIMEOUT_US(250)); - - if (!try_count--) { - atomic_set(&msg_data.delayed_reply, 0); - ipc_cmd_err(&ipc_tr, "ipc4: failed to wait schedule thread"); - return IPC4_FAILURE; - } - } - - return IPC4_SUCCESS; -} -#endif - __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper(void) { assert_can_be_cold(); @@ -670,25 +547,6 @@ static int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) return ret; } -#if CONFIG_LIBRARY_MANAGER -__cold static int ipc4_load_library(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_load_library library; - int ret; - - assert_can_be_cold(); - - library.header.dat = ipc4->primary.dat; - - ret = lib_manager_load_library(library.header.r.dma_id, library.header.r.lib_id, - ipc4->primary.r.type); - if (ret != 0) - return (ret == -EINVAL) ? IPC4_ERROR_INVALID_PARAM : IPC4_FAILURE; - - return IPC4_SUCCESS; -} -#endif - __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) { assert_can_be_cold(); @@ -758,11 +616,11 @@ __cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, &reply_size); /* reply size is returned in header extension dword */ - msg_reply.extension = reply_size; + msg_reply->extension = reply_size; if (reply_size > 0) { - msg_reply.tx_data = ipc->comp_data; - msg_reply.tx_size = reply_size; + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; } return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; @@ -782,12 +640,14 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) #endif } -static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) { uint32_t type; int ret; type = ipc4->primary.r.type; + msg_reply = reply; switch (type) { case SOF_IPC4_GLB_BOOT_CONFIG: @@ -822,15 +682,6 @@ static int ipc4_process_glb_message(struct ipc4_message_request *ipc4) ret = IPC4_UNAVAILABLE; break; - /* Loads library (using Code Load or HD/A Host Output DMA) */ -#ifdef CONFIG_LIBRARY_MANAGER - case SOF_IPC4_GLB_LOAD_LIBRARY: - ret = ipc4_load_library(ipc4); - break; - case SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE: - ret = ipc4_load_library(ipc4); - break; -#endif case SOF_IPC4_GLB_INTERNAL_MESSAGE: ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); ret = IPC4_UNAVAILABLE; @@ -985,7 +836,7 @@ static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4 } if (!set) - msg_reply.extension = config->extension.dat; + msg_reply->extension = config->extension.dat; return ret; } @@ -1020,7 +871,7 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, +__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, bool init_block, bool final_block, @@ -1207,9 +1058,9 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ if (ret) return ret; - msg_reply.extension = reply.extension.dat; - msg_reply.tx_size = data_offset; - msg_reply.tx_data = data; + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; return ret; } @@ -1377,145 +1228,8 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) return ret; } -/* disable power gating on core 0 */ -__cold static int ipc4_module_process_d0ix(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_set_d0ix d0ix; - uint32_t module_id, instance_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&d0ix, sizeof(d0ix), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - module_id = d0ix.primary.r.module_id; - instance_id = d0ix.primary.r.instance_id; - - tr_dbg(&ipc_tr, "%x : %x", module_id, instance_id); - - /* only module 0 can be used to set d0ix state */ - if (d0ix.primary.r.module_id || d0ix.primary.r.instance_id) { - ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); - return IPC4_INVALID_RESOURCE_ID; - } - - if (d0ix.extension.r.prevent_power_gating) - pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); - else - pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); - - return 0; -} - -/* enable/disable cores according to the state mask */ -__cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_set_dx dx; - struct ipc4_dx_state_info dx_info; - uint32_t module_id, instance_id; - uint32_t core_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&dx, sizeof(dx), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - module_id = dx.primary.r.module_id; - instance_id = dx.primary.r.instance_id; - - /* only module 0 can be used to set dx state */ - if (module_id || instance_id) { - ipc_cmd_err(&ipc_tr, "invalid resource id %x : %x", module_id, instance_id); - return IPC4_INVALID_RESOURCE_ID; - } - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - sizeof(dx_info)); - ret = memcpy_s(&dx_info, sizeof(dx_info), - (const void *)MAILBOX_HOSTBOX_BASE, sizeof(dx_info)); - if (ret < 0) - return IPC4_FAILURE; - - /* check if core enable mask is valid */ - if (dx_info.core_mask > MASK(CONFIG_CORE_COUNT - 1, 0)) { - ipc_cmd_err(&ipc_tr, "CONFIG_CORE_COUNT: %d < core enable mask: %d", - CONFIG_CORE_COUNT, dx_info.core_mask); - return IPC4_ERROR_INVALID_PARAM; - } - - /* check primary core first */ - if ((dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) && - (dx_info.dx_mask & BIT(PLATFORM_PRIMARY_CORE_ID))) { - /* core0 can't be activated more, it's already active since we got here */ - ipc_cmd_err(&ipc_tr, "Core0 is already active"); - return IPC4_BAD_STATE; - } - - /* Activate/deactivate requested cores */ - for (core_id = 1; core_id < CONFIG_CORE_COUNT; core_id++) { - if ((dx_info.core_mask & BIT(core_id)) == 0) - continue; - - if (dx_info.dx_mask & BIT(core_id)) { - ret = cpu_enable_core(core_id); - if (ret != 0) { - ipc_cmd_err(&ipc_tr, "failed to enable core %d", core_id); - return IPC4_FAILURE; - } - } else { - cpu_disable_core(core_id); - if (cpu_is_core_enabled(core_id)) { - ipc_cmd_err(&ipc_tr, "failed to disable core %d", core_id); - return IPC4_FAILURE; - } - } - } - - /* Deactivating primary core if requested. */ - if (dx_info.core_mask & BIT(PLATFORM_PRIMARY_CORE_ID)) { - if (cpu_enabled_cores() & ~BIT(PLATFORM_PRIMARY_CORE_ID)) { - ipc_cmd_err(&ipc_tr, "secondary cores 0x%x still active", - cpu_enabled_cores()); - return IPC4_BUSY; - } - - if (is_any_ppl_active()) { - ipc_cmd_err(&ipc_tr, "some pipelines are still active"); - return IPC4_BUSY; - } - -#if !CONFIG_ADSP_IMR_CONTEXT_SAVE - ret = llext_manager_store_to_dram(); - if (ret < 0) - ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", - ret); - -#if CONFIG_L3_HEAP - l3_heap_save(); -#endif -#endif - -#if defined(CONFIG_PM) - ipc_get()->task_mask |= IPC_TASK_POWERDOWN; -#endif - /* do platform specific suspending */ - platform_context_save(sof_get()); - -#if !defined(CONFIG_LIBRARY) && !defined(CONFIG_ZEPHYR_NATIVE_DRIVERS) - arch_irq_lock(); - platform_timer_stop(timer_get()); -#endif - ipc_get()->pm_prepare_D3 = 1; - } - - return IPC4_SUCCESS; -} - -__cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) { uint32_t type; int ret; @@ -1523,6 +1237,7 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) assert_can_be_cold(); type = ipc4->primary.r.type; + msg_reply = reply; switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: @@ -1549,12 +1264,6 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) case SOF_IPC4_MOD_DELETE_INSTANCE: ret = ipc4_delete_module_instance(ipc4); break; - case SOF_IPC4_MOD_SET_D0IX: - ret = ipc4_module_process_d0ix(ipc4); - break; - case SOF_IPC4_MOD_SET_DX: - ret = ipc4_module_process_dx(ipc4); - break; case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: ret = IPC4_UNAVAILABLE; @@ -1566,258 +1275,3 @@ __cold static int ipc4_process_module_message(struct ipc4_message_request *ipc4) return ret; } - -__cold struct ipc_cmd_hdr *mailbox_validate(void) -{ - struct ipc_cmd_hdr *hdr = ipc_get()->comp_data; - - assert_can_be_cold(); - - return hdr; -} - -struct ipc_cmd_hdr *ipc_compact_read_msg(void) -{ - int words; - - words = ipc_platform_compact_read_msg(&msg_data.msg_in, 2); - if (!words) - return mailbox_validate(); - - return &msg_data.msg_in; -} - -struct ipc_cmd_hdr *ipc_prepare_to_send(const struct ipc_msg *msg) -{ - msg_data.msg_out.pri = msg->header; - msg_data.msg_out.ext = msg->extension; - - if (msg->tx_size) - mailbox_dspbox_write(0, (uint32_t *)msg->tx_data, msg->tx_size); - - return &msg_data.msg_out; -} - -__cold void ipc_boot_complete_msg(struct ipc_cmd_hdr *header, uint32_t data) -{ - assert_can_be_cold(); - - header->pri = SOF_IPC4_FW_READY; - header->ext = 0; -} - -#if defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) -__cold void ipc_send_failed_power_transition_response(void) -{ - struct ipc4_message_request *request = ipc_from_hdr(&msg_data.msg_in); - struct ipc4_message_reply response; - - assert_can_be_cold(); - - response.primary.r.status = IPC4_POWER_TRANSITION_FAILED; - response.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; - response.primary.r.msg_tgt = request->primary.r.msg_tgt; - response.primary.r.type = request->primary.r.type; - - msg_reply.header = response.primary.dat; - list_init(&msg_reply.list); - - ipc_msg_send_direct(&msg_reply, NULL); -} -#endif /* defined(CONFIG_PM_DEVICE) && defined(CONFIG_INTEL_ADSP_IPC) */ - -__cold void ipc_send_panic_notification(void) -{ - assert_can_be_cold(); - - msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_EXCEPTION_CAUGHT); - msg_notify.extension = cpu_get_id(); - msg_notify.tx_size = 0; - msg_notify.tx_data = NULL; - list_init(&msg_notify.list); - - ipc_msg_send_direct(&msg_notify, NULL); -} - -#ifdef CONFIG_LOG_BACKEND_ADSP_MTRACE - -static bool is_notification_queued(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - bool queued = false; - - key = k_spin_lock(&ipc->lock); - if (!list_is_empty(&msg->list)) - queued = true; - k_spin_unlock(&ipc->lock, key); - - return queued; -} - -/* Called from ipc_send_buffer_status_notify(), which is currently "hot" */ -void ipc_send_buffer_status_notify(void) -{ - /* a single msg_notify object is used */ - if (is_notification_queued(&msg_notify)) - return; - - msg_notify.header = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); - msg_notify.extension = 0; - msg_notify.tx_size = 0; - - tr_dbg(&ipc_tr, "tx-notify\t: %#x|%#x", msg_notify.header, msg_notify.extension); - - ipc_msg_send(&msg_notify, NULL, true); -} -#endif - -void ipc_msg_reply(struct sof_ipc_reply *reply) -{ - struct ipc4_message_request in; - - in.primary.dat = msg_data.msg_in.pri; - ipc_compound_msg_done(in.primary.r.type, reply->error); -} - -void ipc_cmd(struct ipc_cmd_hdr *_hdr) -{ - struct ipc4_message_request *in = ipc4_get_message_request(); - enum ipc4_message_target target; -#ifdef CONFIG_DEBUG_IPC_TIMINGS - struct ipc4_message_request req; - uint64_t tstamp; -#endif - int err; - -#ifdef CONFIG_DEBUG_IPC_TIMINGS - req = *in; - tstamp = sof_cycle_get_64(); -#else - if (cpu_is_primary(cpu_get_id())) - tr_info(&ipc_tr, "rx\t: %#x|%#x", in->primary.dat, in->extension.dat); -#endif - /* no process on scheduled thread */ - atomic_set(&msg_data.delayed_reply, 0); - msg_data.delayed_error = 0; - msg_reply.tx_data = NULL; - msg_reply.tx_size = 0; - msg_reply.header = in->primary.dat; - msg_reply.extension = in->extension.dat; - - target = in->primary.r.msg_tgt; - - switch (target) { - case SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG: - err = ipc4_process_glb_message(in); - if (err) - ipc_cmd_err(&ipc_tr, "ipc4: FW_GEN_MSG failed with err %d", err); - break; - case SOF_IPC4_MESSAGE_TARGET_MODULE_MSG: - err = ipc4_process_module_message(in); - if (err) - ipc_cmd_err(&ipc_tr, "ipc4: MODULE_MSG failed with err %d", err); - break; - default: - /* should not reach here as we only have 2 message types */ - ipc_cmd_err(&ipc_tr, "ipc4: invalid target %d", target); - err = IPC4_UNKNOWN_MESSAGE_TYPE; - } - - /* FW sends an ipc message to host if request bit is clear */ - if (in->primary.r.rsp == SOF_IPC4_MESSAGE_DIR_MSG_REQUEST) { - struct ipc *ipc = ipc_get(); - struct ipc4_message_reply reply = {{0}}; - - /* Process flow and time stamp for IPC4 msg processed on secondary core : - * core 0 (primary core) core x (secondary core) - * # IPC msg thread #IPC delayed worker #core x idc thread - * ipc_task_ops.run() - * ipc_do_cmd() - * msg_reply.header = in->primary.dat - * ipc4_process_on_core(x) - * mask |= SECONDARY_CORE - * idc_send_message() - * Case 1: - * // Ipc msg processed by secondary core idc_ipc() - * if ((mask & SECONDARY_CORE)) ipc_cmd() - * return; ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * ipc_platform_send_msg - * ---------------------------------------------------------------------------- - * Case 2: - * idc_ipc() - * ipc_cmd() - * //Prepare reply msg - * msg_reply.header = - * reply.primary.dat; - * ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * if ((mask & IPC_TASK_SECONDARY_CORE)) - * return; - * // Ipc reply msg was prepared, so return - * if (msg_reply.header != in->primary.dat) - * return; - * ipc_platform_send_msg - * ---------------------------------------------------------------------------- - * Case 3: - * idc_ipc() - * ipc_cmd() - * //Prepare reply msg - * msg_reply.header = - * reply.primary.dat; - * ipc_msg_send() - * mask &= ~SECONDARY_CORE - * - * ipc_platform_send_msg - * - * if ((mask & IPC_TASK_SECONDARY_CORE)) - * return; - * // Ipc reply msg was prepared, so return - * if (msg_reply.header != in->primary.dat) - * return; - */ - - /* Reply prepared by secondary core */ - if ((ipc->task_mask & IPC_TASK_SECONDARY_CORE) && cpu_is_primary(cpu_get_id())) - return; - /* Reply has been prepared by secondary core */ - if (msg_reply.header != in->primary.dat) - return; - - /* Do not send reply for SET_DX if we are going to enter D3 - * The reply is going to be sent as part of the power down - * sequence - */ - if (ipc->task_mask & IPC_TASK_POWERDOWN) - return; - - if (ipc_wait_for_compound_msg() != 0) { - ipc_cmd_err(&ipc_tr, "ipc4: failed to send delayed reply"); - err = IPC4_FAILURE; - } - - /* copy contents of message received */ - reply.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REPLY; - reply.primary.r.msg_tgt = in->primary.r.msg_tgt; - reply.primary.r.type = in->primary.r.type; - if (msg_data.delayed_error) - reply.primary.r.status = msg_data.delayed_error; - else - reply.primary.r.status = err; - - msg_reply.header = reply.primary.dat; - -#ifdef CONFIG_DEBUG_IPC_TIMINGS - tr_info(&ipc_tr, "tx-reply\t: %#x|%#x to %#x|%#x in %llu us", msg_reply.header, - msg_reply.extension, req.primary.dat, req.extension.dat, - k_cyc_to_us_near64(sof_cycle_get_64() - tstamp)); -#else - tr_dbg(&ipc_tr, "tx-reply\t: %#x|%#x", msg_reply.header, - msg_reply.extension); -#endif - ipc4_send_reply(&reply); - } -} diff --git a/zephyr/Kconfig b/zephyr/Kconfig index c744b965a07f..b4d7a70e81c3 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -256,3 +256,19 @@ config STACK_SIZE_IPC_TX default 2048 help IPC sender work-queue thread stack size. Keep a power of 2. + +config SOF_IPC_USER_THREAD + bool "Enable IPC user thread" + default y + depends on IPC_MAJOR_4 + help + Enable a dedicated IPC user thread and related synchronization + primitives in IPC common code. + +config SOF_IPC_USER_THREAD_STACK_SIZE + int "IPC user thread stack size" + default 4096 + depends on SOF_IPC_USER_THREAD + help + Stack size for the simulated userspace IPC thread. + Keep a power of 2.