diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 3335c108fa29..5fc9f749e7fb 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -12,7 +12,10 @@ #ifndef __SOF_IPC4_HANDLER_H__ #define __SOF_IPC4_HANDLER_H__ +struct comp_dev; +struct comp_driver; struct ipc4_message_request; +struct ipc4_pipeline_set_state; /** * \brief Processes an IPC4 userspace module message. @@ -69,6 +72,73 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_ */ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); +/** + * \brief Get the first pipeline ID of a SET_PIPELINE_STATE message. + * \ingroup ipc4_envelope_api + * @param[in] ipc4 IPC4 message request structure. + * @param[in] state Copy of the SET_PIPELINE_STATE message header. + * @param[out] ppl_id Receives the pipeline ID array of a multi-pipeline + * message, NULL otherwise (may be NULL). + * @param[out] ppl_count Receives the number of pipelines (may be NULL). + * @return The first pipeline ID, or a negative error code. + */ +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count); + +/** + * \brief Copy the kcontrol payload header of a LARGE_CONFIG_GET from the hostbox. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] param_id Large config parameter ID. + * @param[out] data_out Reply data buffer. + * @param[in] data_size Size of the payload in the hostbox. + */ +void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size); + +/** + * \brief Process a vendor config LARGE_CONFIG_GET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in,out] data_off_size Payload size in, reply data size out. + * @param[out] data_out Reply data buffer. + * @param[in] data_in Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in); + +/** + * \brief Process a vendor config LARGE_CONFIG_SET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] module_id Module ID, for error messages. + * @param[in] instance_id Instance ID, for error messages. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in] data_off_size Payload size. + * @param[in] data Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + uint32_t module_id, + uint32_t instance_id, + bool init_block, + bool final_block, + uint32_t data_off_size, + const char *data); + /** * \brief Complete an IPC compound message transaction and signal waiting threads. * \ingroup ipc4_envelope_api diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 64839766dfb3..d0bb5fca9d6e 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -356,6 +356,20 @@ void ipc_complete_cmd(struct ipc *ipc); extern bool ipc_enter_gdb; #ifdef CONFIG_SOF_USERSPACE_LL +/* Events that wake the user-space IPC thread */ +#define IPC_USER_EVENT_CMD BIT(0) +#define IPC_USER_EVENT_STOP BIT(1) +#define IPC_USER_EVENT_XCORE BIT(2) + +/** + * @brief User-space IPC thread entry point. + * + * @param p1 struct ipc_user shared with the kernel + * @param p2 Core ID + * @param p3 Unused + */ +void ipc_user_thread_fn(void *p1, void *p2, void *p3); + /** * @brief Forward an IPC command to the user-space thread and wait for it. * diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index d072abc16a24..1711a0fe3a1c 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -48,6 +48,11 @@ struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); int ipc4_add_comp_dev(struct comp_dev *dev); +struct comp_ipc_config; +int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init); +unsigned char *ipc4_get_comp_new_data(void); +int ipc_comp_free_remote(struct comp_dev *dev); #ifdef CONFIG_SOF_USERSPACE_LL struct ipc4_message_request; struct comp_driver; diff --git a/src/ipc/README.md b/src/ipc/README.md index c240cc54a50e..4e7ea8679663 100644 --- a/src/ipc/README.md +++ b/src/ipc/README.md @@ -1,10 +1,14 @@ # Inter-Processor Communication (IPC) Core Architecture > **Runs in:** Boundary / bridge — the kernel receives and dispatches IPC, then forwards -> module-facing commands to a user thread. The split is visible in the file names -> (`src/kernel/ipc/ipc4/handler-kernel.c` vs `ipc4/handler-user.c`, -> `ipc4/notification-user.c`), and this subsystem hosts the `ipc_msg_*`, `ipc_compound_*` and -> `send_resource_notif` syscalls. See +> module-facing commands to a user thread. The kernel halves of the mixed files are in +> `src/kernel/ipc` (`ipc-common-kernel.c`, `ipc4/handler-kernel.c`, +> `ipc4/handler-user-kernel.c`, `ipc4/handler-ll-user.c`, `ipc4/handler-ll-kernel.c`, +> `ipc4/helper-kernel.c`), the user halves stay here (`ipc-common.c`, `ipc4/handler-user.c`, +> `ipc4/helper.c`, `ipc4/notification-user.c`), and this subsystem hosts the `ipc_msg_*`, +> `ipc_compound_*` and `send_resource_notif` syscalls. Which IPC4 messages are handled in the +> kernel, in user space or in both is listed in +> [IPC4 User/Kernel Partitioning](ipc4/README.md#userkernel-partitioning). See also > [User/kernel split](../include/sof/userspace/README.md). This directory contains the common foundation for all Inter-Processor Communication (IPC) within the Sound Open Firmware (SOF) project. It bridges the gap between hardware mailbox interrupts and the version-specific (IPC3/IPC4) message handlers. diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index b74b4a679d7e..aaa3d6d98123 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -50,25 +50,10 @@ #endif #include -#include -LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC code that runs in kernel mode is in src/kernel/ipc/ipc-common-kernel.c */ -SOF_DEFINE_REG_UUID(ipc); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); - -#ifdef CONFIG_SOF_USERSPACE_LL -K_APPMEM_PARTITION_DEFINE(ipc_context_part); - -K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; - -struct ipc *ipc_get(void) -{ - return &ipc_context; -} -#endif +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, uint32_t extension, uint32_t size) @@ -101,46 +86,6 @@ struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, return msg; } -int ipc_process_on_core(uint32_t core, bool blocking) -{ - struct ipc *ipc = ipc_get(); - struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; - int ret; - - /* check if requested core is enabled */ - if (!cpu_is_core_enabled(core)) { - tr_err(&ipc_tr, "core #%d is disabled", core); - return -EACCES; - } - -#if CONFIG_IPC_MAJOR_3 - /* The other core will write there its response */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); -#endif - - /* - * If the primary core is waiting for secondary cores to complete, it - * will also reply to the host - */ - if (!blocking) { - k_spinlock_key_t key; - - ipc->core = core; - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_SECONDARY_CORE; - k_spin_unlock(&ipc->lock, key); - } - - /* send IDC message */ - ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); - if (ret < 0) - return ret; - - /* reply written by other core */ - return 1; -} - /* * Components, buffers and pipelines are stored in the same lists, hence * type and ID have to be used for the identification. @@ -202,320 +147,7 @@ struct ipc_comp_dev *ipc_get_ppl_comp(struct ipc *ipc, uint32_t pipeline_id, int return next_ppl_icd; } -void ipc_send_queued_msg(void) -{ - struct ipc *ipc = ipc_get(); - struct ipc_msg *msg; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - - if (ipc->pm_prepare_D3) - goto out; - - /* any messages to send ? */ - if (list_is_empty(&ipc->msg_list)) - goto out; - - msg = list_first_item(&ipc->msg_list, struct ipc_msg, - list); - - if (ipc_platform_send_msg(msg) == 0) { - /* Remove the message from the list if it has been successfully sent. */ - list_item_del(&msg->list); - /* Invoke a callback to notify that the message has been sent. */ - if (msg->callback) - msg->callback(msg); -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); -#endif - } -out: - k_spin_unlock(&ipc->lock, key); -} - -#ifdef __ZEPHYR__ -static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); - -/* - * Kernel-only: these must not live in struct ipc, which user threads can - * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of - * a supervisor thread and the work item holds the handler it calls. - */ -static struct k_work_q ipc_send_wq; -static struct k_work_delayable ipc_send_work; - -struct k_work_q *ipc_send_wq_get(void) -{ - return &ipc_send_wq; -} -#endif - -static void schedule_ipc_worker(void) -{ - /* - * note: in XTOS builds, this is handled in - * task_main_primary_core() - */ -#ifdef __ZEPHYR__ - k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); -#endif -} - -__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - assert_can_be_cold(); - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - ipc_platform_send_msg_direct(msg); - - k_spin_unlock(&ipc->lock, key); -} - -void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - /* - * note: This function can be executed in LL or EDF context, from any core. - * In Zephyr builds, there is IPC queue that is always handled by the primary core, - * whereas submitting to the queue is allowed from any core. Therefore disable option - * of sending IPC immediately by any context/core to secure IPC registers/mailbox - * access. - */ -#ifndef __ZEPHYR__ - /* try to send critical notifications right away */ - if (high_priority) { - ret = ipc_platform_send_msg(msg); - if (!ret) { - k_spin_unlock(&ipc->lock, key); - return; - } - } -#endif - /* add to queue unless already there */ - if (list_is_empty(&msg->list)) { - if (high_priority) - list_item_prepend(&msg->list, &ipc->msg_list); - else - list_item_append(&msg->list, &ipc->msg_list); - } - - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -EXPORT_SYMBOL(ipc_msg_send); - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - /* No caller check: DP modules send notifications, see mod_ipc_msg_init() */ - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); - - K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); - - if (msg->tx_size > 0) { - K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); - } - - /* If data != NULL and tx_size > 0, verify the data buffer */ - if (data && msg->tx_size > 0) - K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); - - z_impl_ipc_msg_send(msg, data, high_priority); -} -#include -#endif /* CONFIG_USERSPACE */ - -void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - list_item_del(&msg->list); - k_spin_unlock(&ipc->lock, key); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) -{ - /* No caller check: called by mod_ipc_msg_free() in DP module threads */ - struct ipc *ipc = ipc_get(); - struct list_item *mlist, *_mlist; - bool found = false; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - /* - * special case: empty list was passed. we can't trust where - * list->prev points to, so do not pass to - * z_impl_ipc_msg_list_remove(), but handle here - */ - if (list_is_empty(&msg->list)) - return; - - list_for_item_safe(mlist, _mlist, &ipc->msg_list) { - if (mlist == &msg->list) { - found = true; - break; - } - } - K_OOPS(K_SYSCALL_VERIFY(found)); - z_impl_ipc_msg_list_remove(msg); -} -#include -#endif /* CONFIG_USERSPACE */ - -#ifdef __ZEPHYR__ -static void ipc_work_handler(struct k_work *work) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - ipc_send_queued_msg(); - - key = k_spin_lock(&ipc->lock); - - if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -#endif - -void ipc_schedule_process(struct ipc *ipc) -{ -#if CONFIG_TWB_IPC_TASK - schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); -#else - schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); -#endif -} - #ifdef CONFIG_SOF_USERSPACE_LL -/* Generic user-space IPC handling thread infrastructure. Protocol-specific - * command interpretation lives in ipc_user_thread_dispatch(), implemented by - * the active IPC major (see src/ipc/ipc4/). - */ - -#define IPC_USER_EVENT_CMD BIT(0) -#define IPC_USER_EVENT_STOP BIT(1) -#define IPC_USER_EVENT_XCORE BIT(2) - -BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, - "cross-core request payload smaller than IDC payload"); -#if CONFIG_IPC_MAJOR_4 -BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == - sizeof(struct ipc4_module_bind_unbind), - "cross-core request bind data size mismatch"); -#endif - -SOF_DEFINE_REG_UUID(sec_core_init); - -static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); - -/* - * Kernel-only copies of the IPC user thread state. struct ipc_user and the - * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel - * code must use these instead of reading kernel object pointers from there. - */ -static struct ipc_user *ipc_user_kptr; -static struct k_sem *ipc_user_ksem; -static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; -static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; - -/* Cross-core request slot semaphores, static so they're never freed */ -static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; -static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; - -/** - * @brief Forward an IPC command to the user-space thread. - * - * Called from kernel context (IPC EDF task) to forward the message words - * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in - * task_mask so the host is not signaled until the user thread completes. - * Blocks until the user thread finishes processing and returns the result. - * - * @param primary Primary message word - * @param extension Extension message word - * @return Result from user thread processing - */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) -{ - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc_user_kptr; - k_spinlock_key_t key; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - LOG_DBG("IPC: forward cmd %08x", primary); - - /* Copy message words — original buffer may be reused */ - pdata->ipc_msg_pri = primary; - pdata->ipc_msg_ext = extension; - pdata->ipc = ipc; - - /* Prevent host completion until user thread finishes */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_IN_THREAD; - k_spin_unlock(&ipc->lock, key); - - /* - * Wake the user thread. Post, not set: a cross-core request - * (IPC_USER_EVENT_XCORE) may be pending on the same event. - */ - k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); - - /* Wait for user thread to complete */ - ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); - if (ret) { - LOG_ERR("IPC user: sem error %d\n", ret); - /* fall through to complete the cmd */ - } - - /* Clear the task mask bit and check for completion */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_IN_THREAD; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); - - return !ret ? pdata->result : ret; -} - void ipc_user_xcore_reserve(uint32_t cores) { struct ipc_user *ipc_user = ipc_get()->ipc_user_pdata; @@ -651,7 +283,7 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * User-space IPC thread entry point. p1 points to the ipc_user context * shared with the kernel launcher. */ -static void ipc_user_thread_fn(void *p1, void *p2, void *p3) +void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; unsigned int core = POINTER_TO_UINT(p2); @@ -696,473 +328,4 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; } } - -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) -{ - char thread_name[] = "ll_user0"; - struct k_thread *thread; - struct k_event *event; - int ret; - - assert_can_be_cold(); - - /* - * Build the objects in locals and publish them to struct ipc_user only - * at the end. When a secondary core is brought up, user threads on - * other cores are already running and can overwrite struct ipc_user, - * so its fields must not be read back here. - */ - event = k_object_alloc(K_OBJ_EVENT); - if (!event) { - LOG_ERR("user IPC event alloc failed"); - return -ENOMEM; - } - k_event_init(event); - - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - LOG_ERR("user IPC thread alloc failed"); - ret = -ENOMEM; - goto e_event; - } - - k_thread_create(thread, ipc_user_stack[core], - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, - -1, K_USER, K_FOREVER); - - k_thread_cpu_pin(thread, core); - thread_name[sizeof(thread_name) - 2] = '0' + core; - k_thread_name_set(thread, thread_name); - - /* - * Each userspace IPC thread must be able to wait on its private event - * and signal completion on the primary core semaphore - */ - k_thread_access_grant(thread, ipc_user_ksem, event); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - user_ll_grant_access(thread, core); - pipeline_posn_grant_access(thread); - - ipc_user_kevent[core] = event; - ipc_user_kthread[core] = thread; - sof_sysuser_thread_set(SOF_SYSUSER_IPC, core, thread); - - /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ - ipc_user->event[core] = event; - ipc_user->thread[core] = thread; - - return 0; - -e_event: - k_object_free(event); - - return ret; -} - -/* - * Grant the threads of the newly initialized core and of the other already - * initialized cores access to each other's cross-core request slots and LL - * scheduler locks. Uses only the kernel-only copies of the thread state. - */ -__cold static void ipc_user_xcore_grant(unsigned int core) -{ - struct k_thread *new_threads[] = { - ipc_user_kthread[core], ipc_user_kaudio_thread[core], - }; - unsigned int c, i; - - assert_can_be_cold(); - - for (c = 0; c < CONFIG_CORE_COUNT; c++) { - struct k_thread *threads[] = { - ipc_user_kthread[c], ipc_user_kaudio_thread[c], - }; - - /* - * Grant the slot semaphores of all cores. Note: braces needed, - * k_thread_access_grant() expands to multiple statements. - */ - for (i = 0; i < ARRAY_SIZE(new_threads); i++) { - if (new_threads[i]) { - k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], - &ipc_user_xcore_done[c]); - } - } - - if (!ipc_user_kthread[c]) - continue; - - for (i = 0; i < ARRAY_SIZE(threads); i++) { - if (!threads[i]) - continue; - k_thread_access_grant(threads[i], ipc_user_kevent[core]); - user_ll_grant_access(threads[i], core); - } - - for (i = 0; i < ARRAY_SIZE(new_threads); i++) { - if (!new_threads[i]) - continue; - k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); - user_ll_grant_access(new_threads[i], c); - } - } -} - -__cold int ipc_user_init_secondary(unsigned int core) -{ - struct ipc_user *ipc_user = ipc_user_kptr; - struct k_thread *audio_thread; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - ret = ipc_user_init_thread(ipc_user, core); - - if (ret < 0) - return ret; - - assert_can_be_cold(); - - k_thread_start(ipc_user_kthread[core]); - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("user LL task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, core, 0); - - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - k_thread_access_grant(ipc_user_kthread[core], audio_thread); - ipc_user_kaudio_thread[core] = audio_thread; - ipc_user->audio_thread[core] = audio_thread; - ipc_user_xcore_grant(core); - - return 0; -} - -struct k_thread *ipc_thread_user(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return NULL; - - /* user threads can not read the kernel-only copy */ - if (k_is_user_context()) - return ipc_get()->ipc_user_pdata->thread[core]; - - return ipc_user_kthread[core]; -} - -struct ipc_user *ipc_user_get(void) -{ - return ipc_user_kptr; -} - -void ipc_user_wait_core_init(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return; - - if (ipc_user_kinit_needed[core]) { - /* wait for IPC thread (ipc_user_thread_fn()) */ - k_sem_take(ipc_user_ksem, K_FOREVER); - ipc_user_kinit_needed[core] = false; - } -} - -/* Primary core only */ -__cold static void ipc_user_init(void) -{ - struct ipc *ipc = ipc_get(); - struct k_thread *audio_thread; - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc_user), 0); - int ret, core; - - if (!ipc_user) { - LOG_ERR("user IPC pdata alloc failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ - memset(ipc_user, 0, sizeof(*ipc_user)); - - assert_can_be_cold(); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - if (core != PLATFORM_PRIMARY_CORE_ID) - ipc_user_kinit_needed[core] = true; - } - - ipc_user_ksem = k_object_alloc(K_OBJ_SEM); - if (!ipc_user_ksem) { - LOG_ERR("user IPC sem alloc failed"); - k_panic(); - } - ipc_user_kptr = ipc_user; - ipc_user->sem = ipc_user_ksem; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); - if (ret < 0) - LOG_WRN("ipc context partition add failed: %d", ret); - - /* - * Grant user-space read access to the .coldrodata section in IMR. The - * prepare path walks component code that may reference __cold_rodata - * data. - * - * .cold text is deliberately NOT added as a partition: Zephyr already - * maps it executable in the shared ring. - */ -#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM - extern char __coldrodata_start[]; - extern char _imr_end[]; - - if (&_imr_end[0] > &__coldrodata_start[0]) { - struct k_mem_partition cold_part = { - .start = (uintptr_t)__coldrodata_start, - .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, - CONFIG_MMU_PAGE_SIZE), - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", - cold_part.size, cold_part.start, ret); - } -#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ - - k_sem_init(ipc_user_ksem, 0, 1); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct ipc_user_xcore *xc = &ipc_user->xcore[core]; - - xc->lock = &ipc_user_xcore_lock[core]; - xc->done = &ipc_user_xcore_done[core]; - k_sem_init(xc->lock, 1, 1); - k_sem_init(xc->done, 0, 1); - } - - ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); - if (ret < 0) { - LOG_ERR("user IPC thread initialization failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), - ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - if (ret < 0) { - LOG_ERR("ipc user: mailbox access grant failed: %d", ret); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* Store references in ipc struct so kernel handler can forward commands */ - ipc->ipc_user_pdata = ipc_user; - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - /* Grant ipc_user thread permission on the audio thread object. - * Needed so user-space dai_common_new() can call - * k_thread_access_grant(audio_thread, dai_mutex) from user context. - */ - k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); - ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); - - k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - - /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc_user_ksem, K_FOREVER); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -static void ipc_user_init(void) -{ -} #endif /* CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc_init(struct sof *sof) -{ - struct k_heap *heap; - struct ipc *ipc; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "entry"); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - - ipc = ipc_get(); - memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc->ll_alloc), 0); - if (!ipc->ll_alloc) { - tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); - sof_panic(SOF_IPC_PANIC_IPC); - } - ipc->ll_alloc->heap = heap; - ipc->ll_alloc->vreg = NULL; -#else - heap = NULL; - - /* init ipc data */ - ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); - if (!ipc) { - tr_err(&ipc_tr, "Unable to allocate IPC data"); - return -ENOMEM; - } - sof->ipc = ipc; -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc->comp_data) { - tr_err(&ipc_tr, "Unable to allocate IPC component data"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); - - k_spinlock_init(&ipc->lock); - list_init(&ipc->msg_list); - list_init(&ipc->comp_list); - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - struct io_perf_data_item init_data = {IO_PERF_IPC_ID, - cpu_get_id(), - IO_PERF_INPUT_DIRECTION, - IO_PERF_POWERED_UP_ENABLED, - IO_PERF_D0IX_POWER_MODE, - 0, 0, 0 }; - io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); - init_data.direction = IO_PERF_OUTPUT_DIRECTION; - io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); -#endif - -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - -#ifdef __ZEPHYR__ - k_tid_t thread; - - k_work_queue_init(&ipc_send_wq); - k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, - K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); - - thread = k_work_queue_thread_get(&ipc_send_wq); - - k_thread_suspend(thread); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); -#endif - k_thread_name_set(thread, "ipc_send_wq"); - - k_thread_resume(thread); - - k_work_init_delayable(&ipc_send_work, ipc_work_handler); -#endif /* __ZEPHYR__ */ - - ipc_user_init(); - - return platform_ipc_init(ipc); -} - -/* Locking: call with ipc->lock held and interrupts disabled */ -void ipc_complete_cmd(struct ipc *ipc) -{ - /* - * We have up to three contexts, attempting to complete IPC processing: - * the original IPC EDF task, the IDC EDF task on a secondary core, or - * an LL pipeline thread, running either on the primary or one of - * secondary cores. All these three contexts execute asynchronously. It - * is important to only signal the host that the IPC processing has - * completed after *all* tasks have completed. Therefore only the last - * context should do that. We accomplish this by setting IPC_TASK_* bits - * in ipc->task_mask for each used IPC context and by clearing them when - * each of those contexts completes. Only when the mask is 0 we can - * signal the host. - */ - if (ipc->task_mask) - return; - - ipc_platform_complete_cmd(ipc); -} - -bool ipc_enter_gdb; - -__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) -{ - k_msleep(1); -} - -static void ipc_complete_task(void *data) -{ - struct ipc *ipc = data; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_INLINE; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); -#if CONFIG_GDBSTUB - if (ipc_enter_gdb) { - ipc_enter_gdb = false; - ipc_platform_wait_ack(ipc); - gdb_init(); - } -#endif -} - -static enum task_state ipc_do_cmd(void *data) -{ - struct ipc *ipc = data; - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); -#endif - - /* - * 32-bit writes are atomic and at the moment no IPC processing is - * taking place, so, no need for a lock. - */ - ipc->task_mask = IPC_TASK_INLINE; - - return ipc_platform_do_cmd(ipc); -} - -struct task_ops ipc_task_ops = { - .run = ipc_do_cmd, - .complete = ipc_complete_task, - .get_deadline = ipc_task_deadline, -}; diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md index 237a7c1ecb36..9dfd96825d90 100644 --- a/src/ipc/ipc4/README.md +++ b/src/ipc/ipc4/README.md @@ -1,9 +1,12 @@ # IPC4 Architecture > **Runs in:** Boundary / bridge — `handler-user.c` and `notification-user.c` run in the user -> thread that services modules. Their kernel counterparts, `handler-kernel.c` (the kernel -> dispatcher) and `notification.c`, are in `src/kernel/ipc/ipc4`, together with the +> thread that services modules. Their kernel counterparts are in `src/kernel/ipc/ipc4`: +> `handler-kernel.c` (the kernel dispatcher), `handler-user-kernel.c` (the global and module +> message routers), `handler-ll-user.c` and `handler-ll-kernel.c` (the routing for the LL +> scheduler in user and in kernel space) and `notification.c`, together with the > `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification syscalls. See +> [User/Kernel Partitioning](#userkernel-partitioning) below and > [User/kernel split](../../include/sof/userspace/README.md). This directory holds the handlers and topology parsing logic for Inter-Processor Communication Version 4. IPC4 introduces a significantly denser, compound-command structure heavily based around the concept of "pipelines" and dynamic "modules" rather than static DSP stream roles. @@ -40,6 +43,67 @@ graph TD end ``` +## User/Kernel Partitioning + +With the LL scheduler in user space (`CONFIG_SOF_USERSPACE_LL`), each IPC4 message is handled +in up to two places: + +- **Kernel IPC thread**: `handler-kernel.c` receives the message from the host mailbox and + replies. It handles the kernel-only messages itself and passes the others to the routers in + `handler-user-kernel.c`. The routers handle the messages that need kernel mode in all builds. + For the rest they call the `ipc4_route_*()` functions of `handler-ll-user.c`, which do any + privileged part of the message and forward it to the user-space IPC thread of the target + core with `ipc_user_forward_cmd()`. +- **User-space IPC thread**: `ipc_user_thread_dispatch()` in `handler-user.c` runs the handlers + in `handler-user.c` and `helper.c` for the forwarded message. + +Without `CONFIG_SOF_USERSPACE_LL`, `handler-ll-kernel.c` provides the `ipc4_route_*()` +functions instead and calls the same handlers directly, so all messages are handled in the +kernel IPC thread. The split still marks which code must be user-space safe. + +```mermaid +graph TD + Host[Host IPC] --> Disp["handler-kernel.c
ipc4_process_glb/module_message()"] + Disp -->|kernel only| KOnly["library load, D0ix/Dx"] + Disp --> Routers["handler-user-kernel.c
ipc4_user_process_glb/module_message()"] + Routers -->|kernel only| KRouted["CHAIN_DMA, IPC gateway, GDB,
base firmware LARGE_CONFIG"] + Routers --> Route{"ipc4_route_*()"} + Route -->|CONFIG_SOF_USERSPACE_LL| LLUser["handler-ll-user.c
target core, driver lookup,
module load, multicore sequencing"] + Route -->|otherwise| LLKernel["handler-ll-kernel.c
direct call"] + LLUser -->|ipc_user_forward_cmd| UThread["user-space IPC thread
ipc_user_thread_dispatch()"] + UThread --> Handlers["handler-user.c / helper.c handlers"] + LLKernel --> Handlers +``` + +With `CONFIG_SOF_USERSPACE_LL`, the messages are handled as follows: + +| Message | Kernel only | Kernel and user | User | +|---|---|---|---| +| `GLB_LOAD_LIBRARY`, `GLB_LOAD_LIBRARY_PREPARE` | ✓ library manager | | | +| `GLB_CHAIN_DMA` | ✓ | | | +| `GLB_IPCGATEWAY_CMD` | ✓ | | | +| `GLB_ENTER_GDB` | ✓ | | | +| `MOD_SET_D0IX`, `MOD_SET_DX` | ✓ power management | | | +| `MOD_LARGE_CONFIG_GET`/`SET`, base firmware (module_id 0) | ✓ reads the IMR manifest | | | +| `GLB_CREATE_PIPELINE` | | kernel: target core; user: pipeline creation | | +| `GLB_DELETE_PIPELINE` | | kernel: owning core; user: pipeline free | | +| `GLB_SET_PIPELINE_STATE` | | kernel: pipeline list check, multicore prepare/trigger sequencing; user: prepare and trigger | | +| `MOD_INIT_INSTANCE` | | kernel: driver lookup, loadable module setup; user: component creation | | +| `MOD_CONFIG_GET`/`SET` | | | ✓ | +| `MOD_LARGE_CONFIG_GET`/`SET`, modules | | | ✓ | +| `MOD_BIND`, `MOD_UNBIND` | | | ✓ | +| `MOD_DELETE_INSTANCE` | | | ✓ | + +For the messages in the user column, the kernel only selects the target core from the module +and forwards the message unchanged. DP modules and modules on another core are created by the +kernel in `ipc4_init_module_instance()`. With `CONFIG_SOF_USERSPACE_APPLICATION`, the IPCs of a +DP module are then passed on to its DP thread (`scheduler_dp_thread_ipc()`). + +The other global messages (boot config, ROM control, performance measurements, multiple module +load and unload, pipeline state get, context size, save and restore, internal messages and +notifications) and the module restore messages are not implemented and are rejected by the +routers. + ## Processing Flows ### Pipeline State Management (`ipc4_set_pipeline_state`) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 167639253152..58df68018242 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -44,8 +44,6 @@ #include #include -#include "../audio/copier/ipcgtw_copier.h" - /* Command format errors during fuzzing are reported for virtually all * commands, and the resulting flood of logging becomes a severe * performance penalty (i.e. we get a lot less fuzzing done per CPU @@ -57,11 +55,11 @@ #define ipc_cmd_err(...) tr_err(__VA_ARGS__) #endif -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC4 routers, which run in kernel mode, are in + * src/kernel/ipc/ipc4/handler-user-kernel.c + */ -/* Userspace message context, copied in/out by kernel IPC thread. */ -/* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg *msg_reply; +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); #if CONFIG_LIBRARY @@ -90,54 +88,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifdef CONFIG_SOF_USERSPACE_LL -/* - * Determine the target core for an IPC4 module message. - * Falls back to current core when no component is bound yet. - */ -static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) -{ - /* - * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, - * struct ipc4_module_delete_instance - */ - struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; - uint32_t module_id = config->primary.r.module_id; - - if (module_id) { - uint32_t instance_id = config->primary.r.instance_id; - struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); - - if (dev) - return dev->ipc_config.core; - } - - return cpu_get_id(); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); -} - -__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc4_pipeline_delete *pipe; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - pipe = (struct ipc4_pipeline_delete *)ipc4; - tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); - - return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) { int err, reset_err; @@ -450,9 +400,9 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } -static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, - struct ipc4_pipeline_set_state *state, - const uint32_t **ppl_id, unsigned int *ppl_count) +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) { if (!state->extension.r.multi_ppl) { if (ppl_count) @@ -648,364 +598,8 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint return IPC4_SUCCESS; } - -/* - * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on - * different cores. The user-space IPC thread can't use IDC to reach the - * other cores, so the kernel runs the prepare phase on all pipelines and - * then the trigger phase on all pipelines, forwarding each step to the - * user-space IPC thread of the core, owning the pipeline. - */ -__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, - const uint32_t *ppl_id, - unsigned int ppl_count) -{ - static const uint32_t phases[] = { - IDC_PPL_STATE_PHASE_PREPARE, - IDC_PPL_STATE_PHASE_TRIGGER, - }; - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct ipc4_pipeline_set_state state; - struct ipc_comp_dev *ppl_icd; - unsigned int i, j; - int ret; - - assert_can_be_cold(); - - for (j = 0; j < ARRAY_SIZE(phases); j++) { - for (i = 0; i < ppl_count; i++) { - ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], - IPC_COMP_ALL); - if (!ppl_icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - return IPC4_INVALID_RESOURCE_ID; - } - - /* Forward as a single-pipeline request */ - state.primary.dat = ipc4->primary.dat; - state.extension.dat = ipc4->extension.dat; - state.primary.r.ppl_id = ppl_id[i]; - state.extension.r.multi_ppl = 0; - - pdata->ppl_phase = phases[j]; - ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, - ppl_icd->core); - pdata->ppl_phase = 0; - if (ret != 0) - return ret; - } - } - - return IPC4_SUCCESS; -} #endif /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_COMP_CHAIN_DMA - struct ipc_comp_dev *cdma_comp; - struct ipc *ipc = ipc_get(); - const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; - int comp_id; - int ret; - - comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - - if (!cdma_comp) { - /* - * Nothing to do when the chainDMA is not allocated and asked to - * be freed - */ - if (!cdma->primary.r.allocate && !cdma->primary.r.enable) - return IPC4_SUCCESS; - - ret = ipc4_chain_manager_create(cdma); - if (ret < 0) - return IPC4_FAILURE; - - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - if (!cdma_comp) { - return IPC4_FAILURE; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) { - comp_free(cdma_comp->cd); - list_item_del(&cdma_comp->list); - sof_heap_free(sof_sys_user_heap_get(), cdma_comp); - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) - return IPC4_INVALID_CHAIN_STATE_TRANSITION; - - return IPC4_SUCCESS; -#else /* CONFIG_COMP_CHAIN_DMA */ - return IPC4_UNAVAILABLE; -#endif /* CONFIG_COMP_CHAIN_DMA */ -} - -__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_IPC4_GATEWAY - struct ipc *ipc = ipc_get(); - uint32_t reply_size = 0; - int err; - - err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, - &reply_size); - /* reply size is returned in header extension dword */ - msg_reply->extension = reply_size; - - if (reply_size > 0) { - msg_reply->tx_data = ipc->comp_data; - msg_reply->tx_size = reply_size; - } - - return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; -#else /* CONFIG_IPC4_GATEWAY */ - ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); - return IPC4_UNAVAILABLE; -#endif /* CONFIG_IPC4_GATEWAY */ -} - -static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) -{ -#if CONFIG_GDBSTUB - ipc_enter_gdb = true; - return IPC4_SUCCESS; -#else - return IPC4_UNAVAILABLE; -#endif -} - -int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ -#ifdef CONFIG_SOF_USERSPACE_LL - struct ipc *ipc = ipc_get(); -#endif - uint32_t type; - int ret; - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_GLB_BOOT_CONFIG: - case SOF_IPC4_GLB_ROM_CONTROL: - case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: - case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: - case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_CHAIN_DMA: - ret = ipc4_process_chain_dma(ipc4); - break; - - /* pipeline settings */ - case SOF_IPC4_GLB_CREATE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_create *create = - (const struct ipc4_pipeline_create *)ipc4; - - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - create->extension.r.core_id); - } -#else - ret = ipc4_new_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_DELETE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - del->primary.r.instance_id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); - } -#else - ret = ipc4_delete_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_SET_PIPELINE_STATE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - struct ipc4_pipeline_set_state state = { - .primary.dat = ipc4->primary.dat, - .extension.dat = ipc4->extension.dat, - }; - const uint32_t *ppl_id; - unsigned int ppl_count; - int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); - if (id < 0) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - /* - * Validate all pipelines before any state change and check, - * if they span multiple cores, needing per-core processing - */ - bool multicore = false; - - ret = IPC4_SUCCESS; - for (unsigned int i = 1; i < ppl_count; i++) { - struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - ppl_id[i], IPC_COMP_ALL); - - if (!icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - if (icd->core != ppl->core) - multicore = true; - } - - if (ret != IPC4_SUCCESS) - break; - - if (multicore) - ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); - else - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ppl->core); - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_set_pipeline_state(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - - case SOF_IPC4_GLB_GET_PIPELINE_STATE: - case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: - case SOF_IPC4_GLB_SAVE_PIPELINE: - case SOF_IPC4_GLB_RESTORE_PIPELINE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_INTERNAL_MESSAGE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - /* Notification (FW to SW driver) */ - case SOF_IPC4_GLB_NOTIFICATION: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_IPCGATEWAY_CMD: - ret = ipc4_process_ipcgtw_cmd(ipc4); - break; - - case SOF_IPC4_GLB_ENTER_GDB: - ret = ipc_glb_gdb_debug(ipc4); - break; - - default: - ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - } - - return ret; -} - -/* - * Ipc4 Module message <------> ipc3 module message - * init module <-------> create component - * bind modules <-------> connect components - * module set_large_config <-------> component cmd - * delete module <-------> free component - */ - -__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_init_instance *module_init = - (const struct ipc4_module_init_instance *)ipc4; - struct comp_dev *dev; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - - /* Pass IPC to target core */ - if (!cpu_is_me(module_init->extension.r.core_id)) - return ipc4_process_on_core(module_init->extension.r.core_id, false); - - dev = comp_new_ipc4(module_init); - if (!dev) { - ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - return IPC4_MOD_NOT_INITIALIZED; - } - - return 0; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_connect(ipc, bu); -} - -__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_disconnect(ipc, bu); -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @@ -1077,23 +671,8 @@ __cold int ipc4_process_module_config(struct ipc4_message_request *ipc4, return ret; } -#ifndef CONFIG_SOF_USERSPACE_LL -static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) -{ - uint32_t reply_ext; - int ret; - - ret = ipc4_process_module_config(ipc4, set, &reply_ext); - - if (!set) - msg_reply->extension = reply_ext; - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, - char *data_out, uint32_t data_size) +__cold void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size) { const char *hostbox; @@ -1122,13 +701,13 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, - const struct comp_driver *drv, - bool init_block, - bool final_block, - uint32_t *data_off_size, - char *data_out, - const char *data_in) +__cold int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in) { const struct sof_tl * const input_tl = (struct sof_tl *)data_in; int ret; @@ -1324,119 +903,7 @@ __cold static int ipc4_process_large_config_get(struct ipc4_message_request *ipc } #endif -__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_large_config_reply reply; - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - char *data = ipc_get()->comp_data; - const struct comp_driver *drv; - struct comp_dev *dev = NULL; - uint32_t data_offset; - int ret; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - /* get component dev for non-basefw since there is no - * component dev for basefw - */ - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, - config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.get_large_config) - return IPC4_INVALID_REQUEST; - - data_offset = config->extension.r.data_off_size; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - /* data_off_size is a 20-bit host-controlled field, so it can - * claim far more than the hostbox can physically hold. - */ - if (data_offset > MAILBOX_HOSTBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", - data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - /* For now only vendor_config case uses payload from hostbox */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - ret = ipc4_get_vendor_config_module_instance(dev, drv, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - data += sizeof(reply); -#endif - ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, - data, data_offset); - - ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data); - } - - /* set up ipc4 error code for reply data */ - if (ret < 0) - ret = IPC4_INVALID_RESOURCE_ID; - - /* Copy host config and overwrite */ - reply.extension.dat = config->extension.dat; - reply.extension.r.data_off_size = data_offset; - - /* The last block, no more data */ - if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) - reply.extension.r.final_block = 1; - - /* Indicate last block if error occurs */ - if (ret) - reply.extension.r.final_block = 1; - - /* no need to allocate memory for reply msg */ - if (ret) - return ret; - - /* A module must not report more reply data than the DSP mailbox can hold. */ - if (data_offset > MAILBOX_DSPBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - - msg_reply->extension = reply.extension.dat; - msg_reply->tx_size = data_offset; - msg_reply->tx_data = data; - return ret; -} - -__cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, +__cold int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, uint32_t module_id, uint32_t instance_id, @@ -1586,283 +1053,6 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) return ret; } -__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - struct comp_dev *dev = NULL; - const struct comp_driver *drv; - int ret; - - assert_can_be_cold(); - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.set_large_config) - return IPC4_INVALID_REQUEST; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id, - config->extension.r.init_block, - config->extension.r.final_block, - config->extension.r.data_off_size, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - struct ipc *ipc = ipc_get(); - const char *data = (const char *)ipc->comp_data + sizeof(*config); -#else - const char *data = (const char *)MAILBOX_HOSTBOX_BASE; -#endif - ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, config->extension.r.final_block, - config->extension.r.data_off_size, data); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - } - - return ret; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_delete_instance module; - struct ipc *ipc = ipc_get(); - uint32_t comp_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - - comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); - ret = ipc_comp_free(ipc, comp_id); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", - (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ - const struct ipc4_module_init_instance *mi; - uint32_t type; - int ret; - - assert_can_be_cold(); - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_MOD_INIT_INSTANCE: - mi = (const struct ipc4_module_init_instance *)ipc4; - - if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && - IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { -#ifdef CONFIG_SOF_USERSPACE_LL - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - */ - uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, - mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - - if (!drv || !drv->ops.create) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - - if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { - int err = ipc4_user_module_load(drv, mi); - - if (err < 0) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - } - - ipc_user_get()->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - mi->extension.r.core_id); -#endif /* CONFIG_SOF_USERSPACE_LL */ - } else { - /* - * DP module creation starts running in kernel mode and - * switches to userspace later via a scheduler_dp_thread_ipc() - * call in module_init(). - */ - ret = ipc4_init_module_instance(ipc4); - } - break; - case SOF_IPC4_MOD_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - } -#else - ret = ipc4_set_get_config_module_instance(ipc4, false); -#endif - break; - case SOF_IPC4_MOD_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_set_get_config_module_instance(ipc4, true); -#endif - break; - case SOF_IPC4_MOD_LARGE_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - msg_reply->tx_size = pdata->reply_tx_size; - msg_reply->tx_data = pdata->reply_tx_data; - } - } else { - /* Base firmware (module_id==0): keep in kernel — - * ipc4_get_comp_drv() accesses IMR manifest which - * has no user-space partition. - */ - ret = ipc4_get_large_config_module_instance(ipc4); - } - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_get_large_config_module_instance(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - case SOF_IPC4_MOD_LARGE_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - } else { - /* Base firmware: keep in kernel (IMR access) */ - ret = ipc4_set_large_config_module_instance(ipc4); - } - } -#else - ret = ipc4_set_large_config_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_BIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* - * bind and unbind can connect LL with DP. In that case it isn't - * immediately clear whether the handler should run in the DP - * thread context or in the LL IPC thread context. The LL IPC - * thread has access to DP modules, so we have to perform - * binding in the LL IPC thread context - */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_bind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_UNBIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* DP / LL: see comment above */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_unbind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_DELETE_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_delete_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: - case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: - ret = IPC4_UNAVAILABLE; - break; - default: - ret = IPC4_UNKNOWN_MESSAGE_TYPE; - break; - } - - return ret; -} - #ifdef CONFIG_SOF_USERSPACE_LL /* * IPC4 implementation of the user-thread dispatch hook. Runs in the diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 5aba6378c3ff..8f69c0d635e7 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -30,26 +30,14 @@ #include #include -/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ -#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_SOC_SERIES_INTEL_ADSP_CAVS) -#define RIMAGE_MANIFEST 1 -#endif - -#ifdef RIMAGE_MANIFEST -#include -#endif - #include #include -#include -#include #include #include #include #include #include #include -#include #include #include @@ -60,11 +48,9 @@ #include #include -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx comp_tr; +/* The IPC4 helpers that run in kernel mode are in src/kernel/ipc/ipc4/helper-kernel.c */ -static const struct comp_driver *ipc4_get_drv(const void *uuid); +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, uint32_t id) @@ -78,7 +64,7 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, } #if CONFIG_LIBRARY -static inline unsigned char *ipc4_get_comp_new_data(void) +unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); unsigned char *data = (unsigned char *)ipc->comp_data + @@ -86,13 +72,8 @@ static inline unsigned char *ipc4_get_comp_new_data(void) return data; } - -static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) -{ - return ipc4_get_drv(data); -} #else -__cold static inline unsigned char *ipc4_get_comp_new_data(void) +__cold unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); @@ -100,8 +81,8 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif /* CONFIG_LIBRARY */ -__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, - const struct ipc4_module_init_instance *module_init) +__cold int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, module_init->primary.r.instance_id); @@ -180,112 +161,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, return 0; } -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) -{ - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - unsigned char *data; - uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - - assert_can_be_cold(); - - int ret = ipc4_comp_new_config(&ipc_config, module_init); - - if (ret < 0) - return NULL; - -#ifdef CONFIG_DCACHE_LINE_SIZE - if (!IS_ENABLED(CONFIG_LIBRARY)) - sys_cache_data_invd_range((__sparse_force void __sparse_cache *) - MAILBOX_HOSTBOX_BASE, - ALIGN_UP(ipc_config.ipc_config_size, - CONFIG_DCACHE_LINE_SIZE)); -#endif - data = ipc4_get_comp_new_data(); -#if CONFIG_LIBRARY - ipc_config.ipc_config_size -= sizeof(struct sof_uuid); - drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); -#else - drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); -#endif - if (!drv) - return NULL; - - if (!drv->ops.create) { - tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, - (uint32_t)IPC4_MOD_ID(comp_id)); - return NULL; - } - - if (drv->type == SOF_COMP_MODULE_ADAPTER) { - const struct ipc_config_process spec = { - .data = data, - .size = ipc_config.ipc_config_size, - }; - - dev = drv->ops.create(drv, &ipc_config, &spec); - } else { - dev = drv->ops.create(drv, &ipc_config, data); - } - if (!dev) - return NULL; - - list_init(&dev->bsource_list); - list_init(&dev->bsink_list); - -#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - /* init global performance measurement */ - dev->perf_data.perf_data_item = perf_data_getnext(); - /* this can be null, just no performance measurements in this case */ - if (dev->perf_data.perf_data_item) { - dev->perf_data.perf_data_item->item.resource_id = comp_id; - if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) - comp_init_performance_data(dev); - } -#endif - - ipc4_add_comp_dev(dev); - - comp_update_ibs_obs_cpc(dev); - - return dev; -} - #ifdef CONFIG_SOF_USERSPACE_LL - -int ipc4_user_module_load(const struct comp_driver *drv, - const struct ipc4_module_init_instance *mi) -{ - /* - * A large part of module instantiation is executed in the userspace - * mode, including calling the module's .init() method. But another part - * of it has to be executed in the kernel mode, including calling - * lib_manager_mod_create_priv(). This is the privileged part of module - * instantiation. - */ - struct comp_ipc_config ipc_config; - int ret = ipc4_comp_new_config(&ipc_config, mi); - - if (ret < 0) - return ret; - - const struct ipc_config_process spec = { - .data = ipc4_get_comp_new_data(), - .size = ipc_config.ipc_config_size, - }; - -#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY - sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); -#endif - - struct userspace_context *userspace = NULL; - - return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); -} - /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * @@ -578,16 +454,6 @@ __cold int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) return ipc4_create_pipeline(pipe_desc, valid_pparams ? &pparams : NULL); } -__cold static inline int ipc_comp_free_remote(struct comp_dev *dev) -{ - struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), - dev->ipc_config.core,}; - - assert_can_be_cold(); - - return idc_send_msg(&msg, IDC_BLOCKING); -} - __cold static int ipc_pipeline_module_free(uint32_t pipeline_id, uint32_t ppl_core) { struct ipc *ipc = ipc_get(); @@ -1263,76 +1129,6 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u return IPC4_SUCCESS; } -#if CONFIG_COMP_CHAIN_DMA -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) -{ - const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); - const struct comp_driver *drv; - struct comp_dev *dev; - - assert_can_be_cold(); - - drv = ipc4_get_drv(&uuid); - if (!drv) - return -EINVAL; - - dev = drv->ops.create(drv, NULL, cdma); - if (!dev) - return -EINVAL; - - /* differentiate instance by unique ids assignment */ - const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id - + IPC4_MAX_MODULE_COUNT, 0); - dev->ipc_config.id = comp_id; - dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; - - return ipc4_add_comp_dev(dev); -} - -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) -{ - const bool allocate = cdma->primary.r.allocate; - const bool enable = cdma->primary.r.enable; - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist, *_tmp; - int ret; - - assert_can_be_cold(); - - if (!dev) - return -EINVAL; - - if (allocate) { - if (enable) - ret = comp_trigger(dev, COMP_TRIGGER_START); - else - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - } else { - if (enable) - return -EINVAL; - - /* remove chain part */ - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - if (ret < 0) - return ret; - - list_for_item_safe(clist, _tmp, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->cd != dev) - continue; - list_item_del(&icd->list); - sof_heap_free(sof_sys_user_heap_get(), icd); - break; - } - comp_free(dev); - } - return ret; -} -#endif /* CONFIG_COMP_CHAIN_DMA */ - __cold static int ipc4_update_comps_direction(struct ipc *ipc, uint32_t ppl_id) { struct ipc_comp_dev *icd; @@ -1395,200 +1191,6 @@ int ipc4_pipeline_complete(struct ipc *ipc, uint32_t comp_id, uint32_t cmd) return ipc_pipeline_complete(ipc, comp_id); } -int ipc4_process_on_core(uint32_t core, bool blocking) -{ - int ret; - - ret = ipc_process_on_core(core, blocking); - switch (ret) { - case 0: - case 1: - return IPC4_SUCCESS; - case -EACCES: - return IPC4_INVALID_CORE_ID; - case -ETIME: - case -EBUSY: - return IPC4_BUSY; - default: - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; -} - -__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) -{ - struct comp_driver_list *drivers = comp_drivers_get(); - struct list_item *clist; - const struct comp_driver *drv = NULL; - struct comp_driver_info *info; - - assert_can_be_cold(); - - /* search driver list with UUID */ - list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, list); - if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, "found type %d, uuid %pU", - info->drv->type, SOF_DRV_UID_NAME(info->drv)); - drv = info->drv; - break; - } - } - - return drv; -} - -__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) -{ - const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; - const struct comp_driver *drv; - - assert_can_be_cold(); - - drv = ipc4_search_for_drv(uuid); - if (!drv) - tr_err(&comp_tr, - "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", - sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], - sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], - sof_uuid->d[6], sof_uuid->d[7]); - - return drv; -} - -#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) -/* - * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. - * - * native_sim fuzz builds have no rimage manifest, so the manifest lookup in - * ipc4_get_comp_drv() can never resolve a driver and every module instance - * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, - * DELETE_INSTANCE) would be unreachable from the fuzzer. - * - * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a - * 1-based index into the runtime component driver list, counting only - * instantiable module-adapter drivers so the fuzzer's module_id space matches - * what a real signed manifest would expose. - */ -static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) -{ - static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); - struct comp_driver_list *dlist = comp_drivers_get(); - struct list_item *iter; - uint32_t idx = 0; - - if (!module_id) - return ipc4_search_for_drv(&basefw_reg_uuid); - - list_for_item(iter, &dlist->list) { - struct comp_driver_info *inf = - container_of(iter, struct comp_driver_info, list); - - /* - * Skip query-only entries (e.g. BaseFW) that cannot be - * instantiated because they have no create op. - */ - if (!inf->drv->ops.create) - continue; - - /* - * A real signed manifest only lists module-adapter modules. - * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) - * are registered for IPC3 but are never IPC4 modules: on the - * IPC4 path they receive no params() pass and cannot be - * configured (e.g. their DMA buffer is never allocated). Skip - * non-module-adapter drivers so the fuzzer's module_id space - * matches a real manifest and cannot map to an unconfigurable - * component. - */ - if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) - continue; - - if (++idx == module_id) - return inf->drv; - } - - tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", - module_id, idx); - return NULL; -} -#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ - -/* - * Called from - * - ipc4_get_large_config_module_instance() - * - ipc4_set_large_config_module_instance() - * - comp_new_ipc4() - * all of which are __cold - */ -__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) -{ - const struct sof_man_fw_desc *desc = NULL; - const struct sof_man_module *mod; - uint32_t entry_index; - - assert_can_be_cold(); - -#ifdef RIMAGE_MANIFEST - desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; -#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) - return ipc4_get_fuzzer_drv(module_id); -#else - /* Non-rimage platforms have no component facility yet. - * This needs to move to the platform layer. - */ - return NULL; -#endif - - uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); - - if (lib_idx == 0) { - /* module_id 0 is used for base fw which is in entry 1 or 2 */ - if (!module_id) - entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); - else - entry_index = module_id; - - if (entry_index >= desc->header.num_module_entries) { - tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); - return NULL; - } - - mod = (const struct sof_man_module *)((const char *)desc + - SOF_MAN_MODULE_OFFSET(entry_index)); - } else { - /* Library index greater than 0 possible only when LIBRARY_MANAGER - * support enabled. - */ -#if CONFIG_LIBRARY_MANAGER - mod = lib_manager_get_module_manifest(module_id); - - if (!mod) { - tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", - module_id); - return NULL; - } -#else - tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", - lib_idx); - return NULL; -#endif - } - -#if CONFIG_LIBRARY_MANAGER - /* Check already registered components */ - const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); - - if (drv) - return drv; - - /* New module not registered yet. */ - lib_manager_register_module(module_id); -#endif - return ipc4_get_drv(&mod->uuid); -} - struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id) { struct ipc_comp_dev *icd = ipc_get_comp_by_id(ipc_get(), comp_id); diff --git a/src/kernel/ipc/CMakeLists.txt b/src/kernel/ipc/CMakeLists.txt index 0f9d63c0aee6..a3e7c61638c6 100644 --- a/src/kernel/ipc/CMakeLists.txt +++ b/src/kernel/ipc/CMakeLists.txt @@ -7,7 +7,7 @@ elseif (CONFIG_IPC_MAJOR_4) endif() # Files common to Zephyr and the host library -add_local_sources(sof notification_pool.c) +add_local_sources(sof ipc-common-kernel.c notification_pool.c) is_zephyr(zephyr) if(zephyr) diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c new file mode 100644 index 000000000000..82cfb9d6049c --- /dev/null +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -0,0 +1,895 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Keyon Jie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if CONFIG_IPC_MAJOR_4 +#include +#endif +#include +#include +#include +#include + +#ifdef __ZEPHYR__ +#include +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif + +#include +#include + +/* Kernel part of the common IPC code; the part that also runs in user mode is in + * src/ipc/ipc-common.c + */ + +LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ipc); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); + +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +#endif + +int ipc_process_on_core(uint32_t core, bool blocking) +{ + struct ipc *ipc = ipc_get(); + struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; + int ret; + + /* check if requested core is enabled */ + if (!cpu_is_core_enabled(core)) { + tr_err(&ipc_tr, "core #%d is disabled", core); + return -EACCES; + } + +#if CONFIG_IPC_MAJOR_3 + /* The other core will write there its response */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); +#endif + + /* + * If the primary core is waiting for secondary cores to complete, it + * will also reply to the host + */ + if (!blocking) { + k_spinlock_key_t key; + + ipc->core = core; + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_SECONDARY_CORE; + k_spin_unlock(&ipc->lock, key); + } + + /* send IDC message */ + ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); + if (ret < 0) + return ret; + + /* reply written by other core */ + return 1; +} + +void ipc_send_queued_msg(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_msg *msg; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + + if (ipc->pm_prepare_D3) + goto out; + + /* any messages to send ? */ + if (list_is_empty(&ipc->msg_list)) + goto out; + + msg = list_first_item(&ipc->msg_list, struct ipc_msg, + list); + + if (ipc_platform_send_msg(msg) == 0) { + /* Remove the message from the list if it has been successfully sent. */ + list_item_del(&msg->list); + /* Invoke a callback to notify that the message has been sent. */ + if (msg->callback) + msg->callback(msg); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); +#endif + } +out: + k_spin_unlock(&ipc->lock, key); +} + +#ifdef __ZEPHYR__ +static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); + +/* + * Kernel-only: these must not live in struct ipc, which user threads can + * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of + * a supervisor thread and the work item holds the handler it calls. + */ +static struct k_work_q ipc_send_wq; +static struct k_work_delayable ipc_send_work; + +struct k_work_q *ipc_send_wq_get(void) +{ + return &ipc_send_wq; +} +#endif + +static void schedule_ipc_worker(void) +{ + /* + * note: in XTOS builds, this is handled in + * task_main_primary_core() + */ +#ifdef __ZEPHYR__ + k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); +#endif +} + +__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + assert_can_be_cold(); + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + ipc_platform_send_msg_direct(msg); + + k_spin_unlock(&ipc->lock, key); +} + +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + /* + * note: This function can be executed in LL or EDF context, from any core. + * In Zephyr builds, there is IPC queue that is always handled by the primary core, + * whereas submitting to the queue is allowed from any core. Therefore disable option + * of sending IPC immediately by any context/core to secure IPC registers/mailbox + * access. + */ +#ifndef __ZEPHYR__ + /* try to send critical notifications right away */ + if (high_priority) { + ret = ipc_platform_send_msg(msg); + if (!ret) { + k_spin_unlock(&ipc->lock, key); + return; + } + } +#endif + /* add to queue unless already there */ + if (list_is_empty(&msg->list)) { + if (high_priority) + list_item_prepend(&msg->list, &ipc->msg_list); + else + list_item_append(&msg->list, &ipc->msg_list); + } + + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +EXPORT_SYMBOL(ipc_msg_send); + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + /* No caller check: DP modules send notifications, see mod_ipc_msg_init() */ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif /* CONFIG_USERSPACE */ + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + /* No caller check: called by mod_ipc_msg_free() in DP module threads */ + struct ipc *ipc = ipc_get(); + struct list_item *mlist, *_mlist; + bool found = false; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + /* + * special case: empty list was passed. we can't trust where + * list->prev points to, so do not pass to + * z_impl_ipc_msg_list_remove(), but handle here + */ + if (list_is_empty(&msg->list)) + return; + + list_for_item_safe(mlist, _mlist, &ipc->msg_list) { + if (mlist == &msg->list) { + found = true; + break; + } + } + K_OOPS(K_SYSCALL_VERIFY(found)); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif /* CONFIG_USERSPACE */ + +#ifdef __ZEPHYR__ +static void ipc_work_handler(struct k_work *work) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + ipc_send_queued_msg(); + + key = k_spin_lock(&ipc->lock); + + if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +#endif + +void ipc_schedule_process(struct ipc *ipc) +{ +#if CONFIG_TWB_IPC_TASK + schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); +#else + schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Generic user-space IPC handling thread infrastructure. Protocol-specific + * command interpretation lives in ipc_user_thread_dispatch(), implemented by + * the active IPC major (see src/ipc/ipc4/). + */ + +BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, + "cross-core request payload smaller than IDC payload"); +#if CONFIG_IPC_MAJOR_4 +BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == + sizeof(struct ipc4_module_bind_unbind), + "cross-core request bind data size mismatch"); +#endif + +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/* + * Kernel-only copies of the IPC user thread state. struct ipc_user and the + * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel + * code must use these instead of reading kernel object pointers from there. + */ +static struct ipc_user *ipc_user_kptr; +static struct k_sem *ipc_user_ksem; +static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; +static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; + +/* Cross-core request slot semaphores, static so they're never freed */ +static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; +static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; + +/** + * @brief Forward an IPC command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the message words + * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in + * task_mask so the host is not signaled until the user thread completes. + * Blocks until the user thread finishes processing and returns the result. + * + * @param primary Primary message word + * @param extension Extension message word + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc_user_kptr; + k_spinlock_key_t key; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + LOG_DBG("IPC: forward cmd %08x", primary); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = primary; + pdata->ipc_msg_ext = extension; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* + * Wake the user thread. Post, not set: a cross-core request + * (IPC_USER_EVENT_XCORE) may be pending on the same event. + */ + k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); + if (ret) { + LOG_ERR("IPC user: sem error %d\n", ret); + /* fall through to complete the cmd */ + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return !ret ? pdata->result : ret; +} + +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) +{ + char thread_name[] = "ll_user0"; + struct k_thread *thread; + struct k_event *event; + int ret; + + assert_can_be_cold(); + + /* + * Build the objects in locals and publish them to struct ipc_user only + * at the end. When a secondary core is brought up, user threads on + * other cores are already running and can overwrite struct ipc_user, + * so its fields must not be read back here. + */ + event = k_object_alloc(K_OBJ_EVENT); + if (!event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(event); + + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(thread, ipc_user_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(thread, core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(thread, ipc_user_ksem, event); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + user_ll_grant_access(thread, core); + pipeline_posn_grant_access(thread); + + ipc_user_kevent[core] = event; + ipc_user_kthread[core] = thread; + sof_sysuser_thread_set(SOF_SYSUSER_IPC, core, thread); + + /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ + ipc_user->event[core] = event; + ipc_user->thread[core] = thread; + + return 0; + +e_event: + k_object_free(event); + + return ret; +} + +/* + * Grant the threads of the newly initialized core and of the other already + * initialized cores access to each other's cross-core request slots and LL + * scheduler locks. Uses only the kernel-only copies of the thread state. + */ +__cold static void ipc_user_xcore_grant(unsigned int core) +{ + struct k_thread *new_threads[] = { + ipc_user_kthread[core], ipc_user_kaudio_thread[core], + }; + unsigned int c, i; + + assert_can_be_cold(); + + for (c = 0; c < CONFIG_CORE_COUNT; c++) { + struct k_thread *threads[] = { + ipc_user_kthread[c], ipc_user_kaudio_thread[c], + }; + + /* + * Grant the slot semaphores of all cores. Note: braces needed, + * k_thread_access_grant() expands to multiple statements. + */ + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (new_threads[i]) { + k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], + &ipc_user_xcore_done[c]); + } + } + + if (!ipc_user_kthread[c]) + continue; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + if (!threads[i]) + continue; + k_thread_access_grant(threads[i], ipc_user_kevent[core]); + user_ll_grant_access(threads[i], core); + } + + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (!new_threads[i]) + continue; + k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); + user_ll_grant_access(new_threads[i], c); + } + } +} + +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc_user *ipc_user = ipc_user_kptr; + struct k_thread *audio_thread; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user_kthread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user_kaudio_thread[core] = audio_thread; + ipc_user->audio_thread[core] = audio_thread; + ipc_user_xcore_grant(core); + + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return NULL; + + /* user threads can not read the kernel-only copy */ + if (k_is_user_context()) + return ipc_get()->ipc_user_pdata->thread[core]; + + return ipc_user_kthread[core]; +} + +struct ipc_user *ipc_user_get(void) +{ + return ipc_user_kptr; +} + +void ipc_user_wait_core_init(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return; + + if (ipc_user_kinit_needed[core]) { + /* wait for IPC thread (ipc_user_thread_fn()) */ + k_sem_take(ipc_user_ksem, K_FOREVER); + ipc_user_kinit_needed[core] = false; + } +} + +/* Primary core only */ +__cold static void ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct k_thread *audio_thread; + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); + int ret, core; + + if (!ipc_user) { + LOG_ERR("user IPC pdata alloc failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + memset(ipc_user, 0, sizeof(*ipc_user)); + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + if (core != PLATFORM_PRIMARY_CORE_ID) + ipc_user_kinit_needed[core] = true; + } + + ipc_user_ksem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user_ksem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + ipc_user_kptr = ipc_user; + ipc_user->sem = ipc_user_ksem; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + if (ret < 0) + LOG_WRN("ipc context partition add failed: %d", ret); + + /* + * Grant user-space read access to the .coldrodata section in IMR. The + * prepare path walks component code that may reference __cold_rodata + * data. + * + * .cold text is deliberately NOT added as a partition: Zephyr already + * maps it executable in the shared ring. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ + + k_sem_init(ipc_user_ksem, 0, 1); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + + xc->lock = &ipc_user_xcore_lock[core]; + xc->done = &ipc_user_xcore_done[core]; + k_sem_init(xc->lock, 1, 1); + k_sem_init(xc->done, 0, 1); + } + + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + if (ret < 0) { + LOG_ERR("ipc user: mailbox access grant failed: %d", ret); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); + + k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc_user_ksem, K_FOREVER); +} +#else /* CONFIG_SOF_USERSPACE_LL */ +static void ipc_user_init(void) +{ +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_init(struct sof *sof) +{ + struct k_heap *heap; + struct ipc *ipc; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "entry"); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + sof_panic(SOF_IPC_PANIC_IPC); + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + + /* init ipc data */ + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { + tr_err(&ipc_tr, "Unable to allocate IPC data"); + return -ENOMEM; + } + sof->ipc = ipc; +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { + tr_err(&ipc_tr, "Unable to allocate IPC component data"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); + + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + struct io_perf_data_item init_data = {IO_PERF_IPC_ID, + cpu_get_id(), + IO_PERF_INPUT_DIRECTION, + IO_PERF_POWERED_UP_ENABLED, + IO_PERF_D0IX_POWER_MODE, + 0, 0, 0 }; + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); + init_data.direction = IO_PERF_OUTPUT_DIRECTION; + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); +#endif + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + +#ifdef __ZEPHYR__ + k_tid_t thread; + + k_work_queue_init(&ipc_send_wq); + k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, + K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + + thread = k_work_queue_thread_get(&ipc_send_wq); + + k_thread_suspend(thread); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); +#endif + k_thread_name_set(thread, "ipc_send_wq"); + + k_thread_resume(thread); + + k_work_init_delayable(&ipc_send_work, ipc_work_handler); +#endif /* __ZEPHYR__ */ + + ipc_user_init(); + + return platform_ipc_init(ipc); +} + +/* Locking: call with ipc->lock held and interrupts disabled */ +void ipc_complete_cmd(struct ipc *ipc) +{ + /* + * We have up to three contexts, attempting to complete IPC processing: + * the original IPC EDF task, the IDC EDF task on a secondary core, or + * an LL pipeline thread, running either on the primary or one of + * secondary cores. All these three contexts execute asynchronously. It + * is important to only signal the host that the IPC processing has + * completed after *all* tasks have completed. Therefore only the last + * context should do that. We accomplish this by setting IPC_TASK_* bits + * in ipc->task_mask for each used IPC context and by clearing them when + * each of those contexts completes. Only when the mask is 0 we can + * signal the host. + */ + if (ipc->task_mask) + return; + + ipc_platform_complete_cmd(ipc); +} + +bool ipc_enter_gdb; + +__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) +{ + k_msleep(1); +} + +static void ipc_complete_task(void *data) +{ + struct ipc *ipc = data; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_INLINE; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); +#if CONFIG_GDBSTUB + if (ipc_enter_gdb) { + ipc_enter_gdb = false; + ipc_platform_wait_ack(ipc); + gdb_init(); + } +#endif +} + +static enum task_state ipc_do_cmd(void *data) +{ + struct ipc *ipc = data; + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); +#endif + + /* + * 32-bit writes are atomic and at the moment no IPC processing is + * taking place, so, no need for a lock. + */ + ipc->task_mask = IPC_TASK_INLINE; + + return ipc_platform_do_cmd(ipc); +} + +struct task_ops ipc_task_ops = { + .run = ipc_do_cmd, + .complete = ipc_complete_task, + .get_deadline = ipc_task_deadline, +}; diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 53a37d553675..71d856d78322 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -3,6 +3,16 @@ # Files common to Zephyr and the host library add_local_sources(sof handler-kernel.c + handler-user-kernel.c + helper-kernel.c logging.c notification.c ) + +# The IPC4 message routing for the LL scheduler in user space or in kernel +# space, see handler-user-kernel.c +if(CONFIG_SOF_USERSPACE_LL) + add_local_sources(sof handler-ll-user.c) +else() + add_local_sources(sof handler-ll-kernel.c) +endif() diff --git a/src/kernel/ipc/ipc4/handler-ll-kernel.c b/src/kernel/ipc/ipc4/handler-ll-kernel.c new file mode 100644 index 000000000000..6571cf54288a --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-ll-kernel.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +#include "handler-route.h" + +/* + * IPC4 message routing with the LL scheduler in kernel space (without + * CONFIG_SOF_USERSPACE_LL). The routers in handler-user-kernel.c run in the + * kernel IPC thread and call these functions, which run the handlers in + * src/ipc/ipc4/handler-user.c and src/ipc/ipc4/helper.c directly in that + * thread. With the LL scheduler in user space, handler-ll-user.c forwards + * the messages to the user-space IPC thread instead. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +__cold int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); +} + + +__cold int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc4_pipeline_delete *pipe; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + pipe = (struct ipc4_pipeline_delete *)ipc4; + tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); + + return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); +} + + +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4) +{ + return ipc4_set_pipeline_state(ipc4); +} + +__cold int ipc4_route_init_module(struct ipc4_message_request *ipc4) +{ + return ipc4_init_module_instance(ipc4); +} + +int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply) +{ + uint32_t reply_ext; + int ret; + + ret = ipc4_process_module_config(ipc4, set, &reply_ext); + + if (!set) + reply->extension = reply_ext; + + return ret; +} + + +__cold int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply) +{ + return ipc4_get_large_config_module_instance(ipc4); +} + +__cold int ipc4_route_large_config_set(struct ipc4_message_request *ipc4) +{ + return ipc4_set_large_config_module_instance(ipc4); +} + +__cold int ipc4_route_bind(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_connect(ipc, bu); +} + + +__cold int ipc4_route_unbind(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_disconnect(ipc, bu); +} + + +__cold int ipc4_route_delete_module(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_delete_instance module; + struct ipc *ipc = ipc_get(); + uint32_t comp_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + + comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); + ret = ipc_comp_free(ipc, comp_id); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", + (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + + return ret; +} + diff --git a/src/kernel/ipc/ipc4/handler-ll-user.c b/src/kernel/ipc/ipc4/handler-ll-user.c new file mode 100644 index 000000000000..e69f88485587 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-ll-user.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +#include "handler-route.h" + +/* + * IPC4 message routing with the LL scheduler in user space + * (CONFIG_SOF_USERSPACE_LL). The routers in handler-user-kernel.c run in the + * kernel IPC thread and call these functions for the messages that create, + * configure, connect and delete pipelines and modules. They do what needs + * kernel mode (target core lookup, multicore sequencing, driver lookup and + * module loading) and forward the message to the user-space IPC thread of the + * target core, which runs the handlers in src/ipc/ipc4/handler-user.c. With + * the LL scheduler in kernel space, handler-ll-kernel.c calls those handlers + * directly instead. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} + +/* + * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on + * different cores. The user-space IPC thread can't use IDC to reach the + * other cores, so the kernel runs the prepare phase on all pipelines and + * then the trigger phase on all pipelines, forwarding each step to the + * user-space IPC thread of the core, owning the pipeline. + */ +__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, + const uint32_t *ppl_id, + unsigned int ppl_count) +{ + static const uint32_t phases[] = { + IDC_PPL_STATE_PHASE_PREPARE, + IDC_PPL_STATE_PHASE_TRIGGER, + }; + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + struct ipc4_pipeline_set_state state; + struct ipc_comp_dev *ppl_icd; + unsigned int i, j; + int ret; + + assert_can_be_cold(); + + for (j = 0; j < ARRAY_SIZE(phases); j++) { + for (i = 0; i < ppl_count; i++) { + ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], + IPC_COMP_ALL); + if (!ppl_icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + /* Forward as a single-pipeline request */ + state.primary.dat = ipc4->primary.dat; + state.extension.dat = ipc4->extension.dat; + state.primary.r.ppl_id = ppl_id[i]; + state.extension.r.multi_ppl = 0; + + pdata->ppl_phase = phases[j]; + ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, + ppl_icd->core); + pdata->ppl_phase = 0; + if (ret != 0) + return ret; + } + } + + return IPC4_SUCCESS; +} + +int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4) +{ + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); +} + +int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) + return IPC4_INVALID_RESOURCE_ID; + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); +} + +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + const uint32_t *ppl_id; + unsigned int ppl_count; + int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + + if (id < 0) + return IPC4_INVALID_RESOURCE_ID; + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) + return IPC4_INVALID_RESOURCE_ID; + + /* + * Validate all pipelines before any state change and check, + * if they span multiple cores, needing per-core processing + */ + bool multicore = false; + + for (unsigned int i = 1; i < ppl_count; i++) { + struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + ppl_id[i], IPC_COMP_ALL); + + if (!icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + if (icd->core != ppl->core) + multicore = true; + } + + if (multicore) + return ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); +} + +__cold int ipc4_route_init_module(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *mi = + (const struct ipc4_module_init_instance *)ipc4; + + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain) { + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + + if (!drv || !drv->ops.create) + return IPC4_MOD_NOT_INITIALIZED; + + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); + + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) + return IPC4_MOD_NOT_INITIALIZED; + } + + ipc_user_get()->init_drv = drv; + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); + } + + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + return ipc4_init_module_instance(ipc4); +} + +__cold int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply) +{ + /* Forward to user thread for privilege-separated execution */ + int ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + + if (!set && !ret) { + struct ipc_user *pdata = ipc_user_get(); + + reply->extension = pdata->reply_ext; + } + + return ret; +} + +__cold int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + int ret; + + /* Base firmware (module_id==0): keep in kernel — + * ipc4_get_comp_drv() accesses IMR manifest which + * has no user-space partition. + */ + if (!config->primary.r.module_id) + return ipc4_get_large_config_module_instance(ipc4); + + /* Module case: forward to user thread */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + reply->extension = pdata->reply_ext; + reply->tx_size = pdata->reply_tx_size; + reply->tx_data = pdata->reply_tx_data; + } + + return ret; +} + +__cold int ipc4_route_large_config_set(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + /* Base firmware: keep in kernel (IMR access) */ + if (!config->primary.r.module_id) + return ipc4_set_large_config_module_instance(ipc4); + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +/* + * bind and unbind can connect LL with DP. In that case it isn't + * immediately clear whether the handler should run in the DP + * thread context or in the LL IPC thread context. The LL IPC + * thread has access to DP modules, so we have to perform + * binding in the LL IPC thread context + */ +__cold int ipc4_route_bind(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +/* DP / LL: see ipc4_route_bind() */ +__cold int ipc4_route_unbind(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +__cold int ipc4_route_delete_module(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} diff --git a/src/kernel/ipc/ipc4/handler-route.h b/src/kernel/ipc/ipc4/handler-route.h new file mode 100644 index 000000000000..75d80d0cea4c --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-route.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* + * Private to the kernel IPC4 message routing in src/kernel/ipc/ipc4: + * handler-user-kernel.c, the routers common to all builds, calls the + * ipc4_route_*() functions for the messages that are handled differently + * with the LL scheduler in user space (handler-ll-user.c, built with + * CONFIG_SOF_USERSPACE_LL) and in kernel space (handler-ll-kernel.c). + */ + +#ifndef __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ +#define __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ + +#include + +struct ipc4_message_request; +struct ipc_msg; + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +/* Handlers in handler-user-kernel.c, used by both routing variants */ +int ipc4_init_module_instance(struct ipc4_message_request *ipc4); +int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4); +int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4); + +/* Global messages */ +int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4); +int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4); +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4); + +/* Module messages */ +int ipc4_route_init_module(struct ipc4_message_request *ipc4); +int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply); +int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +int ipc4_route_large_config_set(struct ipc4_message_request *ipc4); +int ipc4_route_bind(struct ipc4_message_request *ipc4); +int ipc4_route_unbind(struct ipc4_message_request *ipc4); +int ipc4_route_delete_module(struct ipc4_message_request *ipc4); + +#endif /* __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ */ diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c new file mode 100644 index 000000000000..e99808955540 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-user-kernel.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +#include "handler-route.h" + +/* + * Kernel part of the IPC4 global and module message handling. + * + * handler-kernel.c receives the host IPC in the kernel IPC thread and passes + * the global messages to ipc4_user_process_glb_message() and the module + * messages to ipc4_user_process_module_message() here. These routers, also + * running in the kernel IPC thread, are the boundary between kernel and user + * space for IPC4: + * - some messages are handled here, in kernel mode, in all builds: CHAIN_DMA, + * the IPC gateway, GDB, and LARGE_CONFIG get and set of the base firmware + * (module_id 0), which reads the IMR manifest; + * - the messages that create, configure, connect and delete pipelines and + * modules are passed to the ipc4_route_*() functions. With the LL scheduler + * in user space (CONFIG_SOF_USERSPACE_LL) these are in handler-ll-user.c and + * forward the message to the user-space IPC thread, which runs the handlers + * in src/ipc/ipc4/handler-user.c. Otherwise they are in handler-ll-kernel.c + * and run those handlers directly in the kernel IPC thread. + * The "user" in the name of this file and of the routers refers to the + * messages, which are handled by the user-space part where possible. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* Userspace message context, copied in/out by kernel IPC thread. */ +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg *msg_reply; + +__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_COMP_CHAIN_DMA + struct ipc_comp_dev *cdma_comp; + struct ipc *ipc = ipc_get(); + const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; + int comp_id; + int ret; + + comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + + if (!cdma_comp) { + /* + * Nothing to do when the chainDMA is not allocated and asked to + * be freed + */ + if (!cdma->primary.r.allocate && !cdma->primary.r.enable) + return IPC4_SUCCESS; + + ret = ipc4_chain_manager_create(cdma); + if (ret < 0) + return IPC4_FAILURE; + + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + if (!cdma_comp) { + return IPC4_FAILURE; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) { + comp_free(cdma_comp->cd); + list_item_del(&cdma_comp->list); + sof_heap_free(sof_sys_user_heap_get(), cdma_comp); + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) + return IPC4_INVALID_CHAIN_STATE_TRANSITION; + + return IPC4_SUCCESS; +#else /* CONFIG_COMP_CHAIN_DMA */ + return IPC4_UNAVAILABLE; +#endif /* CONFIG_COMP_CHAIN_DMA */ +} + + +__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_IPC4_GATEWAY + struct ipc *ipc = ipc_get(); + uint32_t reply_size = 0; + int err; + + err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, + &reply_size); + /* reply size is returned in header extension dword */ + msg_reply->extension = reply_size; + + if (reply_size > 0) { + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; + } + + return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; +#else /* CONFIG_IPC4_GATEWAY */ + ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); + return IPC4_UNAVAILABLE; +#endif /* CONFIG_IPC4_GATEWAY */ +} + + +static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) +{ +#if CONFIG_GDBSTUB + ipc_enter_gdb = true; + return IPC4_SUCCESS; +#else + return IPC4_UNAVAILABLE; +#endif +} + + +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_GLB_BOOT_CONFIG: + case SOF_IPC4_GLB_ROM_CONTROL: + case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: + case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: + case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_CHAIN_DMA: + ret = ipc4_process_chain_dma(ipc4); + break; + + /* pipeline settings */ + case SOF_IPC4_GLB_CREATE_PIPELINE: + ret = ipc4_route_create_pipeline(ipc4); + break; + case SOF_IPC4_GLB_DELETE_PIPELINE: + ret = ipc4_route_delete_pipeline(ipc4); + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + ret = ipc4_route_set_pipeline_state(ipc4); + break; + + case SOF_IPC4_GLB_GET_PIPELINE_STATE: + case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: + case SOF_IPC4_GLB_SAVE_PIPELINE: + case SOF_IPC4_GLB_RESTORE_PIPELINE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_INTERNAL_MESSAGE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + /* Notification (FW to SW driver) */ + case SOF_IPC4_GLB_NOTIFICATION: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_IPCGATEWAY_CMD: + ret = ipc4_process_ipcgtw_cmd(ipc4); + break; + + case SOF_IPC4_GLB_ENTER_GDB: + ret = ipc_glb_gdb_debug(ipc4); + break; + + default: + ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + } + + return ret; +} + + +/* + * Ipc4 Module message <------> ipc3 module message + * init module <-------> create component + * bind modules <-------> connect components + * module set_large_config <-------> component cmd + * delete module <-------> free component + */ + +__cold int ipc4_init_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *module_init = + (const struct ipc4_module_init_instance *)ipc4; + struct comp_dev *dev; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + + /* Pass IPC to target core */ + if (!cpu_is_me(module_init->extension.r.core_id)) + return ipc4_process_on_core(module_init->extension.r.core_id, false); + + dev = comp_new_ipc4(module_init); + if (!dev) { + ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + return IPC4_MOD_NOT_INITIALIZED; + } + + return 0; +} + + +__cold int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_large_config_reply reply; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + char *data = ipc_get()->comp_data; + const struct comp_driver *drv; + struct comp_dev *dev = NULL; + uint32_t data_offset; + int ret; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + /* get component dev for non-basefw since there is no + * component dev for basefw + */ + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.get_large_config) + return IPC4_INVALID_REQUEST; + + data_offset = config->extension.r.data_off_size; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + /* data_off_size is a 20-bit host-controlled field, so it can + * claim far more than the hostbox can physically hold. + */ + if (data_offset > MAILBOX_HOSTBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", + data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + /* For now only vendor_config case uses payload from hostbox */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + ret = ipc4_get_vendor_config_module_instance(dev, drv, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + data += sizeof(reply); +#endif + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, + data, data_offset); + + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data); + } + + /* set up ipc4 error code for reply data */ + if (ret < 0) + ret = IPC4_INVALID_RESOURCE_ID; + + /* Copy host config and overwrite */ + reply.extension.dat = config->extension.dat; + reply.extension.r.data_off_size = data_offset; + + /* The last block, no more data */ + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + reply.extension.r.final_block = 1; + + /* Indicate last block if error occurs */ + if (ret) + reply.extension.r.final_block = 1; + + /* no need to allocate memory for reply msg */ + if (ret) + return ret; + + /* A module must not report more reply data than the DSP mailbox can hold. */ + if (data_offset > MAILBOX_DSPBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; + return ret; +} + + +__cold int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + struct comp_dev *dev = NULL; + const struct comp_driver *drv; + int ret; + + assert_can_be_cold(); + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.set_large_config) + return IPC4_INVALID_REQUEST; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + ret = ipc4_set_vendor_config_module_instance(dev, drv, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + struct ipc *ipc = ipc_get(); + const char *data = (const char *)ipc->comp_data + sizeof(*config); +#else + const char *data = (const char *)MAILBOX_HOSTBOX_BASE; +#endif + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + } + + return ret; +} + + +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_MOD_INIT_INSTANCE: + ret = ipc4_route_init_module(ipc4); + break; + case SOF_IPC4_MOD_CONFIG_GET: + ret = ipc4_route_module_config(ipc4, false, msg_reply); + break; + case SOF_IPC4_MOD_CONFIG_SET: + ret = ipc4_route_module_config(ipc4, true, msg_reply); + break; + case SOF_IPC4_MOD_LARGE_CONFIG_GET: + ret = ipc4_route_large_config_get(ipc4, msg_reply); + break; + case SOF_IPC4_MOD_LARGE_CONFIG_SET: + ret = ipc4_route_large_config_set(ipc4); + break; + case SOF_IPC4_MOD_BIND: + ret = ipc4_route_bind(ipc4); + break; + case SOF_IPC4_MOD_UNBIND: + ret = ipc4_route_unbind(ipc4); + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: + ret = ipc4_route_delete_module(ipc4); + break; + case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: + case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: + ret = IPC4_UNAVAILABLE; + break; + default: + ret = IPC4_UNKNOWN_MESSAGE_TYPE; + break; + } + + return ret; +} + diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c new file mode 100644 index 000000000000..2e8f5ee40474 --- /dev/null +++ b/src/kernel/ipc/ipc4/helper-kernel.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_SOC_SERIES_INTEL_ADSP_CAVS) +#define RIMAGE_MANIFEST 1 +#endif + +#ifdef RIMAGE_MANIFEST +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +/* Kernel part of the IPC4 helpers: the component driver lookup, which reads the + * IMR manifest and the driver list, module creation in kernel mode, IDC to other + * cores and the chain DMA manager. The part that also runs in user mode is in + * src/ipc/ipc4/helper.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx comp_tr; + +static const struct comp_driver *ipc4_get_drv(const void *uuid); + +#if CONFIG_LIBRARY +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) +{ + return ipc4_get_drv(data); +} +#endif /* CONFIG_LIBRARY */ + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + +#ifdef CONFIG_DCACHE_LINE_SIZE + if (!IS_ENABLED(CONFIG_LIBRARY)) + sys_cache_data_invd_range((__sparse_force void __sparse_cache *) + MAILBOX_HOSTBOX_BASE, + ALIGN_UP(ipc_config.ipc_config_size, + CONFIG_DCACHE_LINE_SIZE)); +#endif + data = ipc4_get_comp_new_data(); +#if CONFIG_LIBRARY + ipc_config.ipc_config_size -= sizeof(struct sof_uuid); + drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); +#else + drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); +#endif + if (!drv) + return NULL; + + if (!drv->ops.create) { + tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, + (uint32_t)IPC4_MOD_ID(comp_id)); + return NULL; + } + + if (drv->type == SOF_COMP_MODULE_ADAPTER) { + const struct ipc_config_process spec = { + .data = data, + .size = ipc_config.ipc_config_size, + }; + + dev = drv->ops.create(drv, &ipc_config, &spec); + } else { + dev = drv->ops.create(drv, &ipc_config, data); + } + if (!dev) + return NULL; + + list_init(&dev->bsource_list); + list_init(&dev->bsink_list); + +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + /* init global performance measurement */ + dev->perf_data.perf_data_item = perf_data_getnext(); + /* this can be null, just no performance measurements in this case */ + if (dev->perf_data.perf_data_item) { + dev->perf_data.perf_data_item->item.resource_id = comp_id; + if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) + comp_init_performance_data(dev); + } +#endif + + ipc4_add_comp_dev(dev); + + comp_update_ibs_obs_cpc(dev); + + return dev; +} + +#ifdef CONFIG_SOF_USERSPACE_LL +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * A large part of module instantiation is executed in the userspace + * mode, including calling the module's .init() method. But another part + * of it has to be executed in the kernel mode, including calling + * lib_manager_mod_create_priv(). This is the privileged part of module + * instantiation. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_comp_free_remote(struct comp_dev *dev) +{ + struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), + dev->ipc_config.core,}; + + assert_can_be_cold(); + + return idc_send_msg(&msg, IDC_BLOCKING); +} + +#if CONFIG_COMP_CHAIN_DMA +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) +{ + const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); + const struct comp_driver *drv; + struct comp_dev *dev; + + assert_can_be_cold(); + + drv = ipc4_get_drv(&uuid); + if (!drv) + return -EINVAL; + + dev = drv->ops.create(drv, NULL, cdma); + if (!dev) + return -EINVAL; + + /* differentiate instance by unique ids assignment */ + const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + + IPC4_MAX_MODULE_COUNT, 0); + dev->ipc_config.id = comp_id; + dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; + + return ipc4_add_comp_dev(dev); +} + +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) +{ + const bool allocate = cdma->primary.r.allocate; + const bool enable = cdma->primary.r.enable; + struct ipc *ipc = ipc_get(); + struct ipc_comp_dev *icd; + struct list_item *clist, *_tmp; + int ret; + + assert_can_be_cold(); + + if (!dev) + return -EINVAL; + + if (allocate) { + if (enable) + ret = comp_trigger(dev, COMP_TRIGGER_START); + else + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + } else { + if (enable) + return -EINVAL; + + /* remove chain part */ + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + if (ret < 0) + return ret; + + list_for_item_safe(clist, _tmp, &ipc->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->cd != dev) + continue; + list_item_del(&icd->list); + sof_heap_free(sof_sys_user_heap_get(), icd); + break; + } + comp_free(dev); + } + return ret; +} +#endif /* CONFIG_COMP_CHAIN_DMA */ + +int ipc4_process_on_core(uint32_t core, bool blocking) +{ + int ret; + + ret = ipc_process_on_core(core, blocking); + switch (ret) { + case 0: + case 1: + return IPC4_SUCCESS; + case -EACCES: + return IPC4_INVALID_CORE_ID; + case -ETIME: + case -EBUSY: + return IPC4_BUSY; + default: + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; +} + +__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) +{ + struct comp_driver_list *drivers = comp_drivers_get(); + struct list_item *clist; + const struct comp_driver *drv = NULL; + struct comp_driver_info *info; + + assert_can_be_cold(); + + /* search driver list with UUID */ + list_for_item(clist, &drivers->list) { + info = container_of(clist, struct comp_driver_info, list); + if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, SOF_DRV_UID_NAME(info->drv)); + drv = info->drv; + break; + } + } + + return drv; +} + +__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) +{ + const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; + const struct comp_driver *drv; + + assert_can_be_cold(); + + drv = ipc4_search_for_drv(uuid); + if (!drv) + tr_err(&comp_tr, + "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", + sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], + sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], + sof_uuid->d[6], sof_uuid->d[7]); + + return drv; +} + +#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) +/* + * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. + * + * native_sim fuzz builds have no rimage manifest, so the manifest lookup in + * ipc4_get_comp_drv() can never resolve a driver and every module instance + * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, + * DELETE_INSTANCE) would be unreachable from the fuzzer. + * + * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a + * 1-based index into the runtime component driver list, counting only + * instantiable module-adapter drivers so the fuzzer's module_id space matches + * what a real signed manifest would expose. + */ +static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) +{ + static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); + struct comp_driver_list *dlist = comp_drivers_get(); + struct list_item *iter; + uint32_t idx = 0; + + if (!module_id) + return ipc4_search_for_drv(&basefw_reg_uuid); + + list_for_item(iter, &dlist->list) { + struct comp_driver_info *inf = + container_of(iter, struct comp_driver_info, list); + + /* + * Skip query-only entries (e.g. BaseFW) that cannot be + * instantiated because they have no create op. + */ + if (!inf->drv->ops.create) + continue; + + /* + * A real signed manifest only lists module-adapter modules. + * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) + * are registered for IPC3 but are never IPC4 modules: on the + * IPC4 path they receive no params() pass and cannot be + * configured (e.g. their DMA buffer is never allocated). Skip + * non-module-adapter drivers so the fuzzer's module_id space + * matches a real manifest and cannot map to an unconfigurable + * component. + */ + if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) + continue; + + if (++idx == module_id) + return inf->drv; + } + + tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", + module_id, idx); + return NULL; +} +#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ + +/* + * Called from + * - ipc4_get_large_config_module_instance() + * - ipc4_set_large_config_module_instance() + * - comp_new_ipc4() + * all of which are __cold + */ +__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) +{ + const struct sof_man_fw_desc *desc = NULL; + const struct sof_man_module *mod; + uint32_t entry_index; + + assert_can_be_cold(); + +#ifdef RIMAGE_MANIFEST + desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; +#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) + return ipc4_get_fuzzer_drv(module_id); +#else + /* Non-rimage platforms have no component facility yet. + * This needs to move to the platform layer. + */ + return NULL; +#endif + + uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); + + if (lib_idx == 0) { + /* module_id 0 is used for base fw which is in entry 1 or 2 */ + if (!module_id) + entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); + else + entry_index = module_id; + + if (entry_index >= desc->header.num_module_entries) { + tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); + return NULL; + } + + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(entry_index)); + } else { + /* Library index greater than 0 possible only when LIBRARY_MANAGER + * support enabled. + */ +#if CONFIG_LIBRARY_MANAGER + mod = lib_manager_get_module_manifest(module_id); + + if (!mod) { + tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", + module_id); + return NULL; + } +#else + tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", + lib_idx); + return NULL; +#endif + } + +#if CONFIG_LIBRARY_MANAGER + /* Check already registered components */ + const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); + + if (drv) + return drv; + + /* New module not registered yet. */ + lib_manager_register_module(module_id); +#endif + return ipc4_get_drv(&mod->uuid); +} diff --git a/test/cmocka/src/audio/buffer/CMakeLists.txt b/test/cmocka/src/audio/buffer/CMakeLists.txt index cb60f07aa22e..805aaa2da784 100644 --- a/test/cmocka/src/audio/buffer/CMakeLists.txt +++ b/test/cmocka/src/audio/buffer/CMakeLists.txt @@ -14,6 +14,7 @@ cmocka_test(buffer_copy ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -38,6 +39,7 @@ cmocka_test(buffer_new ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -62,6 +64,7 @@ cmocka_test(buffer_wrap ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -86,6 +89,7 @@ cmocka_test(buffer_write ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/component/CMakeLists.txt b/test/cmocka/src/audio/component/CMakeLists.txt index b82756073b25..50389cfe4535 100644 --- a/test/cmocka/src/audio/component/CMakeLists.txt +++ b/test/cmocka/src/audio/component/CMakeLists.txt @@ -8,6 +8,7 @@ cmocka_test(comp_set_state ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/dcblock/CMakeLists.txt b/test/cmocka/src/audio/dcblock/CMakeLists.txt index 9c7f58e8feed..0e5754b022d5 100644 --- a/test/cmocka/src/audio/dcblock/CMakeLists.txt +++ b/test/cmocka/src/audio/dcblock/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(audio_for_dcblock STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_fir/CMakeLists.txt b/test/cmocka/src/audio/eq_fir/CMakeLists.txt index 0bb81cc57b0a..485d1996dd8d 100644 --- a/test/cmocka/src/audio/eq_fir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_fir/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(audio_for_eq_fir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_iir/CMakeLists.txt b/test/cmocka/src/audio/eq_iir/CMakeLists.txt index 5087e77b4b83..53a550b6b6cf 100644 --- a/test/cmocka/src/audio/eq_iir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_iir/CMakeLists.txt @@ -40,6 +40,7 @@ add_library(audio_for_eq_iir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/mixer/CMakeLists.txt b/test/cmocka/src/audio/mixer/CMakeLists.txt index a6c1cdb1a78f..b0b086816e3b 100644 --- a/test/cmocka/src/audio/mixer/CMakeLists.txt +++ b/test/cmocka/src/audio/mixer/CMakeLists.txt @@ -9,6 +9,7 @@ cmocka_test(mixer ${PROJECT_SOURCE_DIR}/src/audio/mixer/mixer_hifi3.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/src/audio/module_adapter/module_adapter.c diff --git a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt index 4dca80136e68..ee72311e004b 100644 --- a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt +++ b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt @@ -19,6 +19,7 @@ if(CONFIG_FORMAT_FLOAT) ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/pipeline/CMakeLists.txt b/test/cmocka/src/audio/pipeline/CMakeLists.txt index 628398576a43..27d27757d6ae 100644 --- a/test/cmocka/src/audio/pipeline/CMakeLists.txt +++ b/test/cmocka/src/audio/pipeline/CMakeLists.txt @@ -6,6 +6,7 @@ cmocka_test(pipeline_new ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -31,6 +32,7 @@ cmocka_test(pipeline_connect_upstream ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -56,6 +58,7 @@ cmocka_test(pipeline_free ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/selector/CMakeLists.txt b/test/cmocka/src/audio/selector/CMakeLists.txt index 8ea8d969ae7c..eb8f742cc5f5 100644 --- a/test/cmocka/src/audio/selector/CMakeLists.txt +++ b/test/cmocka/src/audio/selector/CMakeLists.txt @@ -26,6 +26,7 @@ add_library(audio_for_selector STATIC ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c diff --git a/test/cmocka/src/audio/volume/CMakeLists.txt b/test/cmocka/src/audio/volume/CMakeLists.txt index d5e1a3ecb757..7a6dc7b2c854 100644 --- a/test/cmocka/src/audio/volume/CMakeLists.txt +++ b/test/cmocka/src/audio/volume/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(audio_for_volume STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/math/fft/CMakeLists.txt b/test/cmocka/src/math/fft/CMakeLists.txt index f8a0510f7027..79234b7b12f9 100644 --- a/test/cmocka/src/math/fft/CMakeLists.txt +++ b/test/cmocka/src/math/fft/CMakeLists.txt @@ -18,6 +18,7 @@ cmocka_test(fft ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/ztest/unit/audio/drc/CMakeLists.txt b/test/ztest/unit/audio/drc/CMakeLists.txt index 2faef72b825c..39691eefa371 100644 --- a/test/ztest/unit/audio/drc/CMakeLists.txt +++ b/test/ztest/unit/audio/drc/CMakeLists.txt @@ -61,6 +61,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/lib/objpool.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c diff --git a/test/ztest/unit/math/fft/CMakeLists.txt b/test/ztest/unit/math/fft/CMakeLists.txt index 920f41bdb642..0c7a8f13783f 100644 --- a/test/ztest/unit/math/fft/CMakeLists.txt +++ b/test/ztest/unit/math/fft/CMakeLists.txt @@ -65,6 +65,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c ${SOF_ROOT}/src/audio/pipeline/pipeline-params.c