Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/audio/module_adapter/module/generic.c
Original file line number Diff line number Diff line change
Expand Up @@ -1030,8 +1030,12 @@ __cold int module_unbind(struct processing_module *mod, const struct bind_info *

uint32_t module_get_deadline(struct processing_module *mod)
{
/* a DP module can write mod: read the count once and bound it */
size_t num_of_sinks = mod->num_of_sinks;
uint32_t deadline;

num_of_sinks = MIN(num_of_sinks, ARRAY_SIZE(mod->sinks));

/* LL modules have no deadline - it is always "now" */
if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL)
return 0;
Expand All @@ -1042,9 +1046,14 @@ uint32_t module_get_deadline(struct processing_module *mod)

deadline = UINT32_MAX;
/* calculate the shortest LFT for all sinks */
for (size_t i = 0; i < mod->num_of_sinks; i++) {
uint32_t sink_lft = sink_get_last_feeding_time(mod->sinks[i]);
for (size_t i = 0; i < num_of_sinks; i++) {
struct sof_sink *sink = mod->sinks[i];
uint32_t sink_lft;

if (!sink)
continue;

sink_lft = sink_get_last_feeding_time(sink);
deadline = MIN(deadline, sink_lft);
}

Expand Down
54 changes: 52 additions & 2 deletions src/include/sof/schedule/dp_schedule.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,10 @@ __syscall void scheduler_dp_internal_free(struct task *task);
/**
* @brief Tick notification from the Low-Latency (LL) timer domain to the DP scheduler.
*
* Invoked on each core after all LL tasks complete. Evaluates readiness for all
* queued DP tasks, updates their absolute deadlines, and signals worker threads.
* Evaluates readiness for all queued DP tasks, updates their absolute
* deadlines, and signals worker threads. Not available with
* CONFIG_SOF_USERSPACE_APPLICATION and CONFIG_SOF_USERSPACE_LL, see
* scheduler_dp_ll_post_tick().
*/
__syscall void scheduler_dp_ll_tick(void);

Expand All @@ -98,6 +100,36 @@ __syscall void scheduler_dp_ll_tick(void);
* @param[in] deadline_clock_ticks Absolute hardware cycle deadline of the run.
*/
__syscall void scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks);

/** @brief scheduler_dp_kick flag: signal the DP thread to process data */
#define SCHEDULER_DP_KICK_PROCESS BIT(0)
/** @brief scheduler_dp_kick flag: set the DP thread deadline */
#define SCHEDULER_DP_KICK_DEADLINE BIT(1)

/** @brief Maximum number of entries passed to one scheduler_dp_kick() call */
#define SCHEDULER_DP_KICK_MAX 8

/** @brief Result of the DP readiness and deadline calculation for one task */
struct scheduler_dp_kick {
const struct task *task; /**< DP task, only used to look up its record */
uint32_t flags; /**< SCHEDULER_DP_KICK_* */
uint32_t deadline; /**< deadline in cycles from now */
};

/**
* @brief Apply DP readiness and deadline results to the DP threads.
*
* The DP thread kernel objects are looked up from the kernel-only DP task
* records, never from DP task memory. Entries for unknown tasks and for
* tasks of other cores are skipped. Only the LL user thread may call this,
* or the LL scheduler in kernel mode without CONFIG_SOF_USERSPACE_LL.
*
* @param[in] kicks Array of results.
* @param[in] count Number of entries, at most SCHEDULER_DP_KICK_MAX.
* @return Number of skipped entries.
*/
__syscall int scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count);

#include <zephyr/syscalls/dp_schedule.h>
#else
static inline void scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks)
Expand All @@ -107,6 +139,24 @@ static inline void scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_
}
#endif

/**
* @brief Run the DP scheduler after an LL tick.
*
* Called by the LL scheduler on each core after all LL tasks complete. With
* CONFIG_SOF_USERSPACE_APPLICATION and CONFIG_SOF_USERSPACE_LL, the DP task
* memory is writable by the DP threads, so the calculation runs in the calling
* LL user thread, and only its results are passed to the kernel with
* scheduler_dp_kick(). Otherwise this calls scheduler_dp_ll_tick().
*/
void scheduler_dp_ll_post_tick(void);

#if CONFIG_SOF_BOOT_TEST_STANDALONE && CONFIG_SOF_USERSPACE_APPLICATION
/* Test hooks: add and remove kernel-only DP task records without a DP task */
int scheduler_dp_test_record_add(struct task *task, struct k_event *event,
struct k_thread *thread, uint16_t core);
void scheduler_dp_test_record_del(struct task *task);
#endif

/**
* @brief Extract telemetry information about Data Processing scheduler tasks.
*
Expand Down
22 changes: 22 additions & 0 deletions src/include/sof/schedule/zephyr_dp_schedule.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,28 @@ int scheduler_dp_task_cancel(void *data, struct task *task);
*/
void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch);

#if CONFIG_SOF_USERSPACE_APPLICATION
/**
* @brief Get the objects for an IPC to the DP thread of a module from the kernel-only record.
* @param[in] mod Module of the DP task.
* @param[out] event Event to post the IPC to.
* @param[out] flat IPC parameter area in the task memory.
* @param[out] config_data LARGE_CONFIG_GET reply buffer, or NULL.
* @param[out] core Core of the DP task.
* @return 0 on success, -EINVAL if @p mod has no DP task. Kernel context only.
*/
int scheduler_dp_task_ipc_params(const struct processing_module *mod, struct k_event **event,
struct ipc4_flat **flat, const void **config_data,
unsigned int *core);

/**
* @brief Count the DP tasks of a core from the kernel-only DP task records.
* @param[in] core DSP core.
* @return Number of scheduled DP tasks on @p core. Kernel context only.
*/
unsigned int scheduler_dp_task_count(unsigned int core);
#endif

/**
* @brief Worker thread entry point function for Data Processing tasks.
*
Expand Down
10 changes: 8 additions & 2 deletions src/include/sof/userspace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ syscall by subsystem:
| CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` |
| Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` |
| Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` |
| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` |
| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick`, `scheduler_dp_kick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` |
| Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | `kernel/debug/debug_stream/debug_stream_slot.c` |
| KPB | `kpb_notifier_schedule` | `sof/audio/kpb.h` | `zephyr/syscall/kpb.c` / `audio/kpb.c` |

Expand All @@ -139,9 +139,15 @@ from user-writable data such as `ipc_user->thread[]`.
|-----------------|----------|
| Any user thread, including DP module threads | `cpu_get_id`, all module management syscalls (`mod_*`), `ipc_msg_send`, `ipc_msg_list_remove`, `send_resource_notif`, `kpb_notifier_schedule` |
| IPC user thread only | `ipc_compound_pre_start`, `ipc_compound_post_start`, `ipc_wait_for_compound_msg`, `lib_manager_free_module` |
| LL user thread only | `scheduler_dp_ll_tick` |
| LL user thread only | `scheduler_dp_ll_tick`, `scheduler_dp_kick` |
| LL or IPC user thread | all other SOF syscalls |

With `CONFIG_SOF_USERSPACE_APPLICATION`, the DP task list and task memory are writable by DP
threads, so the kernel does not walk them. The DP readiness and deadline calculation runs in
the LL user thread (`scheduler_dp_ll_post_tick()`), which passes its results to
`scheduler_dp_kick()`; the kernel takes the DP thread and event objects from its own task
records. `scheduler_dp_ll_tick` is not built when `CONFIG_SOF_USERSPACE_LL` is also set.

DP modules only get the syscalls needed to implement the module interface. That includes
sending notifications: modules queue messages from `mod_ipc_msg_init()` with
`ipc_msg_send()` and free them with `mod_ipc_msg_free()`, and `dp_task_run()` reports
Expand Down
9 changes: 7 additions & 2 deletions src/kernel/schedule/zephyr_dp_schedule-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,12 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data)
* needed 1.2ms for processing - but the example would be too complicated)
*/

#ifdef CONFIG_USERSPACE
#include <zephyr/internal/syscall_handler.h>
#include <sof/userspace/syscall_caller.h>
#endif

#if !(CONFIG_SOF_USERSPACE_APPLICATION && CONFIG_SOF_USERSPACE_LL)
void z_impl_scheduler_dp_ll_tick(void)
{
unsigned int lock_key;
Expand All @@ -238,8 +244,6 @@ void z_impl_scheduler_dp_ll_tick(void)
}

#ifdef CONFIG_USERSPACE
#include <zephyr/internal/syscall_handler.h>
#include <sof/userspace/syscall_caller.h>
void z_vrfy_scheduler_dp_ll_tick(void)
{
SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL);
Expand All @@ -248,6 +252,7 @@ void z_vrfy_scheduler_dp_ll_tick(void)
}
#include <zephyr/syscalls/scheduler_dp_ll_tick_mrsh.c>
#endif
#endif /* !(CONFIG_SOF_USERSPACE_APPLICATION && CONFIG_SOF_USERSPACE_LL) */

void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks)
{
Expand Down
153 changes: 151 additions & 2 deletions src/kernel/schedule/zephyr_dp_schedule_application-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,20 @@ STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INI
* into the DP thread's memory domain, and is also writable by the LL and IPC
* user threads. Freeing must therefore never use the pointers stored there:
* scheduler_dp_internal_free() takes them from this record instead. The task
* pointer is only used as a lookup key.
* and module pointers are only used as lookup keys.
*/
struct scheduler_dp_task_kobj {
struct list_item list;
struct task *task;
struct processing_module *mod;
struct k_event *event;
struct k_thread *thread;
struct ipc4_flat *flat; /* IPC parameter area in task memory */
void *config_data; /* LARGE_CONFIG_GET reply buffer in task memory */
struct k_mem_domain *mdom;
struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT];
uint32_t comp_id;
uint16_t core;
uint32_t gen; /* unique per record, see dp_task_kobj_take() */
};

Expand Down Expand Up @@ -97,6 +100,29 @@ static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task)
return NULL;
}

/* Copy the record for @mod to @copy */
static bool dp_task_kobj_peek_mod(const struct processing_module *mod,
struct scheduler_dp_task_kobj *copy)
{
k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock);
struct scheduler_dp_task_kobj *kobj;
struct list_item *item;
bool found = false;

list_for_item(item, &dp_task_kobj_list) {
kobj = container_of(item, struct scheduler_dp_task_kobj, list);
if (kobj->mod == mod) {
*copy = *kobj;
found = true;
break;
}
}

k_spin_unlock(&dp_task_kobj_lock, key);

return found;
}

/* Copy the record for @task to @copy without unlinking it */
static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy)
{
Expand Down Expand Up @@ -291,6 +317,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid,

kobj->task = ptask;
kobj->mod = mod;
kobj->core = core;
comp_id = mod->dev->ipc_config.id;
kobj->comp_id = comp_id;

Expand All @@ -303,8 +330,10 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid,
}
}
pdata->ipc_config_data = ipc_config_data;
kobj->config_data = ipc_config_data;

pdata->flat = &task_memory->flat;
kobj->flat = &task_memory->flat;
pdata->flat = kobj->flat;

kobj->event = k_object_alloc(K_OBJ_EVENT);
if (!kobj->event) {
Expand Down Expand Up @@ -468,6 +497,111 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid,
return ret;
}

/*
* Get the objects needed to send an IPC to the DP thread of @mod from the
* kernel-only record. Returns -EINVAL if @mod has no DP task.
*/
int scheduler_dp_task_ipc_params(const struct processing_module *mod, struct k_event **event,
struct ipc4_flat **flat, const void **config_data,
unsigned int *core)
{
struct scheduler_dp_task_kobj kobj;

if (!dp_task_kobj_peek_mod(mod, &kobj))
return -EINVAL;

*event = kobj.event;
*flat = kobj.flat;
*config_data = kobj.config_data;
*core = kobj.core;

return 0;
}

int z_impl_scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count)
{
struct scheduler_dp_task_kobj *kobj;
unsigned int core = cpu_get_id();
uint32_t now = k_cycle_get_32();
k_spinlock_key_t key;
int skipped = 0;
uint32_t i;

/* the record cannot be freed while dp_task_kobj_lock is held */
key = k_spin_lock(&dp_task_kobj_lock);

for (i = 0; i < count; i++) {
kobj = dp_task_kobj_find(kicks[i].task);
if (!kobj || kobj->core != core) {
skipped++;
continue;
}

/*
* Overflows are OK. The deadline may be in the past, Zephyr
* still schedules the thread with the earliest deadline first.
*/
if (kicks[i].flags & SCHEDULER_DP_KICK_DEADLINE)
k_thread_absolute_deadline_set(kobj->thread, now + kicks[i].deadline);

if (kicks[i].flags & SCHEDULER_DP_KICK_PROCESS)
k_event_post(kobj->event, DP_TASK_EVENT_PROCESS);
}

k_spin_unlock(&dp_task_kobj_lock, key);

return skipped;
}

unsigned int scheduler_dp_task_count(unsigned int core)
{
k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock);
struct scheduler_dp_task_kobj *kobj;
struct list_item *item;
unsigned int count = 0;

/*
* Task memory is only freed after its record is unlinked, so it can be
* read here. Only the state value is used, no pointers from it: a
* forged state only changes whether that task is counted.
*/
list_for_item(item, &dp_task_kobj_list) {
kobj = container_of(item, struct scheduler_dp_task_kobj, list);
if (kobj->core == core && task_is_active(kobj->task))
count++;
}

k_spin_unlock(&dp_task_kobj_lock, key);

return count;
}

#if CONFIG_SOF_BOOT_TEST_STANDALONE
/* Add a DP task record for zephyr/test/userspace/test_dp_kick.c */
int scheduler_dp_test_record_add(struct task *task, struct k_event *event,
struct k_thread *thread, uint16_t core)
{
struct scheduler_dp_task_kobj *kobj = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT,
sizeof(*kobj));

if (!kobj)
return -ENOMEM;

kobj->task = task;
kobj->event = event;
kobj->thread = thread;
kobj->core = core;
dp_task_kobj_add(kobj);

return 0;
}

void scheduler_dp_test_record_del(struct task *task)
{
rfree(dp_task_kobj_take(task, NULL));
}
#endif

#ifdef CONFIG_USERSPACE
#include <zephyr/internal/syscall_handler.h>
#include <sof/userspace/syscall_caller.h>
Expand Down Expand Up @@ -533,4 +667,19 @@ void z_vrfy_scheduler_dp_internal_free(struct task *task)
K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj));
}
#include <zephyr/syscalls/scheduler_dp_internal_free_mrsh.c>

int z_vrfy_scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count)
{
struct scheduler_dp_kick kkicks[SCHEDULER_DP_KICK_MAX];

/* without user-space LL, only the kernel LL scheduler kicks DP tasks */
K_OOPS(!IS_ENABLED(CONFIG_SOF_USERSPACE_LL));
SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL);

K_OOPS(K_SYSCALL_VERIFY(count <= ARRAY_SIZE(kkicks)));
K_OOPS(k_usermode_from_copy(kkicks, kicks, count * sizeof(kkicks[0])));

return z_impl_scheduler_dp_kick(kkicks, count);
}
#include <zephyr/syscalls/scheduler_dp_kick_mrsh.c>
#endif /* CONFIG_USERSPACE */
Loading
Loading