LL userspace: enable cross-core DP pipelines

app/ll_userspace_overlay.conf

@@ -41,7 +41,6 @@ CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n
 CONFIG_SOF_BOOT_TEST_ALLOWED=n
 
 # misc features not yet compatible with user-space
-CONFIG_CROSS_CORE_STREAM=n
 CONFIG_INTEL_ADSP_MIC_PRIVACY=n
 CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
 

src/audio/pipeline/pipeline-graph.c

@@ -285,9 +285,12 @@ static void buffer_set_comp(struct comp_buffer *buffer, struct comp_dev *comp,
  * lock via user_ll_lock_sched()/ll_block() while modifying pipeline
  * connections, which provides mutual exclusion with the LL thread. No
  * additional lock is taken here; instead assert that the lock is held.
+ * That's the lock of the pipeline core, whose LL thread walks the buffer
+ * lists of the pipeline modules. Not of the buffer core: cross-core
+ * buffers belong to the primary core.
  */
 #define PPL_LOCK_DECLARE
-#define PPL_LOCK(x) user_ll_assert_locked(x)
+#define PPL_LOCK(comp) user_ll_assert_locked(comp_ppl_core(comp))
 #define PPL_UNLOCK()
 #else
 /*
@@ -314,7 +317,7 @@ __cold int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer,
 	else
 		comp_info(comp, "connect buffer %d as source", buf_get_id(buffer));
 
-	PPL_LOCK(buffer->core);
+	PPL_LOCK(comp);
 
 	comp_list = comp_buffer_list(comp, dir);
 	ret = buffer_attach(buffer, comp_list, dir);
@@ -343,7 +346,7 @@ __cold void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffe
 	else
 		comp_dbg(comp, "disconnect buffer %d as source", buf_get_id(buffer));
 
-	PPL_LOCK(buffer->core);
+	PPL_LOCK(comp);
 
 	comp_list = comp_buffer_list(comp, dir);
 	buffer_detach(buffer, comp_list, dir);

src/audio/pipeline/static_pipeline_loader.c

@@ -564,6 +564,9 @@ static int sof_static_topology_init_comps(const struct sof_static_topology *topo
 			.id = cdesc->id,
 			.pipeline_id = cdesc->pipeline_id,
 			.core = comp_core,
+#if CONFIG_IPC_MAJOR_4
+			.ppl_core = comp_core,
+#endif
 			.proc_domain = COMP_PROCESSING_DOMAIN_LL,
 			.frame_fmt = cdesc->caps.default_fmt,
 			.type = drv_type,

src/idc/idc.c

@@ -84,51 +84,39 @@ static void idc_ipc(void)
 	ipc_cmd(ipc->comp_data);
 }
 
-static int idc_ipc4_bind(uint32_t comp_id)
+static int idc_ipc4_bind(uint32_t comp_id, void *payload)
 {
 	struct ipc_comp_dev *ipc_dev;
-	struct idc_payload *payload;
-	struct bind_info *bu;
+	struct bind_info *bu = payload;
 
 	ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id);
 	if (!ipc_dev)
 		return -ENODEV;
 
-	payload = idc_payload_get(*idc_get(), cpu_get_id());
-	bu = (struct bind_info *)payload;
-
 	return comp_bind(ipc_dev->cd, bu);
 }
 
-static int idc_ipc4_unbind(uint32_t comp_id)
+static int idc_ipc4_unbind(uint32_t comp_id, void *payload)
 {
 	struct ipc_comp_dev *ipc_dev;
-	struct idc_payload *payload;
-	struct bind_info *bu;
+	struct bind_info *bu = payload;
 
 	ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id);
 	if (!ipc_dev)
 		return -ENODEV;
 
-	payload = idc_payload_get(*idc_get(), cpu_get_id());
-	bu = (struct bind_info *)payload;
-
 	return comp_unbind(ipc_dev->cd, bu);
 }
 
-static int idc_get_attribute(uint32_t comp_id)
+static int idc_get_attribute(uint32_t comp_id, void *payload)
 {
 	struct ipc_comp_dev *ipc_dev;
-	struct idc_payload *idc_payload;
-	struct get_attribute_remote_payload *get_attr_payload;
+	struct get_attribute_remote_payload *get_attr_payload = payload;
 
 	ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id);
 	if (!ipc_dev)
 		return -ENODEV;
 
-	idc_payload = idc_payload_get(*idc_get(), cpu_get_id());
-	get_attr_payload = (struct get_attribute_remote_payload *)idc_payload;
-
 	return comp_get_attribute(ipc_dev->cd, get_attr_payload->type, get_attr_payload->value);
 }
 
@@ -137,14 +125,11 @@ static int idc_get_attribute(uint32_t comp_id)
  * \param[in] comp_id Component id to have params set.
  * \return Error code.
  */
-static int idc_params(uint32_t comp_id)
+static int idc_params(uint32_t comp_id, void *payload)
 {
 	struct ipc *ipc = ipc_get();
 	struct ipc_comp_dev *ipc_dev;
-	struct idc *idc = *idc_get();
-	struct idc_payload *payload = idc_payload_get(idc, cpu_get_id());
-	struct sof_ipc_stream_params *params =
-		(struct sof_ipc_stream_params *)payload;
+	struct sof_ipc_stream_params *params = payload;
 	int ret;
 
 	ipc_dev = ipc_get_comp_by_id(ipc, comp_id);
@@ -185,6 +170,12 @@ static int idc_prepare(uint32_t comp_id)
 
 	/* we're running LL on different core, so allocate our own task */
 	if (!dev->task && dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) {
+#ifdef CONFIG_SOF_USERSPACE_LL
+		/* user-space LL modules must run on their pipeline core */
+		tr_err(&idc_tr, "comp %#x: LL module on a remote core", comp_id);
+		ret = -EINVAL;
+		goto out;
+#endif
 		/* allocate task for shared component */
 		dev->task = sof_heap_alloc(dev->drv->user_heap, SOF_MEM_FLAG_USER,
 					   sizeof(*dev->task), 0);
@@ -218,12 +209,10 @@ static int idc_prepare(uint32_t comp_id)
  * \param[in] comp_id Component id to be triggered.
  * \return Error code.
  */
-static int idc_trigger(uint32_t comp_id)
+static int idc_trigger(uint32_t comp_id, void *payload)
 {
 	struct ipc *ipc = ipc_get();
 	struct ipc_comp_dev *ipc_dev;
-	struct idc *idc = *idc_get();
-	struct idc_payload *payload = idc_payload_get(idc, cpu_get_id());
 	uint32_t cmd = *(uint32_t *)payload;
 
 	ipc_dev = ipc_get_comp_by_id(ipc, comp_id);
@@ -356,6 +345,7 @@ static void idc_secondary_core_crashed(const uint32_t header)
  */
 void idc_cmd(struct idc_msg *msg)
 {
+	void *payload = idc_payload_get(*idc_get(), cpu_get_id());
 	uint32_t type = iTS(msg->header);
 	int ret = 0;
 
@@ -377,25 +367,25 @@ void idc_cmd(struct idc_msg *msg)
 		dbg_path_hot_stop_watching();
 		break;
 	case iTS(IDC_MSG_BIND):
-		ret = idc_ipc4_bind(msg->extension);
+		ret = idc_ipc4_bind(msg->extension, payload);
 		break;
 	case iTS(IDC_MSG_UNBIND):
-		ret = idc_ipc4_unbind(msg->extension);
+		ret = idc_ipc4_unbind(msg->extension, payload);
 		break;
 	case iTS(IDC_MSG_GET_ATTRIBUTE):
-		ret = idc_get_attribute(msg->extension);
+		ret = idc_get_attribute(msg->extension, payload);
 		break;
 	case iTS(IDC_MSG_FREE):
 		ret = idc_comp_free(msg->extension);
 		break;
 	case iTS(IDC_MSG_PARAMS):
-		ret = idc_params(msg->extension);
+		ret = idc_params(msg->extension, payload);
 		break;
 	case iTS(IDC_MSG_PREPARE):
 		ret = idc_prepare(msg->extension);
 		break;
 	case iTS(IDC_MSG_TRIGGER):
-		ret = idc_trigger(msg->extension);
+		ret = idc_trigger(msg->extension, payload);
 		break;
 	case iTS(IDC_MSG_RESET):
 		ret = idc_reset(msg->extension);
@@ -421,6 +411,42 @@ void idc_cmd(struct idc_msg *msg)
 	idc_msg_status_set(ret, cpu_get_id());
 }
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+/**
+ * \brief Executes a component IDC message in a user-space thread.
+ * \param[in] msg IDC message, forwarded by a user thread on another core.
+ * \param[in,out] payload Message payload.
+ * \return Error code.
+ *
+ * Only the component messages, that can be sent from the user-space IPC
+ * and LL threads, are supported.
+ */
+int idc_cmd_user(const struct idc_msg *msg, void *payload)
+{
+	switch (iTS(msg->header)) {
+	case iTS(IDC_MSG_BIND):
+		return idc_ipc4_bind(msg->extension, payload);
+	case iTS(IDC_MSG_UNBIND):
+		return idc_ipc4_unbind(msg->extension, payload);
+	case iTS(IDC_MSG_GET_ATTRIBUTE):
+		return idc_get_attribute(msg->extension, payload);
+	case iTS(IDC_MSG_FREE):
+		return idc_comp_free(msg->extension);
+	case iTS(IDC_MSG_PARAMS):
+		return idc_params(msg->extension, payload);
+	case iTS(IDC_MSG_PREPARE):
+		return idc_prepare(msg->extension);
+	case iTS(IDC_MSG_TRIGGER):
+		return idc_trigger(msg->extension, payload);
+	case iTS(IDC_MSG_RESET):
+		return idc_reset(msg->extension);
+	default:
+		tr_err(&idc_tr, "unsupported user msg->header = %#x", msg->header);
+		return -EINVAL;
+	}
+}
+#endif /* CONFIG_SOF_USERSPACE_LL */
+
 /* Runs on each CPU */
 __cold int idc_init(void)
 {

src/idc/zephyr_idc.c

@@ -127,7 +127,7 @@ static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT * 2];
 /* Protect the above array */
 static K_MUTEX_DEFINE(idc_mutex);
 
-int idc_send_msg(struct idc_msg *msg, uint32_t mode)
+static int idc_send_msg_kernel(struct idc_msg *msg, uint32_t mode)
 {
 	struct idc *idc = *idc_get();
 	struct idc_payload *payload = idc_payload_get(idc, msg->core);
@@ -200,6 +200,21 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode)
 	return ret;
 }
 
+int idc_send_msg(struct idc_msg *msg, uint32_t mode)
+{
+#ifdef CONFIG_SOF_USERSPACE_LL
+	/*
+	 * IDC uses kernel objects and data. User-space threads instead pass
+	 * the message to the user-space IPC thread of the target core. Such
+	 * messages are always processed synchronously.
+	 */
+	if (k_is_user_context())
+		return ipc_user_xcore_send(msg);
+#endif
+
+	return idc_send_msg_kernel(msg, mode);
+}
+
 __cold void idc_init_thread(void)
 {
 	char thread_name[] = "idc_p4wq0";

src/include/sof/audio/component.h

@@ -571,6 +571,7 @@ struct comp_ipc_config {
 #if CONFIG_IPC_MAJOR_4
 	bool ipc_extended_init;		/**< True if extended init is included in IPC payload */
 	uint32_t ipc_config_size;	/**< Size of config received by IPC */
+	uint32_t ppl_core;		/**< Core of the parent pipeline */
 #endif
 };
 

src/include/sof/audio/component_ext.h

@@ -18,8 +18,9 @@
 
 #include <sof/audio/component.h>
 #include <ipc/topology.h>
-/* TODO: the cross-core helpers below send IDC messages, which is kernel only */
+/* in user-space LL builds idc_send_msg() forwards to the remote user IPC thread */
 #include <rtos/idc.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <sof/schedule/schedule.h>
 
 /** \addtogroup component_infra_api
@@ -299,9 +300,18 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t
 	if (type != COMP_ATTR_BASE_CONFIG)
 		return -EINVAL;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	/* must be writable by the user-space thread on the remote core */
+	base_cfg = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
+				  sizeof(*base_cfg), 0);
+	if (!base_cfg)
+		return -ENOMEM;
+	memset(base_cfg, 0, sizeof(*base_cfg));
+#else
 	base_cfg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*base_cfg));
 	if (!base_cfg)
 		return -ENOMEM;
+#endif
 
 	payload.type = type;
 	payload.value = base_cfg;
@@ -312,7 +322,11 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t
 		memcpy_s(value, sizeof(struct ipc4_base_module_cfg),
 			 base_cfg, sizeof(struct ipc4_base_module_cfg));
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(zephyr_ll_user_heap(), base_cfg);
+#else
 	rfree(base_cfg);
+#endif
 	return ret;
 }
 #endif	/* CONFIG_IPC_MAJOR_4 */
@@ -501,6 +515,23 @@ static inline void comp_make_shared(struct comp_dev *dev)
 	dev->is_shared = true;
 }
 
+/**
+ * \brief Get the core of the parent pipeline of a component.
+ *
+ * Use this instead of dev->pipeline->core on other cores than the pipeline
+ * core: struct pipeline isn't coherent between cores, struct comp_dev is.
+ * \param[in] dev Component device.
+ * \return Pipeline core, or the component core if it has no pipeline.
+ */
+static inline uint32_t comp_ppl_core(const struct comp_dev *dev)
+{
+#if CONFIG_IPC_MAJOR_4
+	return dev->ipc_config.ppl_core;
+#else
+	return dev->ipc_config.core;
+#endif
+}
+
 /**
  * \brief Get the global component driver list.
  * \return Pointer to the component driver list.

src/include/sof/ipc/common.h

@@ -62,6 +62,31 @@ extern struct tr_ctx ipc_tr;
 #define IPC_DSP_CORE_COUNT 1
 #endif
 
+/** @brief Maximum payload size of a cross-core request between user threads */
+#define IPC_USER_XCORE_PAYLOAD_SIZE	128
+
+/**
+ * @brief Cross-core request slot of a user-space IPC thread.
+ *
+ * User-space threads can't use IDC. Instead, they pass IDC messages for
+ * another core to the user-space IPC thread of that core via this slot.
+ */
+struct ipc_user_xcore {
+	/** @brief Serializes senders to this core */
+	struct k_sem *lock;
+	/** @brief Signalled by the target core when the request is done */
+	struct k_sem *done;
+	/** @brief IDC message header and extension */
+	uint32_t header;
+	uint32_t extension;
+	/** @brief Result from the target core */
+	int result;
+	/** @brief Copy of IPC4 bind / unbind message, referenced by the payload */
+	uint32_t ipc4_bind_data[2];
+	/** @brief Copy of the IDC message payload */
+	uint8_t payload[IPC_USER_XCORE_PAYLOAD_SIZE] __aligned(8);
+};
+
 struct ipc_user {
 	struct k_thread *thread[IPC_DSP_CORE_COUNT];
 	struct k_sem *sem;
@@ -96,6 +121,8 @@ struct ipc_user {
 	 * Size verified by BUILD_ASSERT in handler-user.c.
 	 */
 	uint8_t init_drv_data[160] __aligned(4);
+	/** @brief Cross-core request slots, one per target core */
+	struct ipc_user_xcore xcore[IPC_DSP_CORE_COUNT];
 };
 
 struct ipc {